23 #define DEBUG_TYPE "llvm-mca"
30 NextInSequenceMask &= (CandidateMask | (CandidateMask - 1));
36 uint64_t CandidateMask = ReadyMask & NextInSequenceMask;
38 return selectImpl(CandidateMask, NextInSequenceMask);
40 NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;
41 RemovedFromNextInSequence = 0;
42 CandidateMask = ReadyMask & NextInSequenceMask;
44 return selectImpl(CandidateMask, NextInSequenceMask);
46 NextInSequenceMask = ResourceUnitMask;
47 CandidateMask = ReadyMask & NextInSequenceMask;
48 return selectImpl(CandidateMask, NextInSequenceMask);
52 if (
Mask > NextInSequenceMask) {
53 RemovedFromNextInSequence |=
Mask;
57 NextInSequenceMask &= (~
Mask);
58 if (NextInSequenceMask)
61 NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;
62 RemovedFromNextInSequence = 0;
67 : ProcResourceDescIndex(
Index), ResourceMask(
Mask),
68 BufferSize(Desc.BufferSize), IsAGroup(
llvm::
popcount(ResourceMask) > 1) {
73 ResourceSizeMask = (1ULL << Desc.
NumUnits) - 1;
75 ReadyMask = ResourceSizeMask;
76 AvailableSlots = BufferSize == -1 ? 0U :
static_cast<unsigned>(BufferSize);
96 <<
", SZMASK=" <<
format_hex(ResourceSizeMask, 16)
98 <<
", BufferSize=" << BufferSize
99 <<
", AvailableSlots=" << AvailableSlots
100 <<
", Reserved=" << Unavailable <<
'\n';
104 static std::unique_ptr<ResourceStrategy>
107 return std::make_unique<DefaultResourceStrategy>(RS.
getReadyMask());
108 return std::unique_ptr<ResourceStrategy>(
nullptr);
112 : Resources(SM.getNumProcResourceKinds() - 1),
113 Strategies(SM.getNumProcResourceKinds() - 1),
114 Resource2Groups(SM.getNumProcResourceKinds() - 1, 0),
115 ProcResID2Mask(SM.getNumProcResourceKinds(), 0),
116 ResIndex2ProcResID(SM.getNumProcResourceKinds() - 1, 0),
117 ProcResUnitMask(0), ReservedResourceGroups(0), AvailableBuffers(~0ULL),
124 ResIndex2ProcResID[
Index] =
I;
140 ProcResUnitMask |=
Mask;
145 Mask -= GroupMaskIdx;
150 Resource2Groups[IndexUnit] |= GroupMaskIdx;
155 AvailableProcResUnits = ProcResUnitMask;
158 void ResourceManager::setCustomStrategyImpl(std::unique_ptr<ResourceStrategy>
S,
161 assert(
Index < Resources.size() &&
"Invalid processor resource index!");
162 assert(
S &&
"Unexpected null strategy in input!");
170 unsigned ResourceManager::getNumUnits(
uint64_t ResourceID)
const {
179 assert(
Index < Resources.size() &&
"Invalid resource use!");
180 ResourceState &RS = *Resources[
Index];
181 assert(RS.isReady() &&
"No available units to select!");
185 if (!RS.isAResourceGroup() && RS.getNumUnits() == 1)
186 return std::make_pair(ResourceID, RS.getReadyMask());
188 uint64_t SubResourceID = Strategies[
Index]->select(RS.getReadyMask());
189 if (RS.isAResourceGroup())
190 return selectPipe(SubResourceID);
191 return std::make_pair(ResourceID, SubResourceID);
197 ResourceState &RS = *Resources[RSID];
198 RS.markSubResourceAsUsed(RR.second);
201 if (RS.getNumUnits() > 1)
202 Strategies[RSID]->used(RR.second);
209 AvailableProcResUnits ^= RR.first;
216 ResourceState &CurrentUser = *Resources[GroupIndex];
217 CurrentUser.markSubResourceAsUsed(RR.first);
218 Strategies[GroupIndex]->used(RR.first);
224 void ResourceManager::release(
const ResourceRef &RR) {
226 ResourceState &RS = *Resources[RSID];
227 bool WasFullyUsed = !RS.isReady();
228 RS.releaseSubResource(RR.second);
232 AvailableProcResUnits ^= RR.first;
238 ResourceState &CurrentUser = *Resources[GroupIndex];
239 CurrentUser.releaseSubResource(RR.first);
246 if (ConsumedBuffers & ReservedBuffers)
248 if (ConsumedBuffers & (~AvailableBuffers))
254 while (ConsumedBuffers) {
255 uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);
257 ConsumedBuffers ^= CurrentBuffer;
259 if (!RS.reserveBuffer())
260 AvailableBuffers ^= CurrentBuffer;
261 if (RS.isADispatchHazard()) {
265 ReservedBuffers ^= CurrentBuffer;
271 AvailableBuffers |= ConsumedBuffers;
272 while (ConsumedBuffers) {
273 uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);
275 ConsumedBuffers ^= CurrentBuffer;
287 for (
const std::pair<uint64_t, ResourceUsage> &
E : Desc.
Resources) {
288 unsigned NumUnits =
E.second.isReserved() ? 0U :
E.second.NumUnits;
291 BusyResourceMask |=
E.first;
297 NumAvailableUnits -= NumUnits;
298 AvailableUnits[
E.first] = NumAvailableUnits;
299 if (!NumAvailableUnits)
300 ConsumedResourceMask |=
E.first;
304 BusyResourceMask &= ProcResUnitMask;
305 if (BusyResourceMask)
306 return BusyResourceMask;
310 return BusyResourceMask;
314 for (
const std::pair<uint64_t, ResourceUsage> &
E : Desc.
Resources) {
325 auto it = AvailableUnits.
find(ResourceMask);
326 if (
it == AvailableUnits.
end()) {
330 AvailableUnits.
insert(std::make_pair(ResourceMask, NumUnits)).first;
334 BusyResourceMask |=
it->first;
340 ConsumedResourceMask |=
it->first;
344 return BusyResourceMask;
350 for (
const std::pair<uint64_t, ResourceUsage> &R : Desc.
Resources) {
357 assert(CS.
begin() == 0 &&
"Invalid {Start, End} cycles!");
358 if (!R.second.isReserved()) {
361 BusyResources[Pipe] += CS.
size();
362 Pipes.emplace_back(std::pair<ResourceRef, ResourceCycles>(
367 assert(R.second.isReserved());
375 for (std::pair<ResourceRef, unsigned> &
BR : BusyResources) {
385 ResourcesFreed.push_back(RR);
390 BusyResources.
erase(RF);
397 "Unexpected resource state found!");
399 ReservedResourceGroups ^= 1ULL <<
Index;
407 ReservedResourceGroups ^= 1ULL <<
Index;
410 ReservedBuffers ^= 1ULL <<
Index;