LLVM  16.0.0git
GCNSchedStrategy.cpp
Go to the documentation of this file.
1 //===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This contains a MachineSchedStrategy implementation for maximizing wave
11 /// occupancy on GCN hardware.
12 ///
13 /// This pass will apply multiple scheduling stages to the same function.
14 /// Regions are first recorded in GCNScheduleDAGMILive::schedule. The actual
15 /// entry point for the scheduling of those regions is
16 /// GCNScheduleDAGMILive::runSchedStages.
17 
18 /// Generally, the reason for having multiple scheduling stages is to account
19 /// for the kernel-wide effect of register usage on occupancy. Usually, only a
20 /// few scheduling regions will have register pressure high enough to limit
21 /// occupancy for the kernel, so constraints can be relaxed to improve ILP in
22 /// other regions.
23 ///
24 //===----------------------------------------------------------------------===//
25 
26 #include "GCNSchedStrategy.h"
27 #include "AMDGPUIGroupLP.h"
28 #include "SIMachineFunctionInfo.h"
30 
31 #define DEBUG_TYPE "machine-scheduler"
32 
33 using namespace llvm;
34 
35 static cl::opt<bool>
36  DisableUnclusterHighRP("amdgpu-disable-unclustred-high-rp-reschedule",
37  cl::Hidden,
38  cl::desc("Disable unclustred high register pressure "
39  "reduction scheduling stage."),
40  cl::init(false));
41 
43  : GenericScheduler(C), TargetOccupancy(0), MF(nullptr),
44  HasHighPressure(false) {}
45 
48 
49  MF = &DAG->MF;
50 
52 
54  Context->RegClassInfo->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass);
56  Context->RegClassInfo->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass);
57 
59  // Set the initial TargetOccupnacy to the maximum occupancy that we can
60  // achieve for this function. This effectively sets a lower bound on the
61  // 'Critical' register limits in the scheduler.
64  std::min(ST.getMaxNumSGPRs(TargetOccupancy, true), SGPRExcessLimit);
66  std::min(ST.getMaxNumVGPRs(TargetOccupancy), VGPRExcessLimit);
67 
68  // Subtract error margin from register limits and avoid overflow.
75 }
76 
78  bool AtTop,
79  const RegPressureTracker &RPTracker,
80  const SIRegisterInfo *SRI,
81  unsigned SGPRPressure,
82  unsigned VGPRPressure) {
83  Cand.SU = SU;
84  Cand.AtTop = AtTop;
85 
86  if (!DAG->isTrackingPressure())
87  return;
88 
89  // getDownwardPressure() and getUpwardPressure() make temporary changes to
90  // the tracker, so we need to pass those function a non-const copy.
91  RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
92 
93  Pressure.clear();
94  MaxPressure.clear();
95 
96  if (AtTop)
97  TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
98  else {
99  // FIXME: I think for bottom up scheduling, the register pressure is cached
100  // and can be retrieved by DAG->getPressureDif(SU).
101  TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
102  }
103 
104  unsigned NewSGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
105  unsigned NewVGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
106 
107  // If two instructions increase the pressure of different register sets
108  // by the same amount, the generic scheduler will prefer to schedule the
109  // instruction that increases the set with the least amount of registers,
110  // which in our case would be SGPRs. This is rarely what we want, so
111  // when we report excess/critical register pressure, we do it either
112  // only for VGPRs or only for SGPRs.
113 
114  // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
115  const unsigned MaxVGPRPressureInc = 16;
116  bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
117  bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
118 
119 
120  // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
121  // to increase the likelihood we don't go over the limits. We should improve
122  // the analysis to look through dependencies to find the path with the least
123  // register pressure.
124 
125  // We only need to update the RPDelta for instructions that increase register
126  // pressure. Instructions that decrease or keep reg pressure the same will be
127  // marked as RegExcess in tryCandidate() when they are compared with
128  // instructions that increase the register pressure.
129  if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
130  HasHighPressure = true;
131  Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::VGPR_32);
132  Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
133  }
134 
135  if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
136  HasHighPressure = true;
137  Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::SReg_32);
138  Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
139  }
140 
141  // Register pressure is considered 'CRITICAL' if it is approaching a value
142  // that would reduce the wave occupancy for the execution unit. When
143  // register pressure is 'CRITICAL', increasing SGPR and VGPR pressure both
144  // has the same cost, so we don't need to prefer one over the other.
145 
146  int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
147  int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
148 
149  if (SGPRDelta >= 0 || VGPRDelta >= 0) {
150  HasHighPressure = true;
151  if (SGPRDelta > VGPRDelta) {
152  Cand.RPDelta.CriticalMax =
153  PressureChange(AMDGPU::RegisterPressureSets::SReg_32);
154  Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
155  } else {
156  Cand.RPDelta.CriticalMax =
157  PressureChange(AMDGPU::RegisterPressureSets::VGPR_32);
158  Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
159  }
160  }
161 }
162 
163 // This function is mostly cut and pasted from
164 // GenericScheduler::pickNodeFromQueue()
166  const CandPolicy &ZonePolicy,
167  const RegPressureTracker &RPTracker,
168  SchedCandidate &Cand) {
169  const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
171  unsigned SGPRPressure = 0;
172  unsigned VGPRPressure = 0;
173  if (DAG->isTrackingPressure()) {
174  SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
175  VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
176  }
177  ReadyQueue &Q = Zone.Available;
178  for (SUnit *SU : Q) {
179 
180  SchedCandidate TryCand(ZonePolicy);
181  initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
182  SGPRPressure, VGPRPressure);
183  // Pass SchedBoundary only when comparing nodes from the same boundary.
184  SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
185  tryCandidate(Cand, TryCand, ZoneArg);
186  if (TryCand.Reason != NoCand) {
187  // Initialize resource delta if needed in case future heuristics query it.
188  if (TryCand.ResDelta == SchedResourceDelta())
189  TryCand.initResourceDelta(Zone.DAG, SchedModel);
190  Cand.setBest(TryCand);
191  LLVM_DEBUG(traceCandidate(Cand));
192  }
193  }
194 }
195 
196 // This function is mostly cut and pasted from
197 // GenericScheduler::pickNodeBidirectional()
199  // Schedule as far as possible in the direction of no choice. This is most
200  // efficient, but also provides the best heuristics for CriticalPSets.
201  if (SUnit *SU = Bot.pickOnlyChoice()) {
202  IsTopNode = false;
203  return SU;
204  }
205  if (SUnit *SU = Top.pickOnlyChoice()) {
206  IsTopNode = true;
207  return SU;
208  }
209  // Set the bottom-up policy based on the state of the current bottom zone and
210  // the instructions outside the zone, including the top zone.
211  CandPolicy BotPolicy;
212  setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
213  // Set the top-down policy based on the state of the current top zone and
214  // the instructions outside the zone, including the bottom zone.
215  CandPolicy TopPolicy;
216  setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
217 
218  // See if BotCand is still valid (because we previously scheduled from Top).
219  LLVM_DEBUG(dbgs() << "Picking from Bot:\n");
220  if (!BotCand.isValid() || BotCand.SU->isScheduled ||
221  BotCand.Policy != BotPolicy) {
224  assert(BotCand.Reason != NoCand && "failed to find the first candidate");
225  } else {
227 #ifndef NDEBUG
228  if (VerifyScheduling) {
229  SchedCandidate TCand;
230  TCand.reset(CandPolicy());
231  pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), TCand);
232  assert(TCand.SU == BotCand.SU &&
233  "Last pick result should correspond to re-picking right now");
234  }
235 #endif
236  }
237 
238  // Check if the top Q has a better candidate.
239  LLVM_DEBUG(dbgs() << "Picking from Top:\n");
240  if (!TopCand.isValid() || TopCand.SU->isScheduled ||
241  TopCand.Policy != TopPolicy) {
244  assert(TopCand.Reason != NoCand && "failed to find the first candidate");
245  } else {
247 #ifndef NDEBUG
248  if (VerifyScheduling) {
249  SchedCandidate TCand;
250  TCand.reset(CandPolicy());
251  pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TCand);
252  assert(TCand.SU == TopCand.SU &&
253  "Last pick result should correspond to re-picking right now");
254  }
255 #endif
256  }
257 
258  // Pick best from BotCand and TopCand.
259  LLVM_DEBUG(dbgs() << "Top Cand: "; traceCandidate(TopCand);
260  dbgs() << "Bot Cand: "; traceCandidate(BotCand););
261  SchedCandidate Cand = BotCand;
263  tryCandidate(Cand, TopCand, nullptr);
264  if (TopCand.Reason != NoCand) {
265  Cand.setBest(TopCand);
266  }
267  LLVM_DEBUG(dbgs() << "Picking: "; traceCandidate(Cand););
268 
269  IsTopNode = Cand.AtTop;
270  return Cand.SU;
271 }
272 
273 // This function is mostly cut and pasted from
274 // GenericScheduler::pickNode()
276  if (DAG->top() == DAG->bottom()) {
278  Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
279  return nullptr;
280  }
281  SUnit *SU;
282  do {
284  SU = Top.pickOnlyChoice();
285  if (!SU) {
286  CandPolicy NoPolicy;
287  TopCand.reset(NoPolicy);
289  assert(TopCand.Reason != NoCand && "failed to find a candidate");
290  SU = TopCand.SU;
291  }
292  IsTopNode = true;
293  } else if (RegionPolicy.OnlyBottomUp) {
294  SU = Bot.pickOnlyChoice();
295  if (!SU) {
296  CandPolicy NoPolicy;
297  BotCand.reset(NoPolicy);
299  assert(BotCand.Reason != NoCand && "failed to find a candidate");
300  SU = BotCand.SU;
301  }
302  IsTopNode = false;
303  } else {
304  SU = pickNodeBidirectional(IsTopNode);
305  }
306  } while (SU->isScheduled);
307 
308  if (SU->isTopReady())
309  Top.removeReady(SU);
310  if (SU->isBottomReady())
311  Bot.removeReady(SU);
312 
313  LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
314  << *SU->getInstr());
315  return SU;
316 }
317 
320  return *CurrentStage;
321 }
322 
324  assert(CurrentStage != SchedStages.end());
325  if (!CurrentStage)
326  CurrentStage = SchedStages.begin();
327  else
328  CurrentStage++;
329 
330  return CurrentStage != SchedStages.end();
331 }
332 
335  return std::next(CurrentStage) != SchedStages.end();
336 }
337 
339  assert(CurrentStage && std::next(CurrentStage) != SchedStages.end());
340  return *std::next(CurrentStage);
341 }
342 
344  const MachineSchedContext *C)
345  : GCNSchedStrategy(C) {
350 }
351 
353  : GCNSchedStrategy(C) {
355 }
356 
358  SchedCandidate &TryCand,
359  SchedBoundary *Zone) const {
360  // Initialize the candidate if needed.
361  if (!Cand.isValid()) {
362  TryCand.Reason = NodeOrder;
363  return true;
364  }
365 
366  // Avoid spilling by exceeding the register limit.
367  if (DAG->isTrackingPressure() &&
368  tryPressure(TryCand.RPDelta.Excess, Cand.RPDelta.Excess, TryCand, Cand,
369  RegExcess, TRI, DAG->MF))
370  return TryCand.Reason != NoCand;
371 
372  // Bias PhysReg Defs and copies to their uses and defined respectively.
373  if (tryGreater(biasPhysReg(TryCand.SU, TryCand.AtTop),
374  biasPhysReg(Cand.SU, Cand.AtTop), TryCand, Cand, PhysReg))
375  return TryCand.Reason != NoCand;
376 
377  bool SameBoundary = Zone != nullptr;
378  if (SameBoundary) {
379  // Prioritize instructions that read unbuffered resources by stall cycles.
380  if (tryLess(Zone->getLatencyStallCycles(TryCand.SU),
381  Zone->getLatencyStallCycles(Cand.SU), TryCand, Cand, Stall))
382  return TryCand.Reason != NoCand;
383 
384  // Avoid critical resource consumption and balance the schedule.
385  TryCand.initResourceDelta(DAG, SchedModel);
387  TryCand, Cand, ResourceReduce))
388  return TryCand.Reason != NoCand;
390  Cand.ResDelta.DemandedResources, TryCand, Cand,
392  return TryCand.Reason != NoCand;
393 
394  // Unconditionally try to reduce latency.
395  if (tryLatency(TryCand, Cand, *Zone))
396  return TryCand.Reason != NoCand;
397 
398  // Weak edges are for clustering and other constraints.
399  if (tryLess(getWeakLeft(TryCand.SU, TryCand.AtTop),
400  getWeakLeft(Cand.SU, Cand.AtTop), TryCand, Cand, Weak))
401  return TryCand.Reason != NoCand;
402  }
403 
404  // Keep clustered nodes together to encourage downstream peephole
405  // optimizations which may reduce resource requirements.
406  //
407  // This is a best effort to set things up for a post-RA pass. Optimizations
408  // like generating loads of multiple registers should ideally be done within
409  // the scheduler pass by combining the loads during DAG postprocessing.
410  const SUnit *CandNextClusterSU =
412  const SUnit *TryCandNextClusterSU =
414  if (tryGreater(TryCand.SU == TryCandNextClusterSU,
415  Cand.SU == CandNextClusterSU, TryCand, Cand, Cluster))
416  return TryCand.Reason != NoCand;
417 
418  // Avoid increasing the max critical pressure in the scheduled region.
419  if (DAG->isTrackingPressure() &&
421  TryCand, Cand, RegCritical, TRI, DAG->MF))
422  return TryCand.Reason != NoCand;
423 
424  // Avoid increasing the max pressure of the entire region.
425  if (DAG->isTrackingPressure() &&
426  tryPressure(TryCand.RPDelta.CurrentMax, Cand.RPDelta.CurrentMax, TryCand,
427  Cand, RegMax, TRI, DAG->MF))
428  return TryCand.Reason != NoCand;
429 
430  if (SameBoundary) {
431  // Fall through to original instruction order.
432  if ((Zone->isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum) ||
433  (!Zone->isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) {
434  TryCand.Reason = NodeOrder;
435  return true;
436  }
437  }
438  return false;
439 }
440 
442  MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S)
443  : ScheduleDAGMILive(C, std::move(S)), ST(MF.getSubtarget<GCNSubtarget>()),
444  MFI(*MF.getInfo<SIMachineFunctionInfo>()),
445  StartingOccupancy(MFI.getOccupancy()), MinOccupancy(StartingOccupancy) {
446 
447  LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n");
448 }
449 
450 std::unique_ptr<GCNSchedStage>
451 GCNScheduleDAGMILive::createSchedStage(GCNSchedStageID SchedStageID) {
452  switch (SchedStageID) {
454  return std::make_unique<OccInitialScheduleStage>(SchedStageID, *this);
456  return std::make_unique<UnclusteredHighRPStage>(SchedStageID, *this);
458  return std::make_unique<ClusteredLowOccStage>(SchedStageID, *this);
460  return std::make_unique<PreRARematStage>(SchedStageID, *this);
462  return std::make_unique<ILPInitialScheduleStage>(SchedStageID, *this);
463  }
464 
465  llvm_unreachable("Unknown SchedStageID.");
466 }
467 
469  // Collect all scheduling regions. The actual scheduling is performed in
470  // GCNScheduleDAGMILive::finalizeSchedule.
471  Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
472 }
473 
475 GCNScheduleDAGMILive::getRealRegPressure(unsigned RegionIdx) const {
477  RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
478  return RPTracker.moveMaxPressure();
479 }
480 
481 void GCNScheduleDAGMILive::computeBlockPressure(unsigned RegionIdx,
482  const MachineBasicBlock *MBB) {
484 
485  // If the block has the only successor then live-ins of that successor are
486  // live-outs of the current block. We can reuse calculated live set if the
487  // successor will be sent to scheduling past current block.
488  const MachineBasicBlock *OnlySucc = nullptr;
489  if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
490  SlotIndexes *Ind = LIS->getSlotIndexes();
491  if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
492  OnlySucc = *MBB->succ_begin();
493  }
494 
495  // Scheduler sends regions from the end of the block upwards.
496  size_t CurRegion = RegionIdx;
497  for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
498  if (Regions[CurRegion].first->getParent() != MBB)
499  break;
500  --CurRegion;
501 
502  auto I = MBB->begin();
503  auto LiveInIt = MBBLiveIns.find(MBB);
504  auto &Rgn = Regions[CurRegion];
505  auto *NonDbgMI = &*skipDebugInstructionsForward(Rgn.first, Rgn.second);
506  if (LiveInIt != MBBLiveIns.end()) {
507  auto LiveIn = std::move(LiveInIt->second);
508  RPTracker.reset(*MBB->begin(), &LiveIn);
509  MBBLiveIns.erase(LiveInIt);
510  } else {
511  I = Rgn.first;
512  auto LRS = BBLiveInMap.lookup(NonDbgMI);
513 #ifdef EXPENSIVE_CHECKS
514  assert(isEqual(getLiveRegsBefore(*NonDbgMI, *LIS), LRS));
515 #endif
516  RPTracker.reset(*I, &LRS);
517  }
518 
519  for (;;) {
520  I = RPTracker.getNext();
521 
522  if (Regions[CurRegion].first == I || NonDbgMI == I) {
523  LiveIns[CurRegion] = RPTracker.getLiveRegs();
524  RPTracker.clearMaxPressure();
525  }
526 
527  if (Regions[CurRegion].second == I) {
528  Pressure[CurRegion] = RPTracker.moveMaxPressure();
529  if (CurRegion-- == RegionIdx)
530  break;
531  }
532  RPTracker.advanceToNext();
533  RPTracker.advanceBeforeNext();
534  }
535 
536  if (OnlySucc) {
537  if (I != MBB->end()) {
538  RPTracker.advanceToNext();
539  RPTracker.advance(MBB->end());
540  }
541  RPTracker.advanceBeforeNext();
542  MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
543  }
544 }
545 
547 GCNScheduleDAGMILive::getBBLiveInMap() const {
548  assert(!Regions.empty());
549  std::vector<MachineInstr *> BBStarters;
550  BBStarters.reserve(Regions.size());
551  auto I = Regions.rbegin(), E = Regions.rend();
552  auto *BB = I->first->getParent();
553  do {
554  auto *MI = &*skipDebugInstructionsForward(I->first, I->second);
555  BBStarters.push_back(MI);
556  do {
557  ++I;
558  } while (I != E && I->first->getParent() == BB);
559  } while (I != E);
560  return getLiveRegMap(BBStarters, false /*After*/, *LIS);
561 }
562 
564  // Start actual scheduling here. This function is called by the base
565  // MachineScheduler after all regions have been recorded by
566  // GCNScheduleDAGMILive::schedule().
567  LiveIns.resize(Regions.size());
568  Pressure.resize(Regions.size());
569  RescheduleRegions.resize(Regions.size());
570  RegionsWithHighRP.resize(Regions.size());
571  RegionsWithExcessRP.resize(Regions.size());
572  RegionsWithMinOcc.resize(Regions.size());
573  RegionsWithIGLPInstrs.resize(Regions.size());
574  RescheduleRegions.set();
575  RegionsWithHighRP.reset();
576  RegionsWithExcessRP.reset();
577  RegionsWithMinOcc.reset();
578  RegionsWithIGLPInstrs.reset();
579 
580  runSchedStages();
581 }
582 
583 void GCNScheduleDAGMILive::runSchedStages() {
584  LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n");
585 
586  if (!Regions.empty())
587  BBLiveInMap = getBBLiveInMap();
588 
589  GCNSchedStrategy &S = static_cast<GCNSchedStrategy &>(*SchedImpl);
590  while (S.advanceStage()) {
591  auto Stage = createSchedStage(S.getCurrentStage());
592  if (!Stage->initGCNSchedStage())
593  continue;
594 
595  for (auto Region : Regions) {
596  RegionBegin = Region.first;
597  RegionEnd = Region.second;
598  // Setup for scheduling the region and check whether it should be skipped.
599  if (!Stage->initGCNRegion()) {
600  Stage->advanceRegion();
601  exitRegion();
602  continue;
603  }
604 
606  Stage->finalizeGCNRegion();
607  }
608 
609  Stage->finalizeGCNSchedStage();
610  }
611 }
612 
613 #ifndef NDEBUG
615  switch (StageID) {
617  OS << "Max Occupancy Initial Schedule";
618  break;
620  OS << "Unclustered High Register Pressure Reschedule";
621  break;
623  OS << "Clustered Low Occupancy Reschedule";
624  break;
626  OS << "Pre-RA Rematerialize";
627  break;
629  OS << "Max ILP Initial Schedule";
630  break;
631  }
632 
633  return OS;
634 }
635 #endif
636 
638  : DAG(DAG), S(static_cast<GCNSchedStrategy &>(*DAG.SchedImpl)), MF(DAG.MF),
639  MFI(DAG.MFI), ST(DAG.ST), StageID(StageID) {}
640 
642  if (!DAG.LIS)
643  return false;
644 
645  LLVM_DEBUG(dbgs() << "Starting scheduling stage: " << StageID << "\n");
646  return true;
647 }
648 
651  return false;
652 
654  return false;
655 
656  if (DAG.RegionsWithHighRP.none() && DAG.RegionsWithExcessRP.none())
657  return false;
658 
661 
662  InitialOccupancy = DAG.MinOccupancy;
663  // Aggressivly try to reduce register pressure in the unclustered high RP
664  // stage. Temporarily increase occupancy target in the region.
666  if (MFI.getMaxWavesPerEU() > DAG.MinOccupancy)
667  MFI.increaseOccupancy(MF, ++DAG.MinOccupancy);
668 
669  LLVM_DEBUG(
670  dbgs()
671  << "Retrying function scheduling without clustering. "
672  "Aggressivly try to reduce register pressure to achieve occupancy "
673  << DAG.MinOccupancy << ".\n");
674 
675  return true;
676 }
677 
680  return false;
681 
682  // Don't bother trying to improve ILP in lower RP regions if occupancy has not
683  // been dropped. All regions will have already been scheduled with the ideal
684  // occupancy targets.
685  if (DAG.StartingOccupancy <= DAG.MinOccupancy)
686  return false;
687 
688  LLVM_DEBUG(
689  dbgs() << "Retrying function scheduling with lowest recorded occupancy "
690  << DAG.MinOccupancy << ".\n");
691  return true;
692 }
693 
696  return false;
697 
698  if (DAG.RegionsWithMinOcc.none() || DAG.Regions.size() == 1)
699  return false;
700 
702  // Check maximum occupancy
704  DAG.MinOccupancy)
705  return false;
706 
707  // FIXME: This pass will invalidate cached MBBLiveIns for regions
708  // inbetween the defs and region we sinked the def to. Cached pressure
709  // for regions where a def is sinked from will also be invalidated. Will
710  // need to be fixed if there is another pass after this pass.
711  assert(!S.hasNextStage());
712 
713  collectRematerializableInstructions();
714  if (RematerializableInsts.empty() || !sinkTriviallyRematInsts(ST, TII))
715  return false;
716 
717  LLVM_DEBUG(
718  dbgs() << "Retrying function scheduling with improved occupancy of "
719  << DAG.MinOccupancy << " from rematerializing\n");
720  return true;
721 }
722 
724  DAG.finishBlock();
725  LLVM_DEBUG(dbgs() << "Ending scheduling stage: " << StageID << "\n");
726 }
727 
731  if (DAG.MinOccupancy > InitialOccupancy) {
732  for (unsigned IDX = 0; IDX < DAG.Pressure.size(); ++IDX)
733  DAG.RegionsWithMinOcc[IDX] =
734  DAG.Pressure[IDX].getOccupancy(DAG.ST) == DAG.MinOccupancy;
735 
736  LLVM_DEBUG(dbgs() << StageID
737  << " stage successfully increased occupancy to "
738  << DAG.MinOccupancy << '\n');
739  }
740 
742 }
743 
745  // Check whether this new region is also a new block.
746  if (DAG.RegionBegin->getParent() != CurrentMBB)
747  setupNewBlock();
748 
749  unsigned NumRegionInstrs = std::distance(DAG.begin(), DAG.end());
750  DAG.enterRegion(CurrentMBB, DAG.begin(), DAG.end(), NumRegionInstrs);
751 
752  // Skip empty scheduling regions (0 or 1 schedulable instructions).
753  if (DAG.begin() == DAG.end() || DAG.begin() == std::prev(DAG.end()))
754  return false;
755 
756  LLVM_DEBUG(dbgs() << "********** MI Scheduling **********\n");
758  << " " << CurrentMBB->getName()
759  << "\n From: " << *DAG.begin() << " To: ";
760  if (DAG.RegionEnd != CurrentMBB->end()) dbgs() << *DAG.RegionEnd;
761  else dbgs() << "End";
762  dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n');
763 
764  // Save original instruction order before scheduling for possible revert.
765  Unsched.clear();
766  Unsched.reserve(DAG.NumRegionInstrs);
769  for (auto &I : DAG) {
770  Unsched.push_back(&I);
771  if (I.getOpcode() == AMDGPU::SCHED_GROUP_BARRIER ||
772  I.getOpcode() == AMDGPU::IGLP_OPT)
773  DAG.RegionsWithIGLPInstrs[RegionIdx] = true;
774  }
775  } else {
776  for (auto &I : DAG)
777  Unsched.push_back(&I);
778  }
779 
780  PressureBefore = DAG.Pressure[RegionIdx];
781 
782  LLVM_DEBUG(
783  dbgs() << "Pressure before scheduling:\nRegion live-ins:"
784  << print(DAG.LiveIns[RegionIdx], DAG.MRI)
785  << "Region live-in pressure: "
787  << "Region register pressure: " << print(PressureBefore));
788 
789  S.HasHighPressure = false;
790 
791  if (DAG.RegionsWithIGLPInstrs[RegionIdx] &&
793  SavedMutations.clear();
796  }
797 
798  return true;
799 }
800 
802  // Only reschedule regions with the minimum occupancy or regions that may have
803  // spilling (excess register pressure).
804  if ((!DAG.RegionsWithMinOcc[RegionIdx] ||
805  DAG.MinOccupancy <= InitialOccupancy) &&
806  !DAG.RegionsWithExcessRP[RegionIdx])
807  return false;
808 
810 }
811 
813  // We may need to reschedule this region if it wasn't rescheduled in the last
814  // stage, or if we found it was testing critical register pressure limits in
815  // the unclustered reschedule stage. The later is because we may not have been
816  // able to raise the min occupancy in the previous stage so the region may be
817  // overly constrained even if it was already rescheduled.
818  if (!DAG.RegionsWithHighRP[RegionIdx])
819  return false;
820 
822 }
823 
825  if (!DAG.RescheduleRegions[RegionIdx])
826  return false;
827 
829 }
830 
832  if (CurrentMBB)
833  DAG.finishBlock();
834 
835  CurrentMBB = DAG.RegionBegin->getParent();
837  // Get real RP for the region if it hasn't be calculated before. After the
838  // initial schedule stage real RP will be collected after scheduling.
840  DAG.computeBlockPressure(RegionIdx, CurrentMBB);
841 }
842 
844  DAG.Regions[RegionIdx] = std::make_pair(DAG.RegionBegin, DAG.RegionEnd);
845  DAG.RescheduleRegions[RegionIdx] = false;
846  if (S.HasHighPressure)
847  DAG.RegionsWithHighRP[RegionIdx] = true;
848 
849  // Revert scheduling if we have dropped occupancy or there is some other
850  // reason that the original schedule is better.
851  checkScheduling();
852 
853  if (DAG.RegionsWithIGLPInstrs[RegionIdx] &&
856 
857  DAG.exitRegion();
858  RegionIdx++;
859 }
860 
862  // Check the results of scheduling.
863  PressureAfter = DAG.getRealRegPressure(RegionIdx);
864  LLVM_DEBUG(dbgs() << "Pressure after scheduling: " << print(PressureAfter));
865 
868  DAG.Pressure[RegionIdx] = PressureAfter;
869  DAG.RegionsWithMinOcc[RegionIdx] =
870  PressureAfter.getOccupancy(ST) == DAG.MinOccupancy;
871 
872  // Early out if we have achieve the occupancy target.
873  LLVM_DEBUG(dbgs() << "Pressure in desired limits, done.\n");
874  return;
875  }
876 
877  unsigned WavesAfter =
879  unsigned WavesBefore =
881  LLVM_DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore
882  << ", after " << WavesAfter << ".\n");
883 
884  // We may not be able to keep the current target occupancy because of the just
885  // scheduled region. We might still be able to revert scheduling if the
886  // occupancy before was higher, or if the current schedule has register
887  // pressure higher than the excess limits which could lead to more spilling.
888  unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
889 
890  // Allow memory bound functions to drop to 4 waves if not limited by an
891  // attribute.
892  if (WavesAfter < WavesBefore && WavesAfter < DAG.MinOccupancy &&
893  WavesAfter >= MFI.getMinAllowedOccupancy()) {
894  LLVM_DEBUG(dbgs() << "Function is memory bound, allow occupancy drop up to "
895  << MFI.getMinAllowedOccupancy() << " waves\n");
896  NewOccupancy = WavesAfter;
897  }
898 
899  if (NewOccupancy < DAG.MinOccupancy) {
900  DAG.MinOccupancy = NewOccupancy;
901  MFI.limitOccupancy(DAG.MinOccupancy);
902  DAG.RegionsWithMinOcc.reset();
903  LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to "
904  << DAG.MinOccupancy << ".\n");
905  }
906 
907  unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);
908  unsigned MaxSGPRs = ST.getMaxNumSGPRs(MF);
909  if (PressureAfter.getVGPRNum(false) > MaxVGPRs ||
910  PressureAfter.getAGPRNum() > MaxVGPRs ||
911  PressureAfter.getSGPRNum() > MaxSGPRs) {
912  DAG.RescheduleRegions[RegionIdx] = true;
913  DAG.RegionsWithHighRP[RegionIdx] = true;
914  DAG.RegionsWithExcessRP[RegionIdx] = true;
915  }
916 
917  // Revert if this region's schedule would cause a drop in occupancy or
918  // spilling.
919  if (shouldRevertScheduling(WavesAfter)) {
921  } else {
922  DAG.Pressure[RegionIdx] = PressureAfter;
923  DAG.RegionsWithMinOcc[RegionIdx] =
924  PressureAfter.getOccupancy(ST) == DAG.MinOccupancy;
925  }
926 }
927 
928 bool GCNSchedStage::shouldRevertScheduling(unsigned WavesAfter) {
929  if (WavesAfter < DAG.MinOccupancy)
930  return true;
931 
932  return false;
933 }
934 
937  return true;
938 
939  if (mayCauseSpilling(WavesAfter))
940  return true;
941 
942  return false;
943 }
944 
946  // If RP is not reduced in the unclustred reschedule stage, revert to the
947  // old schedule.
948  if ((WavesAfter <= PressureBefore.getOccupancy(ST) &&
949  mayCauseSpilling(WavesAfter)) ||
951  LLVM_DEBUG(dbgs() << "Unclustered reschedule did not help.\n");
952  return true;
953  }
954 
955  return false;
956 }
957 
960  return true;
961 
962  if (mayCauseSpilling(WavesAfter))
963  return true;
964 
965  return false;
966 }
967 
968 bool PreRARematStage::shouldRevertScheduling(unsigned WavesAfter) {
970  return true;
971 
972  if (mayCauseSpilling(WavesAfter))
973  return true;
974 
975  return false;
976 }
977 
979  if (mayCauseSpilling(WavesAfter))
980  return true;
981 
982  return false;
983 }
984 
985 bool GCNSchedStage::mayCauseSpilling(unsigned WavesAfter) {
986  if (WavesAfter <= MFI.getMinWavesPerEU() &&
988  DAG.RegionsWithExcessRP[RegionIdx]) {
989  LLVM_DEBUG(dbgs() << "New pressure will result in more spilling.\n");
990  return true;
991  }
992 
993  return false;
994 }
995 
997  DAG.RegionsWithMinOcc[RegionIdx] =
998  PressureBefore.getOccupancy(ST) == DAG.MinOccupancy;
999  LLVM_DEBUG(dbgs() << "Attempting to revert scheduling.\n");
1000  DAG.RescheduleRegions[RegionIdx] =
1001  S.hasNextStage() &&
1004  int SkippedDebugInstr = 0;
1005  for (MachineInstr *MI : Unsched) {
1006  if (MI->isDebugInstr()) {
1007  ++SkippedDebugInstr;
1008  continue;
1009  }
1010 
1011  if (MI->getIterator() != DAG.RegionEnd) {
1012  DAG.BB->remove(MI);
1013  DAG.BB->insert(DAG.RegionEnd, MI);
1014  if (!MI->isDebugInstr())
1015  DAG.LIS->handleMove(*MI, true);
1016  }
1017 
1018  // Reset read-undef flags and update them later.
1019  for (auto &Op : MI->operands())
1020  if (Op.isReg() && Op.isDef())
1021  Op.setIsUndef(false);
1022  RegisterOperands RegOpers;
1023  RegOpers.collect(*MI, *DAG.TRI, DAG.MRI, DAG.ShouldTrackLaneMasks, false);
1024  if (!MI->isDebugInstr()) {
1025  if (DAG.ShouldTrackLaneMasks) {
1026  // Adjust liveness and add missing dead+read-undef flags.
1027  SlotIndex SlotIdx = DAG.LIS->getInstructionIndex(*MI).getRegSlot();
1028  RegOpers.adjustLaneLiveness(*DAG.LIS, DAG.MRI, SlotIdx, MI);
1029  } else {
1030  // Adjust for missing dead-def flags.
1031  RegOpers.detectDeadDefs(*MI, *DAG.LIS);
1032  }
1033  }
1034  DAG.RegionEnd = MI->getIterator();
1035  ++DAG.RegionEnd;
1036  LLVM_DEBUG(dbgs() << "Scheduling " << *MI);
1037  }
1038 
1039  // After reverting schedule, debug instrs will now be at the end of the block
1040  // and RegionEnd will point to the first debug instr. Increment RegionEnd
1041  // pass debug instrs to the actual end of the scheduling region.
1042  while (SkippedDebugInstr-- > 0)
1043  ++DAG.RegionEnd;
1044 
1045  // If Unsched.front() instruction is a debug instruction, this will actually
1046  // shrink the region since we moved all debug instructions to the end of the
1047  // block. Find the first instruction that is not a debug instruction.
1048  DAG.RegionBegin = Unsched.front()->getIterator();
1049  if (DAG.RegionBegin->isDebugInstr()) {
1050  for (MachineInstr *MI : Unsched) {
1051  if (MI->isDebugInstr())
1052  continue;
1053  DAG.RegionBegin = MI->getIterator();
1054  break;
1055  }
1056  }
1057 
1058  // Then move the debug instructions back into their correct place and set
1059  // RegionBegin and RegionEnd if needed.
1061 
1062  DAG.Regions[RegionIdx] = std::make_pair(DAG.RegionBegin, DAG.RegionEnd);
1063 }
1064 
1065 void PreRARematStage::collectRematerializableInstructions() {
1066  const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(DAG.TRI);
1067  for (unsigned I = 0, E = DAG.MRI.getNumVirtRegs(); I != E; ++I) {
1069  if (!DAG.LIS->hasInterval(Reg))
1070  continue;
1071 
1072  // TODO: Handle AGPR and SGPR rematerialization
1073  if (!SRI->isVGPRClass(DAG.MRI.getRegClass(Reg)) ||
1075  continue;
1076 
1078  MachineInstr *Def = Op->getParent();
1079  if (Op->getSubReg() != 0 || !isTriviallyReMaterializable(*Def))
1080  continue;
1081 
1083  if (Def->getParent() == UseI->getParent())
1084  continue;
1085 
1086  // We are only collecting defs that are defined in another block and are
1087  // live-through or used inside regions at MinOccupancy. This means that the
1088  // register must be in the live-in set for the region.
1089  bool AddedToRematList = false;
1090  for (unsigned I = 0, E = DAG.Regions.size(); I != E; ++I) {
1091  auto It = DAG.LiveIns[I].find(Reg);
1092  if (It != DAG.LiveIns[I].end() && !It->second.none()) {
1093  if (DAG.RegionsWithMinOcc[I]) {
1094  RematerializableInsts[I][Def] = UseI;
1095  AddedToRematList = true;
1096  }
1097 
1098  // Collect regions with rematerializable reg as live-in to avoid
1099  // searching later when updating RP.
1100  RematDefToLiveInRegions[Def].push_back(I);
1101  }
1102  }
1103  if (!AddedToRematList)
1104  RematDefToLiveInRegions.erase(Def);
1105  }
1106 }
1107 
1108 bool PreRARematStage::sinkTriviallyRematInsts(const GCNSubtarget &ST,
1109  const TargetInstrInfo *TII) {
1110  // Temporary copies of cached variables we will be modifying and replacing if
1111  // sinking succeeds.
1112  SmallVector<
1113  std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>, 32>
1114  NewRegions;
1117  BitVector NewRescheduleRegions;
1118  LiveIntervals *LIS = DAG.LIS;
1119 
1120  NewRegions.resize(DAG.Regions.size());
1121  NewRescheduleRegions.resize(DAG.Regions.size());
1122 
1123  // Collect only regions that has a rematerializable def as a live-in.
1124  SmallSet<unsigned, 16> ImpactedRegions;
1125  for (const auto &It : RematDefToLiveInRegions)
1126  ImpactedRegions.insert(It.second.begin(), It.second.end());
1127 
1128  // Make copies of register pressure and live-ins cache that will be updated
1129  // as we rematerialize.
1130  for (auto Idx : ImpactedRegions) {
1131  NewPressure[Idx] = DAG.Pressure[Idx];
1132  NewLiveIns[Idx] = DAG.LiveIns[Idx];
1133  }
1134  NewRegions = DAG.Regions;
1135  NewRescheduleRegions.reset();
1136 
1137  DenseMap<MachineInstr *, MachineInstr *> InsertedMIToOldDef;
1138  bool Improved = false;
1139  for (auto I : ImpactedRegions) {
1140  if (!DAG.RegionsWithMinOcc[I])
1141  continue;
1142 
1143  Improved = false;
1144  int VGPRUsage = NewPressure[I].getVGPRNum(ST.hasGFX90AInsts());
1145  int SGPRUsage = NewPressure[I].getSGPRNum();
1146 
1147  // TODO: Handle occupancy drop due to AGPR and SGPR.
1148  // Check if cause of occupancy drop is due to VGPR usage and not SGPR.
1149  if (ST.getOccupancyWithNumSGPRs(SGPRUsage) == DAG.MinOccupancy)
1150  break;
1151 
1152  // The occupancy of this region could have been improved by a previous
1153  // iteration's sinking of defs.
1154  if (NewPressure[I].getOccupancy(ST) > DAG.MinOccupancy) {
1155  NewRescheduleRegions[I] = true;
1156  Improved = true;
1157  continue;
1158  }
1159 
1160  // First check if we have enough trivially rematerializable instructions to
1161  // improve occupancy. Optimistically assume all instructions we are able to
1162  // sink decreased RP.
1163  int TotalSinkableRegs = 0;
1164  for (const auto &It : RematerializableInsts[I]) {
1165  MachineInstr *Def = It.first;
1166  Register DefReg = Def->getOperand(0).getReg();
1167  TotalSinkableRegs +=
1168  SIRegisterInfo::getNumCoveredRegs(NewLiveIns[I][DefReg]);
1169  }
1170  int VGPRsAfterSink = VGPRUsage - TotalSinkableRegs;
1171  unsigned OptimisticOccupancy = ST.getOccupancyWithNumVGPRs(VGPRsAfterSink);
1172  // If in the most optimistic scenario, we cannot improve occupancy, then do
1173  // not attempt to sink any instructions.
1174  if (OptimisticOccupancy <= DAG.MinOccupancy)
1175  break;
1176 
1177  unsigned ImproveOccupancy = 0;
1178  SmallVector<MachineInstr *, 4> SinkedDefs;
1179  for (auto &It : RematerializableInsts[I]) {
1180  MachineInstr *Def = It.first;
1181  MachineBasicBlock::iterator InsertPos =
1182  MachineBasicBlock::iterator(It.second);
1183  Register Reg = Def->getOperand(0).getReg();
1184  // Rematerialize MI to its use block. Since we are only rematerializing
1185  // instructions that do not have any virtual reg uses, we do not need to
1186  // call LiveRangeEdit::allUsesAvailableAt() and
1187  // LiveRangeEdit::canRematerializeAt().
1188  TII->reMaterialize(*InsertPos->getParent(), InsertPos, Reg,
1189  Def->getOperand(0).getSubReg(), *Def, *DAG.TRI);
1190  MachineInstr *NewMI = &*std::prev(InsertPos);
1191  LIS->InsertMachineInstrInMaps(*NewMI);
1192  LIS->removeInterval(Reg);
1194  InsertedMIToOldDef[NewMI] = Def;
1195 
1196  // Update region boundaries in scheduling region we sinked from since we
1197  // may sink an instruction that was at the beginning or end of its region
1198  DAG.updateRegionBoundaries(NewRegions, Def, /*NewMI =*/nullptr,
1199  /*Removing =*/true);
1200 
1201  // Update region boundaries in region we sinked to.
1202  DAG.updateRegionBoundaries(NewRegions, InsertPos, NewMI);
1203 
1204  LaneBitmask PrevMask = NewLiveIns[I][Reg];
1205  // FIXME: Also update cached pressure for where the def was sinked from.
1206  // Update RP for all regions that has this reg as a live-in and remove
1207  // the reg from all regions as a live-in.
1208  for (auto Idx : RematDefToLiveInRegions[Def]) {
1209  NewLiveIns[Idx].erase(Reg);
1210  if (InsertPos->getParent() != DAG.Regions[Idx].first->getParent()) {
1211  // Def is live-through and not used in this block.
1212  NewPressure[Idx].inc(Reg, PrevMask, LaneBitmask::getNone(), DAG.MRI);
1213  } else {
1214  // Def is used and rematerialized into this block.
1215  GCNDownwardRPTracker RPT(*LIS);
1216  auto *NonDbgMI = &*skipDebugInstructionsForward(
1217  NewRegions[Idx].first, NewRegions[Idx].second);
1218  RPT.reset(*NonDbgMI, &NewLiveIns[Idx]);
1219  RPT.advance(NewRegions[Idx].second);
1220  NewPressure[Idx] = RPT.moveMaxPressure();
1221  }
1222  }
1223 
1224  SinkedDefs.push_back(Def);
1225  ImproveOccupancy = NewPressure[I].getOccupancy(ST);
1226  if (ImproveOccupancy > DAG.MinOccupancy)
1227  break;
1228  }
1229 
1230  // Remove defs we just sinked from all regions' list of sinkable defs
1231  for (auto &Def : SinkedDefs)
1232  for (auto TrackedIdx : RematDefToLiveInRegions[Def])
1233  RematerializableInsts[TrackedIdx].erase(Def);
1234 
1235  if (ImproveOccupancy <= DAG.MinOccupancy)
1236  break;
1237 
1238  NewRescheduleRegions[I] = true;
1239  Improved = true;
1240  }
1241 
1242  if (!Improved) {
1243  // Occupancy was not improved for all regions that were at MinOccupancy.
1244  // Undo sinking and remove newly rematerialized instructions.
1245  for (auto &Entry : InsertedMIToOldDef) {
1246  MachineInstr *MI = Entry.first;
1247  MachineInstr *OldMI = Entry.second;
1248  Register Reg = MI->getOperand(0).getReg();
1250  MI->eraseFromParent();
1251  OldMI->clearRegisterDeads(Reg);
1252  LIS->removeInterval(Reg);
1254  }
1255  return false;
1256  }
1257 
1258  // Occupancy was improved for all regions.
1259  for (auto &Entry : InsertedMIToOldDef) {
1260  MachineInstr *MI = Entry.first;
1261  MachineInstr *OldMI = Entry.second;
1262 
1263  // Remove OldMI from BBLiveInMap since we are sinking it from its MBB.
1264  DAG.BBLiveInMap.erase(OldMI);
1265 
1266  // Remove OldMI and update LIS
1267  Register Reg = MI->getOperand(0).getReg();
1268  LIS->RemoveMachineInstrFromMaps(*OldMI);
1269  OldMI->eraseFromParent();
1270  LIS->removeInterval(Reg);
1272  }
1273 
1274  // Update live-ins, register pressure, and regions caches.
1275  for (auto Idx : ImpactedRegions) {
1276  DAG.LiveIns[Idx] = NewLiveIns[Idx];
1277  DAG.Pressure[Idx] = NewPressure[Idx];
1278  DAG.MBBLiveIns.erase(DAG.Regions[Idx].first->getParent());
1279  }
1280  DAG.Regions = NewRegions;
1281  DAG.RescheduleRegions = NewRescheduleRegions;
1282 
1284  MFI.increaseOccupancy(MF, ++DAG.MinOccupancy);
1285 
1286  return true;
1287 }
1288 
1289 // Copied from MachineLICM
1290 bool PreRARematStage::isTriviallyReMaterializable(const MachineInstr &MI) {
1292  return false;
1293 
1294  for (const MachineOperand &MO : MI.operands())
1295  if (MO.isReg() && MO.isUse() && MO.getReg().isVirtual())
1296  return false;
1297 
1298  return true;
1299 }
1300 
1301 // When removing, we will have to check both beginning and ending of the region.
1302 // When inserting, we will only have to check if we are inserting NewMI in front
1303 // of a scheduling region and do not need to check the ending since we will only
1304 // ever be inserting before an already existing MI.
1305 void GCNScheduleDAGMILive::updateRegionBoundaries(
1307  MachineBasicBlock::iterator>> &RegionBoundaries,
1308  MachineBasicBlock::iterator MI, MachineInstr *NewMI, bool Removing) {
1309  unsigned I = 0, E = RegionBoundaries.size();
1310  // Search for first region of the block where MI is located
1311  while (I != E && MI->getParent() != RegionBoundaries[I].first->getParent())
1312  ++I;
1313 
1314  for (; I != E; ++I) {
1315  if (MI->getParent() != RegionBoundaries[I].first->getParent())
1316  return;
1317 
1318  if (Removing && MI == RegionBoundaries[I].first &&
1319  MI == RegionBoundaries[I].second) {
1320  // MI is in a region with size 1, after removing, the region will be
1321  // size 0, set RegionBegin and RegionEnd to pass end of block iterator.
1322  RegionBoundaries[I] =
1323  std::make_pair(MI->getParent()->end(), MI->getParent()->end());
1324  return;
1325  }
1326  if (MI == RegionBoundaries[I].first) {
1327  if (Removing)
1328  RegionBoundaries[I] =
1329  std::make_pair(std::next(MI), RegionBoundaries[I].second);
1330  else
1331  // Inserted NewMI in front of region, set new RegionBegin to NewMI
1332  RegionBoundaries[I] = std::make_pair(MachineBasicBlock::iterator(NewMI),
1333  RegionBoundaries[I].second);
1334  return;
1335  }
1336  if (Removing && MI == RegionBoundaries[I].second) {
1337  RegionBoundaries[I] =
1338  std::make_pair(RegionBoundaries[I].first, std::prev(MI));
1339  return;
1340  }
1341  }
1342 }
1343 
1345  return std::any_of(
1346  DAG->begin(), DAG->end(), [](MachineBasicBlock::iterator MI) {
1347  unsigned Opc = MI->getOpcode();
1348  return Opc == AMDGPU::SCHED_GROUP_BARRIER || Opc == AMDGPU::IGLP_OPT;
1349  });
1350 }
1351 
1353  MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S,
1354  bool RemoveKillFlags)
1355  : ScheduleDAGMI(C, std::move(S), RemoveKillFlags) {}
1356 
1358  HasIGLPInstrs = hasIGLPInstrs(this);
1359  if (HasIGLPInstrs) {
1360  SavedMutations.clear();
1361  SavedMutations.swap(Mutations);
1363  }
1364 
1366 }
1367 
1369  if (HasIGLPInstrs)
1370  SavedMutations.swap(Mutations);
1371 
1373 }
llvm::LaneBitmask
Definition: LaneBitmask.h:40
llvm::GenericScheduler::Top
SchedBoundary Top
Definition: MachineScheduler.h:1004
llvm::GCNSchedStageID::OccInitialSchedule
@ OccInitialSchedule
llvm::MachineSchedPolicy::OnlyBottomUp
bool OnlyBottomUp
Definition: MachineScheduler.h:191
llvm::GenericSchedulerBase::SchedResourceDelta::DemandedResources
unsigned DemandedResources
Definition: MachineScheduler.h:848
llvm::ClusteredLowOccStage::initGCNSchedStage
bool initGCNSchedStage() override
Definition: GCNSchedStrategy.cpp:678
llvm::ScheduleDAG::MRI
MachineRegisterInfo & MRI
Virtual/real register map.
Definition: ScheduleDAG.h:560
llvm::GenericSchedulerBase::Weak
@ Weak
Definition: MachineScheduler.h:816
llvm::GCNSchedStrategy::TargetOccupancy
unsigned TargetOccupancy
Definition: GCNSchedStrategy.h:63
llvm::MachineBasicBlock::succ_size
unsigned succ_size() const
Definition: MachineBasicBlock.h:381
llvm::GenericSchedulerBase::TRI
const TargetRegisterInfo * TRI
Definition: MachineScheduler.h:912
llvm::MachineInstr::clearRegisterDeads
void clearRegisterDeads(Register Reg)
Clear all dead flags on operands defining register Reg.
Definition: MachineInstr.cpp:2009
llvm::RegisterOperands::detectDeadDefs
void detectDeadDefs(const MachineInstr &MI, const LiveIntervals &LIS)
Use liveness information to find dead defs not marked with a dead flag and move them to the DeadDefs ...
Definition: RegisterPressure.cpp:581
llvm::GenericSchedulerBase::SchedCandidate::setBest
void setBest(SchedCandidate &Best)
Definition: MachineScheduler.h:896
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:108
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::GCNRegPressure
Definition: GCNRegPressure.h:30
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::GenericSchedulerBase::NodeOrder
@ NodeOrder
Definition: MachineScheduler.h:818
llvm::ClusteredLowOccStage::initGCNRegion
bool initGCNRegion() override
Definition: GCNSchedStrategy.cpp:812
llvm::getLiveRegsBefore
GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, const LiveIntervals &LIS)
Definition: GCNRegPressure.h:253
llvm::GCNSchedStrategy::pickNodeBidirectional
SUnit * pickNodeBidirectional(bool &IsTopNode)
Definition: GCNSchedStrategy.cpp:198
llvm::LiveIntervals::getSlotIndexes
SlotIndexes * getSlotIndexes() const
Definition: LiveIntervals.h:209
llvm::SIMachineFunctionInfo::getMinAllowedOccupancy
unsigned getMinAllowedOccupancy() const
Definition: SIMachineFunctionInfo.h:969
llvm::GCNSchedStage::MF
MachineFunction & MF
Definition: GCNSchedStrategy.h:214
llvm::GCNSchedStrategy::CurrentStage
SmallVectorImpl< GCNSchedStageID >::iterator CurrentStage
Definition: GCNSchedStrategy.h:71
SIMachineFunctionInfo.h
llvm::GenericSchedulerBase::Cluster
@ Cluster
Definition: MachineScheduler.h:816
llvm::MachineSchedContext::RegClassInfo
RegisterClassInfo * RegClassInfo
Definition: MachineScheduler.h:135
llvm::getRegPressure
GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, Range &&LiveRegs)
Definition: GCNRegPressure.h:260
llvm::getLiveRegMap
DenseMap< MachineInstr *, GCNRPTracker::LiveRegSet > getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS)
creates a map MachineInstr -> LiveRegSet R - range of iterators on instructions After - upon entry or...
Definition: GCNRegPressure.h:210
llvm::SchedBoundary::DAG
ScheduleDAGMI * DAG
Definition: MachineScheduler.h:620
llvm::PreRARematStage::shouldRevertScheduling
bool shouldRevertScheduling(unsigned WavesAfter) override
Definition: GCNSchedStrategy.cpp:968
llvm::ScheduleDAGInstrs::begin
MachineBasicBlock::iterator begin() const
Returns an iterator to the top of the current scheduling region.
Definition: ScheduleDAGInstrs.h:278
llvm::BitVector::set
BitVector & set()
Definition: BitVector.h:344
llvm::RegisterOperands
List of registers defined and used by a machine instruction.
Definition: RegisterPressure.h:166
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:95
llvm::GenericSchedulerBase::Stall
@ Stall
Definition: MachineScheduler.h:816
llvm::BitVector::none
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:181
llvm::GenericSchedulerBase::SchedCandidate::reset
void reset(const CandPolicy &NewPolicy)
Definition: MachineScheduler.h:884
llvm::GCNMaxILPSchedStrategy::GCNMaxILPSchedStrategy
GCNMaxILPSchedStrategy(const MachineSchedContext *C)
Definition: GCNSchedStrategy.cpp:352
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::GCNSchedStageID
GCNSchedStageID
Definition: GCNSchedStrategy.h:27
llvm::RegPressureTracker::getUpwardPressure
void getUpwardPressure(const MachineInstr *MI, std::vector< unsigned > &PressureResult, std::vector< unsigned > &MaxPressureResult)
Get the pressure of each PSet after traversing this instruction bottom-up.
Definition: RegisterPressure.cpp:1367
llvm::SchedBoundary::Pending
ReadyQueue Pending
Definition: MachineScheduler.h:625
llvm::ScheduleDAGInstrs::NumRegionInstrs
unsigned NumRegionInstrs
Instructions in this region (distance(RegionBegin, RegionEnd)).
Definition: ScheduleDAGInstrs.h:155
llvm::printMBBReference
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Definition: MachineBasicBlock.cpp:117
GCNSchedStrategy.h
llvm::GenericSchedulerBase::SchedModel
const TargetSchedModel * SchedModel
Definition: MachineScheduler.h:911
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::tryGreater
bool tryGreater(int TryVal, int CandVal, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason)
Definition: MachineScheduler.cpp:2838
llvm::GCNMaxILPSchedStrategy::tryCandidate
bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, SchedBoundary *Zone) const override
Apply a set of heuristics to a new candidate.
Definition: GCNSchedStrategy.cpp:357
llvm::GenericSchedulerBase::SchedCandidate::RPDelta
RegPressureDelta RPDelta
Definition: MachineScheduler.h:876
llvm::GCNSchedStageID::UnclusteredHighRPReschedule
@ UnclusteredHighRPReschedule
llvm::GenericSchedulerBase::RegCritical
@ RegCritical
Definition: MachineScheduler.h:816
RegisterClassInfo.h
llvm::GenericScheduler::tryCandidate
virtual bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, SchedBoundary *Zone) const
Apply a set of heuristics to a new candidate.
Definition: MachineScheduler.cpp:3182
llvm::MachineRegisterInfo::use_instr_nodbg_begin
use_instr_nodbg_iterator use_instr_nodbg_begin(Register RegNo) const
Definition: MachineRegisterInfo.h:543
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:140
llvm::TargetInstrInfo::isTriviallyReMaterializable
bool isTriviallyReMaterializable(const MachineInstr &MI) const
Return true if the instruction is trivially rematerializable, meaning it has no side effects and requ...
Definition: TargetInstrInfo.h:130
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::erase
bool erase(const KeyT &Val)
Definition: DenseMap.h:302
llvm::GCNSchedStrategy::SGPRExcessLimit
unsigned SGPRExcessLimit
Definition: GCNSchedStrategy.h:59
llvm::AMDGPUMachineFunction::getLDSSize
uint32_t getLDSSize() const
Definition: AMDGPUMachineFunction.h:72
llvm::LiveIntervals::removeInterval
void removeInterval(Register Reg)
Interval removal.
Definition: LiveIntervals.h:143
llvm::RegPressureTracker
Track the current register pressure at some position in the instruction stream, and remember the high...
Definition: RegisterPressure.h:357
llvm::GenericSchedulerBase::setPolicy
void setPolicy(CandPolicy &Policy, bool IsPostRA, SchedBoundary &CurrZone, SchedBoundary *OtherZone)
Set the CandPolicy given a scheduling zone given the current resources and latencies inside and outsi...
Definition: MachineScheduler.cpp:2686
llvm::BitVector::resize
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:334
llvm::GenericScheduler::initialize
void initialize(ScheduleDAGMI *dag) override
Initialize the strategy after building the DAG for a new region.
Definition: MachineScheduler.cpp:2897
llvm::SmallSet< unsigned, 16 >
llvm::SIMachineFunctionInfo::increaseOccupancy
void increaseOccupancy(const MachineFunction &MF, unsigned Limit)
Definition: SIMachineFunctionInfo.h:982
llvm::GenericSchedulerBase::SchedCandidate::Reason
CandReason Reason
Definition: MachineScheduler.h:870
llvm::GCNScheduleDAGMILive
Definition: GCNSchedStrategy.h:129
llvm::MachineRegisterInfo::getNumVirtRegs
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
Definition: MachineRegisterInfo.h:770
llvm::GCNSubtarget
Definition: GCNSubtarget.h:31
llvm::Register::index2VirtReg
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
Definition: Register.h:84
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::GenericSchedulerBase::ResourceDemand
@ ResourceDemand
Definition: MachineScheduler.h:817
AMDGPUIGroupLP.h
llvm::GCNRegPressure::less
bool less(const GCNSubtarget &ST, const GCNRegPressure &O, unsigned MaxOccupancy=std::numeric_limits< unsigned >::max()) const
Definition: GCNRegPressure.cpp:89
llvm::SUnit::isTopReady
bool isTopReady() const
Definition: ScheduleDAG.h:446
llvm::ScheduleDAGInstrs::end
MachineBasicBlock::iterator end() const
Returns an iterator to the bottom of the current scheduling region.
Definition: ScheduleDAGInstrs.h:281
llvm::isEqual
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
Definition: GCNRegPressure.cpp:21
llvm::LiveIntervals::createAndComputeVirtRegInterval
LiveInterval & createAndComputeVirtRegInterval(Register Reg)
Definition: LiveIntervals.h:136
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::LiveIntervals::getInstructionIndex
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
Definition: LiveIntervals.h:220
llvm::GCNScheduleDAGMILive::finalizeSchedule
void finalizeSchedule() override
Allow targets to perform final scheduling actions at the level of the whole MachineFunction.
Definition: GCNSchedStrategy.cpp:563
llvm::GCNSchedStrategy::ErrorMargin
unsigned ErrorMargin
Definition: GCNSchedStrategy.h:85
llvm::GCNDownwardRPTracker
Definition: GCNRegPressure.h:159
llvm::ScheduleDAGMI::Mutations
std::vector< std::unique_ptr< ScheduleDAGMutation > > Mutations
Ordered list of DAG postprocessing steps.
Definition: MachineScheduler.h:280
llvm::LiveIntervals::handleMove
void handleMove(MachineInstr &MI, bool UpdateFlags=false)
Call this method to notify LiveIntervals that instruction MI has been moved within a basic block.
Definition: LiveIntervals.cpp:1508
llvm::PreRARematStage::initGCNRegion
bool initGCNRegion() override
Definition: GCNSchedStrategy.cpp:824
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::GenericScheduler::TopCand
SchedCandidate TopCand
Candidate last picked from Top boundary.
Definition: MachineScheduler.h:1008
llvm::GCNSchedStage::MFI
SIMachineFunctionInfo & MFI
Definition: GCNSchedStrategy.h:216
llvm::UnclusteredHighRPStage::initGCNSchedStage
bool initGCNSchedStage() override
Definition: GCNSchedStrategy.cpp:649
llvm::SchedBoundary::isTop
bool isTop() const
Definition: MachineScheduler.h:705
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:98
llvm::MachineBasicBlock::remove
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
Definition: MachineBasicBlock.h:989
llvm::SUnit::isBottomReady
bool isBottomReady() const
Definition: ScheduleDAG.h:449
llvm::UnclusteredHighRPStage::initGCNRegion
bool initGCNRegion() override
Definition: GCNSchedStrategy.cpp:801
llvm::RegPressureDelta::CurrentMax
PressureChange CurrentMax
Definition: RegisterPressure.h:241
llvm::GCNSubtarget::getMaxNumSGPRs
unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const
Definition: GCNSubtarget.h:1157
llvm::GCNSchedStrategy::MF
MachineFunction * MF
Definition: GCNSchedStrategy.h:65
llvm::GCNSchedStrategy::HighRPErrorMargin
const unsigned HighRPErrorMargin
Definition: GCNSchedStrategy.h:83
llvm::GCNRegPressure::getSGPRNum
unsigned getSGPRNum() const
Definition: GCNRegPressure.h:49
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::GCNSchedStage::S
GCNSchedStrategy & S
Definition: GCNSchedStrategy.h:212
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:755
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::GCNRegPressure::getOccupancy
unsigned getOccupancy(const GCNSubtarget &ST) const
Definition: GCNRegPressure.h:63
llvm::ScheduleDAGInstrs::exitRegion
virtual void exitRegion()
Called when the scheduler has finished scheduling the current region.
Definition: ScheduleDAGInstrs.cpp:193
llvm::RegPressureTracker::getDownwardPressure
void getDownwardPressure(const MachineInstr *MI, std::vector< unsigned > &PressureResult, std::vector< unsigned > &MaxPressureResult)
Get the pressure of each PSet after traversing this instruction top-down.
Definition: RegisterPressure.cpp:1383
llvm::tryLatency
bool tryLatency(GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary &Zone)
Definition: MachineScheduler.cpp:2854
llvm::LiveIntervals::InsertMachineInstrInMaps
SlotIndex InsertMachineInstrInMaps(MachineInstr &MI)
Definition: LiveIntervals.h:260
llvm::SUnit::NodeNum
unsigned NodeNum
Entry # of node in the node vector.
Definition: ScheduleDAG.h:264
llvm::GCNSubtarget::getOccupancyWithNumSGPRs
unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const
Return the maximum number of waves per SIMD for kernels using SGPRs SGPRs.
Definition: AMDGPUSubtarget.cpp:602
llvm::LaneBitmask::getNone
static constexpr LaneBitmask getNone()
Definition: LaneBitmask.h:83
llvm::GenericSchedulerBase::RegMax
@ RegMax
Definition: MachineScheduler.h:817
false
Definition: StackSlotColoring.cpp:141
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:48
llvm::GenericSchedulerBase::CandPolicy
Policy for scheduling the next instruction in the candidate's zone.
Definition: MachineScheduler.h:825
llvm::ScheduleDAGMI::addMutation
void addMutation(std::unique_ptr< ScheduleDAGMutation > Mutation)
Add a postprocessing step to the DAG builder.
Definition: MachineScheduler.h:325
llvm::UnclusteredHighRPStage::shouldRevertScheduling
bool shouldRevertScheduling(unsigned WavesAfter) override
Definition: GCNSchedStrategy.cpp:945
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:642
llvm::operator<<
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:291
llvm::SlotIndexes
SlotIndexes pass.
Definition: SlotIndexes.h:319
llvm::GCNSchedStageID::PreRARematerialize
@ PreRARematerialize
llvm::ScheduleDAGMI::startBlock
void startBlock(MachineBasicBlock *bb) override
Prepares to perform scheduling in the given block.
Definition: MachineScheduler.cpp:708
DisableUnclusterHighRP
static cl::opt< bool > DisableUnclusterHighRP("amdgpu-disable-unclustred-high-rp-reschedule", cl::Hidden, cl::desc("Disable unclustred high register pressure " "reduction scheduling stage."), cl::init(false))
llvm::GCNSchedStage::finalizeGCNSchedStage
virtual void finalizeGCNSchedStage()
Definition: GCNSchedStrategy.cpp:723
llvm::RegPressureDelta::Excess
PressureChange Excess
Definition: RegisterPressure.h:239
llvm::ScheduleDAGInstrs::RegionEnd
MachineBasicBlock::iterator RegionEnd
The end of the range to be scheduled.
Definition: ScheduleDAGInstrs.h:152
llvm::SIRegisterInfo
Definition: SIRegisterInfo.h:30
llvm::BitVector
Definition: BitVector.h:75
llvm::GCNSchedStrategy::MaxPressure
std::vector< unsigned > MaxPressure
Definition: GCNSchedStrategy.h:57
llvm::SlotIndex
SlotIndex - An opaque wrapper around machine indexes.
Definition: SlotIndexes.h:82
llvm::GCNSubtarget::getMaxNumVGPRs
unsigned getMaxNumVGPRs(unsigned WavesPerEU) const
Definition: GCNSubtarget.h:1227
llvm::GCNSchedStrategy::initialize
void initialize(ScheduleDAGMI *DAG) override
Initialize the strategy after building the DAG for a new region.
Definition: GCNSchedStrategy.cpp:46
llvm::GCNSchedStrategy::getNextStage
GCNSchedStageID getNextStage() const
Definition: GCNSchedStrategy.cpp:338
llvm::GenericSchedulerBase::SchedCandidate::SU
SUnit * SU
Definition: MachineScheduler.h:867
llvm::ScheduleDAGMILive::RPTracker
RegPressureTracker RPTracker
Definition: MachineScheduler.h:415
llvm::GenericScheduler::RegionPolicy
MachineSchedPolicy RegionPolicy
Definition: MachineScheduler.h:1001
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::RegisterOperands::collect
void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, bool TrackLaneMasks, bool IgnoreDead)
Analyze the given instruction MI and fill in the Uses, Defs and DeadDefs list based on the MachineOpe...
Definition: RegisterPressure.cpp:570
llvm::GCNSchedStrategy::getTargetOccupancy
unsigned getTargetOccupancy()
Definition: GCNSchedStrategy.h:97
llvm::RegisterClassInfo::getNumAllocatableRegs
unsigned getNumAllocatableRegs(const TargetRegisterClass *RC) const
getNumAllocatableRegs - Returns the number of actually allocatable registers in RC in the current fun...
Definition: RegisterClassInfo.h:94
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Definition: MachineRegisterInfo.h:647
llvm::tryLess
bool tryLess(int TryVal, int CandVal, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason)
Return true if this heuristic determines order.
Definition: MachineScheduler.cpp:2822
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:657
llvm::RegPressureTracker::getRegSetPressureAtPos
const std::vector< unsigned > & getRegSetPressureAtPos() const
Get the register set pressure at the current position, which may be less than the pressure across the...
Definition: RegisterPressure.h:463
llvm::cl::opt< bool >
llvm::GenericSchedulerBase::SchedCandidate
Store the state used by GenericScheduler heuristics, required for the lifetime of one invocation of p...
Definition: MachineScheduler.h:863
llvm::GenericSchedulerBase::NoCand
@ NoCand
Definition: MachineScheduler.h:816
llvm::GCNSchedStage::Unsched
std::vector< MachineInstr * > Unsched
Definition: GCNSchedStrategy.h:229
llvm::ScheduleDAGMI::LIS
LiveIntervals * LIS
Definition: MachineScheduler.h:276
llvm::GCNSchedStage::initGCNSchedStage
virtual bool initGCNSchedStage()
Definition: GCNSchedStrategy.cpp:641
llvm::GCNSchedStage::setupNewBlock
void setupNewBlock()
Definition: GCNSchedStrategy.cpp:831
llvm::SlotIndexes::getMBBStartIdx
SlotIndex getMBBStartIdx(unsigned Num) const
Returns the first index in the given basic block number.
Definition: SlotIndexes.h:471
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::GCNSchedStrategy::initCandidate
void initCandidate(SchedCandidate &Cand, SUnit *SU, bool AtTop, const RegPressureTracker &RPTracker, const SIRegisterInfo *SRI, unsigned SGPRPressure, unsigned VGPRPressure)
Definition: GCNSchedStrategy.cpp:77
llvm::SIRegisterInfo::isVGPRClass
static bool isVGPRClass(const TargetRegisterClass *RC)
Definition: SIRegisterInfo.h:202
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::RegPressureTracker::reset
void reset()
Definition: RegisterPressure.cpp:243
llvm::SIMachineFunctionInfo::getMaxWavesPerEU
unsigned getMaxWavesPerEU() const
Definition: SIMachineFunctionInfo.h:930
llvm::GCNSchedStage::mayCauseSpilling
bool mayCauseSpilling(unsigned WavesAfter)
Definition: GCNSchedStrategy.cpp:985
llvm::GenericSchedulerBase::Context
const MachineSchedContext * Context
Definition: MachineScheduler.h:910
llvm::tryPressure
bool tryPressure(const PressureChange &TryP, const PressureChange &CandP, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason, const TargetRegisterInfo *TRI, const MachineFunction &MF)
Definition: MachineScheduler.cpp:3047
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::DenseMap
Definition: DenseMap.h:714
llvm::SchedBoundary::getLatencyStallCycles
unsigned getLatencyStallCycles(SUnit *SU)
Get the difference between the given SUnit's ready time and the current cycle.
Definition: MachineScheduler.cpp:2073
llvm::MachineSchedContext
MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...
Definition: MachineScheduler.h:127
llvm::PressureChange::setUnitInc
void setUnitInc(int Inc)
Definition: RegisterPressure.h:126
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SUnit::getInstr
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:373
llvm::GCNSchedStage::revertScheduling
void revertScheduling()
Definition: GCNSchedStrategy.cpp:996
llvm::GCNSchedStage::DAG
GCNScheduleDAGMILive & DAG
Definition: GCNSchedStrategy.h:210
llvm::GCNSchedStrategy
This is a minimal scheduler strategy.
Definition: GCNSchedStrategy.h:42
llvm::RegPressureTracker::advance
void advance()
Advance across the current instruction.
Definition: RegisterPressure.cpp:948
llvm::SUnit::isScheduled
bool isScheduled
True once scheduled.
Definition: ScheduleDAG.h:284
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:447
llvm::GCNSchedStage::StageID
const GCNSchedStageID StageID
Definition: GCNSchedStrategy.h:220
llvm::SIMachineFunctionInfo::getMinWavesPerEU
unsigned getMinWavesPerEU() const
Definition: SIMachineFunctionInfo.h:925
llvm::MachineFunction::getName
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Definition: MachineFunction.cpp:567
llvm::RegPressureDelta::CriticalMax
PressureChange CriticalMax
Definition: RegisterPressure.h:240
llvm::MachineRegisterInfo::getOneDef
MachineOperand * getOneDef(Register Reg) const
Returns the defining operand if there is exactly one operand defining the specified register,...
Definition: MachineRegisterInfo.h:458
llvm::ClusteredLowOccStage::shouldRevertScheduling
bool shouldRevertScheduling(unsigned WavesAfter) override
Definition: GCNSchedStrategy.cpp:958
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::move
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1836
llvm::GCNSchedStage::RegionIdx
unsigned RegionIdx
Definition: GCNSchedStrategy.h:226
llvm::MachineBasicBlock::succ_begin
succ_iterator succ_begin()
Definition: MachineBasicBlock.h:369
llvm::GenericSchedulerBase::SchedCandidate::Policy
CandPolicy Policy
Definition: MachineScheduler.h:864
llvm::GenericSchedulerBase::SchedResourceDelta::CritResources
unsigned CritResources
Definition: MachineScheduler.h:845
llvm::GCNSubtarget::hasGFX90AInsts
bool hasGFX90AInsts() const
Definition: GCNSubtarget.h:1048
llvm::GCNSubtarget::getOccupancyWithNumVGPRs
unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const
Return the maximum number of waves per SIMD for kernels using VGPRs VGPRs.
Definition: AMDGPUSubtarget.cpp:628
llvm::GenericScheduler::BotCand
SchedCandidate BotCand
Candidate last picked from Bot boundary.
Definition: MachineScheduler.h:1010
llvm::MachineRegisterInfo::hasOneDef
bool hasOneDef(Register RegNo) const
Return true if there is exactly one operand defining the specified register.
Definition: MachineRegisterInfo.h:452
llvm::ScheduleDAGMI
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
Definition: MachineScheduler.h:273
llvm::GCNPostScheduleDAGMILive::finalizeSchedule
void finalizeSchedule() override
Allow targets to perform final scheduling actions at the level of the whole MachineFunction.
Definition: GCNSchedStrategy.cpp:1368
llvm::GCNSchedStage::initGCNRegion
virtual bool initGCNRegion()
Definition: GCNSchedStrategy.cpp:744
llvm::ScheduleDAGMI::placeDebugValues
void placeDebugValues()
Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues.
Definition: MachineScheduler.cpp:907
llvm::GCNSchedStageID::ILPInitialSchedule
@ ILPInitialSchedule
llvm::SIMachineFunctionInfo::getOccupancy
unsigned getOccupancy() const
Definition: SIMachineFunctionInfo.h:965
llvm::SlotIndex::getRegSlot
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
Definition: SlotIndexes.h:259
llvm::GCNPostScheduleDAGMILive::GCNPostScheduleDAGMILive
GCNPostScheduleDAGMILive(MachineSchedContext *C, std::unique_ptr< MachineSchedStrategy > S, bool RemoveKillFlags)
Definition: GCNSchedStrategy.cpp:1352
llvm::print
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr)
Definition: GCNRegPressure.cpp:138
llvm::GCNSchedStage::CurrentMBB
MachineBasicBlock * CurrentMBB
Definition: GCNSchedStrategy.h:223
llvm::GenericScheduler::Bot
SchedBoundary Bot
Definition: MachineScheduler.h:1005
llvm::ScheduleDAGInstrs::finalizeSchedule
virtual void finalizeSchedule()
Allow targets to perform final scheduling actions at the level of the whole MachineFunction.
Definition: ScheduleDAGInstrs.h:339
llvm::skipDebugInstructionsForward
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
Definition: MachineBasicBlock.h:1265
llvm::ScheduleDAG::TRI
const TargetRegisterInfo * TRI
Target processor register info.
Definition: ScheduleDAG.h:558
llvm::MachineBasicBlock::iterator
MachineInstrBundleIterator< MachineInstr > iterator
Definition: MachineBasicBlock.h:269
llvm::ArrayRef< unsigned >
llvm::UnclusteredHighRPStage::finalizeGCNSchedStage
void finalizeGCNSchedStage() override
Definition: GCNSchedStrategy.cpp:728
llvm::GenericScheduler::DAG
ScheduleDAGMILive * DAG
Definition: MachineScheduler.h:999
llvm::GCNScheduleDAGMILive::schedule
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
Definition: GCNSchedStrategy.cpp:468
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::SIRegisterInfo::getNumCoveredRegs
static unsigned getNumCoveredRegs(LaneBitmask LM)
Definition: SIRegisterInfo.h:362
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1716
llvm::ScheduleDAGMILive::enterRegion
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
Definition: MachineScheduler.cpp:989
llvm::GCNSchedStrategy::SchedStages
SmallVector< GCNSchedStageID, 4 > SchedStages
Definition: GCNSchedStrategy.h:68
llvm::PreRARematStage::initGCNSchedStage
bool initGCNSchedStage() override
Definition: GCNSchedStrategy.cpp:694
llvm::OccInitialScheduleStage::shouldRevertScheduling
bool shouldRevertScheduling(unsigned WavesAfter) override
Definition: GCNSchedStrategy.cpp:935
llvm::GCNSchedStage::finalizeGCNRegion
void finalizeGCNRegion()
Definition: GCNSchedStrategy.cpp:843
llvm::MachineRegisterInfo::hasOneNonDBGUse
bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
Definition: MachineRegisterInfo.cpp:415
llvm::ScheduleDAGMILive::getBotRPTracker
const RegPressureTracker & getBotRPTracker() const
Definition: MachineScheduler.h:451
llvm::ScheduleDAG::MF
MachineFunction & MF
Machine function.
Definition: ScheduleDAG.h:559
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
if
if(llvm_vc STREQUAL "") set(fake_version_inc "$
Definition: CMakeLists.txt:14
llvm::GCNSchedStrategy::advanceStage
bool advanceStage()
Definition: GCNSchedStrategy.cpp:323
llvm::SIMachineFunctionInfo::limitOccupancy
void limitOccupancy(const MachineFunction &MF)
Definition: SIMachineFunctionInfo.cpp:210
llvm::ScheduleDAGMILive::ShouldTrackLaneMasks
bool ShouldTrackLaneMasks
Definition: MachineScheduler.h:413
llvm::biasPhysReg
int biasPhysReg(const SUnit *SU, bool isTop)
Minimize physical register live ranges.
Definition: MachineScheduler.cpp:3097
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::GenericSchedulerBase::traceCandidate
void traceCandidate(const SchedCandidate &Cand)
Definition: MachineScheduler.cpp:2765
llvm::GCNSchedStrategy::HasHighPressure
bool HasHighPressure
Definition: GCNSchedStrategy.h:76
llvm::GCNSchedStrategy::DefaultErrorMargin
const unsigned DefaultErrorMargin
Definition: GCNSchedStrategy.h:81
llvm::GenericSchedulerBase::SchedCandidate::ResDelta
SchedResourceDelta ResDelta
Definition: MachineScheduler.h:879
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:313
llvm::GCNSchedStage::checkScheduling
void checkScheduling()
Definition: GCNSchedStrategy.cpp:861
llvm::GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy
GCNMaxOccupancySchedStrategy(const MachineSchedContext *C)
Definition: GCNSchedStrategy.cpp:343
llvm::VerifyScheduling
cl::opt< bool > VerifyScheduling
llvm::SchedBoundary::pickOnlyChoice
SUnit * pickOnlyChoice()
Call this before applying any other heuristics to the Available queue.
Definition: MachineScheduler.cpp:2561
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::GCNSchedStrategy::SGPRCriticalLimit
unsigned SGPRCriticalLimit
Definition: GCNSchedStrategy.h:87
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::GCNSchedStage::PressureAfter
GCNRegPressure PressureAfter
Definition: GCNSchedStrategy.h:235
llvm::GenericSchedulerBase::SchedCandidate::initResourceDelta
void initResourceDelta(const ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel)
Definition: MachineScheduler.cpp:2622
llvm::ScheduleDAGInstrs::RegionBegin
MachineBasicBlock::iterator RegionBegin
The beginning of the range to be scheduled.
Definition: ScheduleDAGInstrs.h:149
std
Definition: BitVector.h:851
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:623
llvm::MachineSchedPolicy::OnlyTopDown
bool OnlyTopDown
Definition: MachineScheduler.h:190
llvm::GenericSchedulerBase::RegExcess
@ RegExcess
Definition: MachineScheduler.h:816
llvm::GCNRegPressure::getVGPRNum
unsigned getVGPRNum(bool UnifiedVGPRFile) const
Definition: GCNRegPressure.h:50
llvm::ReadyQueue::empty
bool empty() const
Definition: MachineScheduler.h:549
llvm::GCNSchedStrategy::VGPRExcessLimit
unsigned VGPRExcessLimit
Definition: GCNSchedStrategy.h:61
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::MachineBasicBlock::insert
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
Definition: MachineBasicBlock.cpp:1327
llvm::ScheduleDAGMI::bottom
MachineBasicBlock::iterator bottom() const
Definition: MachineScheduler.h:331
llvm::GenericSchedulerBase::SchedCandidate::AtTop
bool AtTop
Definition: MachineScheduler.h:873
llvm::ScheduleDAG::TII
const TargetInstrInfo * TII
Target instruction information.
Definition: ScheduleDAG.h:557
llvm::GCNSchedStage::GCNSchedStage
GCNSchedStage(GCNSchedStageID StageID, GCNScheduleDAGMILive &DAG)
Definition: GCNSchedStrategy.cpp:637
llvm::GCNSchedStage::SavedMutations
std::vector< std::unique_ptr< ScheduleDAGMutation > > SavedMutations
Definition: GCNSchedStrategy.h:237
llvm::ScheduleDAGMI::getNextClusterPred
const SUnit * getNextClusterPred() const
Definition: MachineScheduler.h:352
hasIGLPInstrs
static bool hasIGLPInstrs(ScheduleDAGInstrs *DAG)
Definition: GCNSchedStrategy.cpp:1344
llvm::ScheduleDAGMI::finishBlock
void finishBlock() override
Cleans up after scheduling in the given block.
Definition: MachineScheduler.cpp:713
llvm::ScheduleDAGMI::getNextClusterSucc
const SUnit * getNextClusterSucc() const
Definition: MachineScheduler.h:354
llvm::ILPInitialScheduleStage::shouldRevertScheduling
bool shouldRevertScheduling(unsigned WavesAfter) override
Definition: GCNSchedStrategy.cpp:978
llvm::ScheduleDAGMILive::isTrackingPressure
bool isTrackingPressure() const
Return true if register pressure tracking is enabled.
Definition: MachineScheduler.h:443
llvm::PressureChange
Capture a change in pressure for a single pressure set.
Definition: RegisterPressure.h:102
llvm::SchedBoundary
Each Scheduling boundary is associated with ready queues.
Definition: MachineScheduler.h:611
llvm::LiveIntervals
Definition: LiveIntervals.h:53
llvm::GCNSchedStrategy::hasNextStage
bool hasNextStage() const
Definition: GCNSchedStrategy.cpp:333
llvm::GCNSchedStrategy::pickNode
SUnit * pickNode(bool &IsTopNode) override
Pick the best node to balance the schedule. Implements MachineSchedStrategy.
Definition: GCNSchedStrategy.cpp:275
llvm::SmallSet::insert
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:178
llvm::GenericSchedulerBase::SchedResourceDelta
Status of an instruction's critical resource consumption.
Definition: MachineScheduler.h:843
llvm::GCNSchedStageID::ClusteredLowOccupancyReschedule
@ ClusteredLowOccupancyReschedule
llvm::GCNSchedStrategy::pickNodeFromQueue
void pickNodeFromQueue(SchedBoundary &Zone, const CandPolicy &ZonePolicy, const RegPressureTracker &RPTracker, SchedCandidate &Cand)
Definition: GCNSchedStrategy.cpp:165
llvm::BitVector::reset
BitVector & reset()
Definition: BitVector.h:385
llvm::RegisterOperands::adjustLaneLiveness
void adjustLaneLiveness(const LiveIntervals &LIS, const MachineRegisterInfo &MRI, SlotIndex Pos, MachineInstr *AddFlagsMI=nullptr)
Use liveness information to find out which uses/defs are partially undefined/dead and adjust the Regi...
Definition: RegisterPressure.cpp:601
llvm::ScheduleDAGInstrs::BB
MachineBasicBlock * BB
The block in which to insert instructions.
Definition: ScheduleDAGInstrs.h:146
llvm::GenericSchedulerBase::ResourceReduce
@ ResourceReduce
Definition: MachineScheduler.h:817
llvm::ScheduleDAGMILive::schedule
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
Definition: MachineScheduler.cpp:1215
llvm::SchedBoundary::Available
ReadyQueue Available
Definition: MachineScheduler.h:624
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:305
llvm::LiveIntervals::RemoveMachineInstrFromMaps
void RemoveMachineInstrFromMaps(MachineInstr &MI)
Definition: LiveIntervals.h:270
llvm::SIMachineFunctionInfo
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Definition: SIMachineFunctionInfo.h:351
llvm::ScheduleDAGMILive::getTopRPTracker
const RegPressureTracker & getTopRPTracker() const
Definition: MachineScheduler.h:447
llvm::GCNSchedStage::ST
const GCNSubtarget & ST
Definition: GCNSchedStrategy.h:218
llvm::ScheduleDAGMI::top
MachineBasicBlock::iterator top() const
Definition: MachineScheduler.h:330
llvm::ScheduleDAGMI::schedule
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
Definition: MachineScheduler.cpp:767
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::SUnit
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:242
llvm::GCNSchedStrategy::GCNSchedStrategy
GCNSchedStrategy(const MachineSchedContext *C)
Definition: GCNSchedStrategy.cpp:42
llvm::getWeakLeft
unsigned getWeakLeft(const SUnit *SU, bool isTop)
Definition: MachineScheduler.cpp:3086
llvm::GCNSchedStrategy::Pressure
std::vector< unsigned > Pressure
Definition: GCNSchedStrategy.h:55
llvm::LiveIntervals::hasInterval
bool hasInterval(Register Reg) const
Definition: LiveIntervals.h:123
llvm::GCNSchedStrategy::VGPRCriticalLimit
unsigned VGPRCriticalLimit
Definition: GCNSchedStrategy.h:89
llvm::GCNSchedStrategy::getCurrentStage
GCNSchedStageID getCurrentStage()
Definition: GCNSchedStrategy.cpp:318
llvm::cl::desc
Definition: CommandLine.h:413
llvm::ScheduleDAGMILive
ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules machine instructions while...
Definition: MachineScheduler.h:392
llvm::Region
Definition: RegionInfo.h:889
llvm::ScheduleDAGInstrs
A ScheduleDAG for scheduling lists of MachineInstr.
Definition: ScheduleDAGInstrs.h:120
llvm::ReadyQueue
Helpers for implementing custom MachineSchedStrategy classes.
Definition: MachineScheduler.h:534
llvm::GenericScheduler
GenericScheduler shrinks the unscheduled zone using heuristics to balance the schedule.
Definition: MachineScheduler.h:954
llvm::MachineInstr::eraseFromParent
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Definition: MachineInstr.cpp:692
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::GCNPostScheduleDAGMILive::schedule
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
Definition: GCNSchedStrategy.cpp:1357
llvm::GCNSubtarget::computeOccupancy
unsigned computeOccupancy(const Function &F, unsigned LDSSize=0, unsigned NumSGPRs=0, unsigned NumVGPRs=0) const
Return occupancy for the given function.
Definition: AMDGPUSubtarget.cpp:667
llvm::GCNSchedStage::PressureBefore
GCNRegPressure PressureBefore
Definition: GCNSchedStrategy.h:232
llvm::MachineBasicBlock::getName
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
Definition: MachineBasicBlock.cpp:311
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:307
llvm::SchedBoundary::removeReady
void removeReady(SUnit *SU)
Remove SU from the ready set for this boundary.
Definition: MachineScheduler.cpp:2549
llvm::createIGroupLPDAGMutation
std::unique_ptr< ScheduleDAGMutation > createIGroupLPDAGMutation()
Definition: AMDGPUIGroupLP.cpp:1156
llvm::GCNRegPressure::getAGPRNum
unsigned getAGPRNum() const
Definition: GCNRegPressure.h:57
llvm::GenericSchedulerBase::SchedCandidate::isValid
bool isValid() const
Definition: MachineScheduler.h:893
llvm::GCNScheduleDAGMILive::GCNScheduleDAGMILive
GCNScheduleDAGMILive(MachineSchedContext *C, std::unique_ptr< MachineSchedStrategy > S)
Definition: GCNSchedStrategy.cpp:441
llvm::GenericSchedulerBase::PhysReg
@ PhysReg
Definition: MachineScheduler.h:816
llvm::GCNSchedStage::shouldRevertScheduling
virtual bool shouldRevertScheduling(unsigned WavesAfter)
Definition: GCNSchedStrategy.cpp:928