LLVM  15.0.0git
GCNHazardRecognizer.cpp
Go to the documentation of this file.
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "GCNSubtarget.h"
16 #include "SIMachineFunctionInfo.h"
20 
21 using namespace llvm;
22 
23 namespace {
24 
25 struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
26  MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
27 
28  bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
29  if (Arg.getAsInteger(0, Value))
30  return O.error("'" + Arg + "' value invalid for uint argument!");
31 
32  if (Value > 100)
33  return O.error("'" + Arg + "' value must be in the range [0, 100]!");
34 
35  return false;
36  }
37 };
38 
39 } // end anonymous namespace
40 
42  MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden,
43  cl::desc("Fill a percentage of the latency between "
44  "neighboring MFMA with s_nops."));
45 
46 //===----------------------------------------------------------------------===//
47 // Hazard Recognizer Implementation
48 //===----------------------------------------------------------------------===//
49 
51  const GCNSubtarget &ST);
52 
54  IsHazardRecognizerMode(false),
55  CurrCycleInstr(nullptr),
56  MF(MF),
57  ST(MF.getSubtarget<GCNSubtarget>()),
58  TII(*ST.getInstrInfo()),
59  TRI(TII.getRegisterInfo()),
60  ClauseUses(TRI.getNumRegUnits()),
61  ClauseDefs(TRI.getNumRegUnits()) {
62  MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
63  TSchedModel.init(&ST);
64  RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
65 }
66 
68  EmittedInstrs.clear();
69 }
70 
73 }
74 
76  CurrCycleInstr = MI;
77 }
78 
79 static bool isDivFMas(unsigned Opcode) {
80  return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
81 }
82 
83 static bool isSGetReg(unsigned Opcode) {
84  return Opcode == AMDGPU::S_GETREG_B32;
85 }
86 
87 static bool isSSetReg(unsigned Opcode) {
88  switch (Opcode) {
89  case AMDGPU::S_SETREG_B32:
90  case AMDGPU::S_SETREG_B32_mode:
91  case AMDGPU::S_SETREG_IMM32_B32:
92  case AMDGPU::S_SETREG_IMM32_B32_mode:
93  return true;
94  }
95  return false;
96 }
97 
98 static bool isRWLane(unsigned Opcode) {
99  return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
100 }
101 
102 static bool isRFE(unsigned Opcode) {
103  return Opcode == AMDGPU::S_RFE_B64;
104 }
105 
106 static bool isSMovRel(unsigned Opcode) {
107  switch (Opcode) {
108  case AMDGPU::S_MOVRELS_B32:
109  case AMDGPU::S_MOVRELS_B64:
110  case AMDGPU::S_MOVRELD_B32:
111  case AMDGPU::S_MOVRELD_B64:
112  return true;
113  default:
114  return false;
115  }
116 }
117 
118 static bool isDGEMM(unsigned Opcode) {
119  return AMDGPU::getMAIIsDGEMM(Opcode);
120 }
121 
122 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
123  unsigned Opcode = MI.getOpcode();
124 
125  if (!SIInstrInfo::isMAI(MI) ||
126  isDGEMM(Opcode) ||
127  Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
128  Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
129  return false;
130 
131  if (!ST.hasGFX940Insts())
132  return true;
133 
134  return AMDGPU::getMAIIsGFX940XDL(Opcode);
135 }
136 
138  const MachineInstr &MI) {
139  if (TII.isAlwaysGDS(MI.getOpcode()))
140  return true;
141 
142  switch (MI.getOpcode()) {
143  case AMDGPU::S_SENDMSG:
144  case AMDGPU::S_SENDMSGHALT:
145  case AMDGPU::S_TTRACEDATA:
146  return true;
147  // These DS opcodes don't support GDS.
148  case AMDGPU::DS_NOP:
149  case AMDGPU::DS_PERMUTE_B32:
150  case AMDGPU::DS_BPERMUTE_B32:
151  return false;
152  default:
153  if (TII.isDS(MI.getOpcode())) {
154  int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
155  AMDGPU::OpName::gds);
156  if (MI.getOperand(GDS).getImm())
157  return true;
158  }
159  return false;
160  }
161 }
162 
163 static bool isPermlane(const MachineInstr &MI) {
164  unsigned Opcode = MI.getOpcode();
165  return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
166  Opcode == AMDGPU::V_PERMLANEX16_B32_e64;
167 }
168 
169 static bool isLdsDma(const MachineInstr &MI) {
170  return SIInstrInfo::isVALU(MI) &&
172 }
173 
174 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
175  const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
176  AMDGPU::OpName::simm16);
177  return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
178 }
179 
182  MachineInstr *MI = SU->getInstr();
183  // If we are not in "HazardRecognizerMode" and therefore not being run from
184  // the scheduler, track possible stalls from hazards but don't insert noops.
185  auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard;
186 
187  if (MI->isBundle())
188  return NoHazard;
189 
190  if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
191  return HazardType;
192 
193  if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
194  return HazardType;
195 
196  if (checkFPAtomicToDenormModeHazard(MI) > 0)
197  return HazardType;
198 
199  if (ST.hasNoDataDepHazard())
200  return NoHazard;
201 
202  // FIXME: Should flat be considered vmem?
203  if ((SIInstrInfo::isVMEM(*MI) ||
205  && checkVMEMHazards(MI) > 0)
206  return HazardType;
207 
208  if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
209  return HazardType;
210 
211  if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
212  return HazardType;
213 
214  if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
215  return HazardType;
216 
217  if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
218  return HazardType;
219 
222  SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
223  return HazardType;
224 
225  if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
226  return HazardType;
227 
228  if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
229  return HazardType;
230 
231  if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
232  return HazardType;
233 
234  if (((ST.hasReadM0MovRelInterpHazard() &&
235  (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) ||
237  (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
238  (ST.hasReadM0LdsDirectHazard() &&
239  MI->readsRegister(AMDGPU::LDS_DIRECT))) &&
240  checkReadM0Hazards(MI) > 0)
241  return HazardType;
242 
243  if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
244  return HazardType;
245 
246  if ((SIInstrInfo::isVMEM(*MI) ||
248  SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0)
249  return HazardType;
250 
251  if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
252  return HazardType;
253 
254  return NoHazard;
255 }
256 
258  unsigned Quantity) {
259  while (Quantity > 0) {
260  unsigned Arg = std::min(Quantity, 8u);
261  Quantity -= Arg;
262  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
263  .addImm(Arg - 1);
264  }
265 }
266 
267 unsigned
268 GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
269  const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI);
270  assert(TSchedModel.getWriteProcResBegin(SC) !=
271  TSchedModel.getWriteProcResEnd(SC));
272  return TSchedModel.getWriteProcResBegin(SC)->Cycles;
273 }
274 
275 void GCNHazardRecognizer::processBundle() {
276  MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
278  // Check bundled MachineInstr's for hazards.
279  for (; MI != E && MI->isInsideBundle(); ++MI) {
280  CurrCycleInstr = &*MI;
281  unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
282 
283  if (IsHazardRecognizerMode) {
284  fixHazards(CurrCycleInstr);
285 
286  insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
287  }
288 
289  // It’s unnecessary to track more than MaxLookAhead instructions. Since we
290  // include the bundled MI directly after, only add a maximum of
291  // (MaxLookAhead - 1) noops to EmittedInstrs.
292  for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
293  EmittedInstrs.push_front(nullptr);
294 
295  EmittedInstrs.push_front(CurrCycleInstr);
296  EmittedInstrs.resize(MaxLookAhead);
297  }
298  CurrCycleInstr = nullptr;
299 }
300 
302  IsHazardRecognizerMode = true;
303  CurrCycleInstr = MI;
304  unsigned W = PreEmitNoopsCommon(MI);
305  fixHazards(MI);
306  CurrCycleInstr = nullptr;
307  return W;
308 }
309 
311  if (MI->isBundle())
312  return 0;
313 
314  int WaitStates = 0;
315 
316  if (SIInstrInfo::isSMRD(*MI))
317  return std::max(WaitStates, checkSMRDHazards(MI));
318 
319  if (ST.hasNSAtoVMEMBug())
320  WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
321 
322  WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI));
323 
324  if (ST.hasNoDataDepHazard())
325  return WaitStates;
326 
328  WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
329 
330  if (SIInstrInfo::isVALU(*MI))
331  WaitStates = std::max(WaitStates, checkVALUHazards(MI));
332 
333  if (SIInstrInfo::isDPP(*MI))
334  WaitStates = std::max(WaitStates, checkDPPHazards(MI));
335 
336  if (isDivFMas(MI->getOpcode()))
337  WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
338 
339  if (isRWLane(MI->getOpcode()))
340  WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
341 
344  SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
345  WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
346 
347  if (MI->isInlineAsm())
348  return std::max(WaitStates, checkInlineAsmHazards(MI));
349 
350  if (isSGetReg(MI->getOpcode()))
351  return std::max(WaitStates, checkGetRegHazards(MI));
352 
353  if (isSSetReg(MI->getOpcode()))
354  return std::max(WaitStates, checkSetRegHazards(MI));
355 
356  if (isRFE(MI->getOpcode()))
357  return std::max(WaitStates, checkRFEHazards(MI));
358 
359  if ((ST.hasReadM0MovRelInterpHazard() &&
360  (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()))) ||
362  (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
363  (ST.hasReadM0LdsDirectHazard() && MI->readsRegister(AMDGPU::LDS_DIRECT)))
364  return std::max(WaitStates, checkReadM0Hazards(MI));
365 
366  if (SIInstrInfo::isMAI(*MI))
367  return std::max(WaitStates, checkMAIHazards(MI));
368 
369  if (SIInstrInfo::isVMEM(*MI) ||
372  return std::max(WaitStates, checkMAILdStHazards(MI));
373 
374  return WaitStates;
375 }
376 
378  EmittedInstrs.push_front(nullptr);
379 }
380 
382  // When the scheduler detects a stall, it will call AdvanceCycle() without
383  // emitting any instructions.
384  if (!CurrCycleInstr) {
385  EmittedInstrs.push_front(nullptr);
386  return;
387  }
388 
389  if (CurrCycleInstr->isBundle()) {
390  processBundle();
391  return;
392  }
393 
394  unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
395  if (!NumWaitStates) {
396  CurrCycleInstr = nullptr;
397  return;
398  }
399 
400  // Keep track of emitted instructions
401  EmittedInstrs.push_front(CurrCycleInstr);
402 
403  // Add a nullptr for each additional wait state after the first. Make sure
404  // not to add more than getMaxLookAhead() items to the list, since we
405  // truncate the list to that size right after this loop.
406  for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
407  i < e; ++i) {
408  EmittedInstrs.push_front(nullptr);
409  }
410 
411  // getMaxLookahead() is the largest number of wait states we will ever need
412  // to insert, so there is no point in keeping track of more than that many
413  // wait states.
414  EmittedInstrs.resize(getMaxLookAhead());
415 
416  CurrCycleInstr = nullptr;
417 }
418 
420  llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
421 }
422 
423 //===----------------------------------------------------------------------===//
424 // Helper Functions
425 //===----------------------------------------------------------------------===//
426 
427 typedef function_ref<bool(const MachineInstr &, int WaitStates)> IsExpiredFn;
428 
429 // Returns a minimum wait states since \p I walking all predecessors.
430 // Only scans until \p IsExpired does not return true.
431 // Can only be run in a hazard recognizer mode.
433  const MachineBasicBlock *MBB,
435  int WaitStates, IsExpiredFn IsExpired,
437  for (auto E = MBB->instr_rend(); I != E; ++I) {
438  // Don't add WaitStates for parent BUNDLE instructions.
439  if (I->isBundle())
440  continue;
441 
442  if (IsHazard(*I))
443  return WaitStates;
444 
445  if (I->isInlineAsm())
446  continue;
447 
448  WaitStates += SIInstrInfo::getNumWaitStates(*I);
449 
450  if (IsExpired(*I, WaitStates))
452  }
453 
454  int MinWaitStates = std::numeric_limits<int>::max();
455  for (MachineBasicBlock *Pred : MBB->predecessors()) {
456  if (!Visited.insert(Pred).second)
457  continue;
458 
459  int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(),
460  WaitStates, IsExpired, Visited);
461 
462  MinWaitStates = std::min(MinWaitStates, W);
463  }
464 
465  return MinWaitStates;
466 }
467 
469  const MachineInstr *MI, IsExpiredFn IsExpired) {
471  return getWaitStatesSince(IsHazard, MI->getParent(),
472  std::next(MI->getReverseIterator()),
473  0, IsExpired, Visited);
474 }
475 
476 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
477  if (IsHazardRecognizerMode) {
478  auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
479  return WaitStates >= Limit;
480  };
481  return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
482  }
483 
484  int WaitStates = 0;
485  for (MachineInstr *MI : EmittedInstrs) {
486  if (MI) {
487  if (IsHazard(*MI))
488  return WaitStates;
489 
490  if (MI->isInlineAsm())
491  continue;
492  }
493  ++WaitStates;
494 
495  if (WaitStates >= Limit)
496  break;
497  }
499 }
500 
501 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
502  IsHazardFn IsHazardDef,
503  int Limit) {
504  const SIRegisterInfo *TRI = ST.getRegisterInfo();
505 
506  auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
507  return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
508  };
509 
510  return getWaitStatesSince(IsHazardFn, Limit);
511 }
512 
513 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
514  int Limit) {
515  auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
516  return isSSetReg(MI.getOpcode()) && IsHazard(MI);
517  };
518 
519  return getWaitStatesSince(IsHazardFn, Limit);
520 }
521 
522 //===----------------------------------------------------------------------===//
523 // No-op Hazard Detection
524 //===----------------------------------------------------------------------===//
525 
526 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
527  MCRegister Reg) {
528  for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
529  BV.set(*RUI);
530 }
531 
532 static void addRegsToSet(const SIRegisterInfo &TRI,
534  BitVector &Set) {
535  for (const MachineOperand &Op : Ops) {
536  if (Op.isReg())
537  addRegUnits(TRI, Set, Op.getReg().asMCReg());
538  }
539 }
540 
541 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
542  // XXX: Do we need to worry about implicit operands
543  addRegsToSet(TRI, MI.defs(), ClauseDefs);
544  addRegsToSet(TRI, MI.uses(), ClauseUses);
545 }
546 
548  return !SIInstrInfo::isSMRD(*MI);
549 }
550 
553 }
554 
555 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
556  // SMEM soft clause are only present on VI+, and only matter if xnack is
557  // enabled.
558  if (!ST.isXNACKEnabled())
559  return 0;
560 
561  bool IsSMRD = TII.isSMRD(*MEM);
562 
563  resetClause();
564 
565  // A soft-clause is any group of consecutive SMEM instructions. The
566  // instructions in this group may return out of order and/or may be
567  // replayed (i.e. the same instruction issued more than once).
568  //
569  // In order to handle these situations correctly we need to make sure that
570  // when a clause has more than one instruction, no instruction in the clause
571  // writes to a register that is read by another instruction in the clause
572  // (including itself). If we encounter this situation, we need to break the
573  // clause by inserting a non SMEM instruction.
574 
575  for (MachineInstr *MI : EmittedInstrs) {
576  // When we hit a non-SMEM instruction then we have passed the start of the
577  // clause and we can stop.
578  if (!MI)
579  break;
580 
582  break;
583 
584  addClauseInst(*MI);
585  }
586 
587  if (ClauseDefs.none())
588  return 0;
589 
590  // We need to make sure not to put loads and stores in the same clause if they
591  // use the same address. For now, just start a new clause whenever we see a
592  // store.
593  if (MEM->mayStore())
594  return 1;
595 
596  addClauseInst(*MEM);
597 
598  // If the set of defs and uses intersect then we cannot add this instruction
599  // to the clause, so we have a hazard.
600  return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
601 }
602 
603 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
604  int WaitStatesNeeded = 0;
605 
606  WaitStatesNeeded = checkSoftClauseHazards(SMRD);
607 
608  // This SMRD hazard only affects SI.
609  if (!ST.hasSMRDReadVALUDefHazard())
610  return WaitStatesNeeded;
611 
612  // A read of an SGPR by SMRD instruction requires 4 wait states when the
613  // SGPR was written by a VALU instruction.
614  int SmrdSgprWaitStates = 4;
615  auto IsHazardDefFn = [this](const MachineInstr &MI) {
616  return TII.isVALU(MI);
617  };
618  auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
619  return TII.isSALU(MI);
620  };
621 
622  bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
623 
624  for (const MachineOperand &Use : SMRD->uses()) {
625  if (!Use.isReg())
626  continue;
627  int WaitStatesNeededForUse =
628  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
629  SmrdSgprWaitStates);
630  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
631 
632  // This fixes what appears to be undocumented hardware behavior in SI where
633  // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
634  // needs some number of nops in between. We don't know how many we need, but
635  // let's use 4. This wasn't discovered before probably because the only
636  // case when this happens is when we expand a 64-bit pointer into a full
637  // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
638  // probably never encountered in the closed-source land.
639  if (IsBufferSMRD) {
640  int WaitStatesNeededForUse =
641  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
642  IsBufferHazardDefFn,
643  SmrdSgprWaitStates);
644  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
645  }
646  }
647 
648  return WaitStatesNeeded;
649 }
650 
651 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
653  return 0;
654 
655  int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
656 
657  // A read of an SGPR by a VMEM instruction requires 5 wait states when the
658  // SGPR was written by a VALU Instruction.
659  const int VmemSgprWaitStates = 5;
660  auto IsHazardDefFn = [this](const MachineInstr &MI) {
661  return TII.isVALU(MI);
662  };
663  for (const MachineOperand &Use : VMEM->uses()) {
664  if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
665  continue;
666 
667  int WaitStatesNeededForUse =
668  VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
669  VmemSgprWaitStates);
670  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
671  }
672  return WaitStatesNeeded;
673 }
674 
675 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
676  const SIRegisterInfo *TRI = ST.getRegisterInfo();
677  const SIInstrInfo *TII = ST.getInstrInfo();
678 
679  // Check for DPP VGPR read after VALU VGPR write and EXEC write.
680  int DppVgprWaitStates = 2;
681  int DppExecWaitStates = 5;
682  int WaitStatesNeeded = 0;
683  auto IsHazardDefFn = [TII](const MachineInstr &MI) {
684  return TII->isVALU(MI);
685  };
686 
687  for (const MachineOperand &Use : DPP->uses()) {
688  if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
689  continue;
690  int WaitStatesNeededForUse =
691  DppVgprWaitStates - getWaitStatesSinceDef(
692  Use.getReg(),
693  [](const MachineInstr &) { return true; },
694  DppVgprWaitStates);
695  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
696  }
697 
698  WaitStatesNeeded = std::max(
699  WaitStatesNeeded,
700  DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
701  DppExecWaitStates));
702 
703  return WaitStatesNeeded;
704 }
705 
706 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
707  const SIInstrInfo *TII = ST.getInstrInfo();
708 
709  // v_div_fmas requires 4 wait states after a write to vcc from a VALU
710  // instruction.
711  const int DivFMasWaitStates = 4;
712  auto IsHazardDefFn = [TII](const MachineInstr &MI) {
713  return TII->isVALU(MI);
714  };
715  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
716  DivFMasWaitStates);
717 
718  return DivFMasWaitStates - WaitStatesNeeded;
719 }
720 
721 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
722  const SIInstrInfo *TII = ST.getInstrInfo();
723  unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
724 
725  const int GetRegWaitStates = 2;
726  auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
727  return GetRegHWReg == getHWReg(TII, MI);
728  };
729  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
730 
731  return GetRegWaitStates - WaitStatesNeeded;
732 }
733 
734 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
735  const SIInstrInfo *TII = ST.getInstrInfo();
736  unsigned HWReg = getHWReg(TII, *SetRegInstr);
737 
738  const int SetRegWaitStates = ST.getSetRegWaitStates();
739  auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
740  return HWReg == getHWReg(TII, MI);
741  };
742  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
743  return SetRegWaitStates - WaitStatesNeeded;
744 }
745 
746 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
747  if (!MI.mayStore())
748  return -1;
749 
750  const SIInstrInfo *TII = ST.getInstrInfo();
751  unsigned Opcode = MI.getOpcode();
752  const MCInstrDesc &Desc = MI.getDesc();
753 
754  int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
755  int VDataRCID = -1;
756  if (VDataIdx != -1)
757  VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
758 
759  if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
760  // There is no hazard if the instruction does not use vector regs
761  // (like wbinvl1)
762  if (VDataIdx == -1)
763  return -1;
764  // For MUBUF/MTBUF instructions this hazard only exists if the
765  // instruction is not using a register in the soffset field.
766  const MachineOperand *SOffset =
767  TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
768  // If we have no soffset operand, then assume this field has been
769  // hardcoded to zero.
770  if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
771  (!SOffset || !SOffset->isReg()))
772  return VDataIdx;
773  }
774 
775  // MIMG instructions create a hazard if they don't use a 256-bit T# and
776  // the store size is greater than 8 bytes and they have more than two bits
777  // of their dmask set.
778  // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
779  if (TII->isMIMG(MI)) {
780  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
781  assert(SRsrcIdx != -1 &&
782  AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
783  (void)SRsrcIdx;
784  }
785 
786  if (TII->isFLAT(MI)) {
787  int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
788  if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
789  return DataIdx;
790  }
791 
792  return -1;
793 }
794 
795 int
796 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
797  const MachineRegisterInfo &MRI) {
798  // Helper to check for the hazard where VMEM instructions that store more than
799  // 8 bytes can have there store data over written by the next instruction.
800  const SIRegisterInfo *TRI = ST.getRegisterInfo();
801 
802  const int VALUWaitStates = ST.hasGFX940Insts() ? 2 : 1;
803  int WaitStatesNeeded = 0;
804 
805  if (!TRI->isVectorRegister(MRI, Def.getReg()))
806  return WaitStatesNeeded;
807  Register Reg = Def.getReg();
808  auto IsHazardFn = [this, Reg, TRI](const MachineInstr &MI) {
809  int DataIdx = createsVALUHazard(MI);
810  return DataIdx >= 0 &&
811  TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg);
812  };
813  int WaitStatesNeededForDef =
814  VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
815  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
816 
817  return WaitStatesNeeded;
818 }
819 
820 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
821  int WaitStatesNeeded = 0;
822 
824  const int TransDefWaitstates = 1;
825 
826  auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
827  if (!SIInstrInfo::isTRANS(MI))
828  return false;
829  const SIRegisterInfo *TRI = ST.getRegisterInfo();
830  const SIInstrInfo *TII = ST.getInstrInfo();
831  Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
832 
833  for (const MachineOperand &Use : VALU->explicit_uses()) {
834  if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
835  return true;
836  }
837 
838  return false;
839  };
840 
841  int WaitStatesNeededForDef =
842  TransDefWaitstates -
843  getWaitStatesSince(IsTransDefFn, TransDefWaitstates);
844  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
845  }
846 
847  if (ST.hasDstSelForwardingHazard()) {
848  const int Shift16DefWaitstates = 1;
849 
850  auto IsShift16BitDefFn = [this, VALU](const MachineInstr &MI) {
851  if (!SIInstrInfo::isVALU(MI))
852  return false;
853  const SIInstrInfo *TII = ST.getInstrInfo();
854  if (SIInstrInfo::isSDWA(MI)) {
855  if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel))
856  if (DstSel->getImm() == AMDGPU::SDWA::DWORD)
857  return false;
858  } else {
859  if ((AMDGPU::getNamedOperandIdx(MI.getOpcode(),
860  AMDGPU::OpName::op_sel) == -1) ||
861  !(TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)
862  ->getImm() &
864  return false;
865  }
866  const SIRegisterInfo *TRI = ST.getRegisterInfo();
867  if (auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) {
868  Register Def = Dst->getReg();
869 
870  for (const MachineOperand &Use : VALU->explicit_uses()) {
871  if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
872  return true;
873  }
874  }
875 
876  return false;
877  };
878 
879  int WaitStatesNeededForDef =
880  Shift16DefWaitstates -
881  getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
882  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
883  }
884 
885  if (ST.hasVDecCoExecHazard()) {
886  const int VALUWriteSGPRVALUReadWaitstates = 2;
887  const int VALUWriteEXECRWLane = 4;
888  const int VALUWriteVGPRReadlaneRead = 1;
889 
890  const SIRegisterInfo *TRI = ST.getRegisterInfo();
891  const MachineRegisterInfo &MRI = MF.getRegInfo();
893  auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
894  if (!SIInstrInfo::isVALU(MI))
895  return false;
896  return MI.modifiesRegister(UseReg, TRI);
897  };
898 
899  for (const MachineOperand &Use : VALU->explicit_uses()) {
900  if (!Use.isReg())
901  continue;
902 
903  UseReg = Use.getReg();
904  if (TRI->isSGPRReg(MRI, UseReg)) {
905  int WaitStatesNeededForDef =
906  VALUWriteSGPRVALUReadWaitstates -
907  getWaitStatesSince(IsVALUDefSGPRFn,
908  VALUWriteSGPRVALUReadWaitstates);
909  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
910  }
911  }
912 
913  if (VALU->readsRegister(AMDGPU::VCC, TRI)) {
914  UseReg = AMDGPU::VCC;
915  int WaitStatesNeededForDef =
916  VALUWriteSGPRVALUReadWaitstates -
917  getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates);
918  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
919  }
920 
921  switch (VALU->getOpcode()) {
922  case AMDGPU::V_READLANE_B32:
923  case AMDGPU::V_READFIRSTLANE_B32: {
924  MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0);
925  UseReg = Src->getReg();
926  int WaitStatesNeededForDef =
927  VALUWriteVGPRReadlaneRead -
928  getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead);
929  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
930  }
932  case AMDGPU::V_WRITELANE_B32: {
933  UseReg = AMDGPU::EXEC;
934  int WaitStatesNeededForDef =
935  VALUWriteEXECRWLane -
936  getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane);
937  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
938  break;
939  }
940  default:
941  break;
942  }
943  }
944 
945  // This checks for the hazard where VMEM instructions that store more than
946  // 8 bytes can have there store data over written by the next instruction.
947  if (!ST.has12DWordStoreHazard())
948  return WaitStatesNeeded;
949 
950  const MachineRegisterInfo &MRI = MF.getRegInfo();
951 
952  for (const MachineOperand &Def : VALU->defs()) {
953  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
954  }
955 
956  return WaitStatesNeeded;
957 }
958 
959 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
960  // This checks for hazards associated with inline asm statements.
961  // Since inline asms can contain just about anything, we use this
962  // to call/leverage other check*Hazard routines. Note that
963  // this function doesn't attempt to address all possible inline asm
964  // hazards (good luck), but is a collection of what has been
965  // problematic thus far.
966 
967  // see checkVALUHazards()
968  if (!ST.has12DWordStoreHazard())
969  return 0;
970 
971  const MachineRegisterInfo &MRI = MF.getRegInfo();
972  int WaitStatesNeeded = 0;
973 
974  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
975  I != E; ++I) {
976  const MachineOperand &Op = IA->getOperand(I);
977  if (Op.isReg() && Op.isDef()) {
978  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
979  }
980  }
981 
982  return WaitStatesNeeded;
983 }
984 
985 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
986  const SIInstrInfo *TII = ST.getInstrInfo();
987  const SIRegisterInfo *TRI = ST.getRegisterInfo();
988  const MachineRegisterInfo &MRI = MF.getRegInfo();
989 
990  const MachineOperand *LaneSelectOp =
991  TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
992 
993  if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
994  return 0;
995 
996  Register LaneSelectReg = LaneSelectOp->getReg();
997  auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVALU(MI); };
998 
999  const int RWLaneWaitStates = 4;
1000  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
1001  RWLaneWaitStates);
1002  return RWLaneWaitStates - WaitStatesSince;
1003 }
1004 
1005 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
1006  if (!ST.hasRFEHazards())
1007  return 0;
1008 
1009  const SIInstrInfo *TII = ST.getInstrInfo();
1010 
1011  const int RFEWaitStates = 1;
1012 
1013  auto IsHazardFn = [TII](const MachineInstr &MI) {
1014  return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS;
1015  };
1016  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
1017  return RFEWaitStates - WaitStatesNeeded;
1018 }
1019 
1020 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
1021  const SIInstrInfo *TII = ST.getInstrInfo();
1022  const int ReadM0WaitStates = 1;
1023  auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
1024  return ReadM0WaitStates -
1025  getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates);
1026 }
1027 
1028 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
1029  fixVMEMtoScalarWriteHazards(MI);
1030  fixVcmpxPermlaneHazards(MI);
1031  fixSMEMtoVectorWriteHazards(MI);
1032  fixVcmpxExecWARHazard(MI);
1033  fixLdsBranchVmemWARHazard(MI);
1034 }
1035 
1036 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
1037  if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI))
1038  return false;
1039 
1040  const SIInstrInfo *TII = ST.getInstrInfo();
1041  auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isVOPC(MI); };
1042 
1043  auto IsExpiredFn = [](const MachineInstr &MI, int) {
1044  unsigned Opc = MI.getOpcode();
1045  return SIInstrInfo::isVALU(MI) && Opc != AMDGPU::V_NOP_e32 &&
1046  Opc != AMDGPU::V_NOP_e64 && Opc != AMDGPU::V_NOP_sdwa;
1047  };
1048 
1049  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1051  return false;
1052 
1053  // V_NOP will be discarded by SQ.
1054  // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
1055  // which is always a VGPR and available.
1056  auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
1057  Register Reg = Src0->getReg();
1058  bool IsUndef = Src0->isUndef();
1059  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1060  TII->get(AMDGPU::V_MOV_B32_e32))
1061  .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0))
1062  .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill);
1063 
1064  return true;
1065 }
1066 
1067 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
1068  if (!ST.hasVMEMtoScalarWriteHazard())
1069  return false;
1070 
1072  return false;
1073 
1074  if (MI->getNumDefs() == 0)
1075  return false;
1076 
1077  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1078 
1079  auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
1082  return false;
1083 
1084  for (const MachineOperand &Def : MI->defs()) {
1085  const MachineOperand *Op =
1086  I.findRegisterUseOperand(Def.getReg(), false, TRI);
1087  if (!Op)
1088  continue;
1089  return true;
1090  }
1091  return false;
1092  };
1093 
1094  auto IsExpiredFn = [](const MachineInstr &MI, int) {
1095  return SIInstrInfo::isVALU(MI) ||
1096  (MI.getOpcode() == AMDGPU::S_WAITCNT &&
1097  !MI.getOperand(0).getImm()) ||
1098  (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1099  MI.getOperand(0).getImm() == 0xffe3);
1100  };
1101 
1102  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1104  return false;
1105 
1106  const SIInstrInfo *TII = ST.getInstrInfo();
1107  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1108  TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1109  .addImm(0xffe3);
1110  return true;
1111 }
1112 
1113 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
1114  if (!ST.hasSMEMtoVectorWriteHazard())
1115  return false;
1116 
1117  if (!SIInstrInfo::isVALU(*MI))
1118  return false;
1119 
1120  unsigned SDSTName;
1121  switch (MI->getOpcode()) {
1122  case AMDGPU::V_READLANE_B32:
1123  case AMDGPU::V_READFIRSTLANE_B32:
1124  SDSTName = AMDGPU::OpName::vdst;
1125  break;
1126  default:
1127  SDSTName = AMDGPU::OpName::sdst;
1128  break;
1129  }
1130 
1131  const SIInstrInfo *TII = ST.getInstrInfo();
1132  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1133  const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
1134  const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
1135  if (!SDST) {
1136  for (const auto &MO : MI->implicit_operands()) {
1137  if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) {
1138  SDST = &MO;
1139  break;
1140  }
1141  }
1142  }
1143 
1144  if (!SDST)
1145  return false;
1146 
1147  const Register SDSTReg = SDST->getReg();
1148  auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
1149  return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI);
1150  };
1151 
1152  auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
1153  if (TII->isSALU(MI)) {
1154  switch (MI.getOpcode()) {
1155  case AMDGPU::S_SETVSKIP:
1156  case AMDGPU::S_VERSION:
1157  case AMDGPU::S_WAITCNT_VSCNT:
1158  case AMDGPU::S_WAITCNT_VMCNT:
1159  case AMDGPU::S_WAITCNT_EXPCNT:
1160  // These instructions cannot not mitigate the hazard.
1161  return false;
1162  case AMDGPU::S_WAITCNT_LGKMCNT:
1163  // Reducing lgkmcnt count to 0 always mitigates the hazard.
1164  return (MI.getOperand(1).getImm() == 0) &&
1165  (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1166  case AMDGPU::S_WAITCNT: {
1167  const int64_t Imm = MI.getOperand(0).getImm();
1168  AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm);
1169  return (Decoded.LgkmCnt == 0);
1170  }
1171  default:
1172  // SOPP instructions cannot mitigate the hazard.
1173  if (TII->isSOPP(MI))
1174  return false;
1175  // At this point the SALU can be assumed to mitigate the hazard
1176  // because either:
1177  // (a) it is independent of the at risk SMEM (breaking chain),
1178  // or
1179  // (b) it is dependent on the SMEM, in which case an appropriate
1180  // s_waitcnt lgkmcnt _must_ exist between it and the at risk
1181  // SMEM instruction.
1182  return true;
1183  }
1184  }
1185  return false;
1186  };
1187 
1188  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1190  return false;
1191 
1192  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1193  TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
1194  .addImm(0);
1195  return true;
1196 }
1197 
1198 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
1200  return false;
1201 
1202  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1203  if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
1204  return false;
1205 
1206  auto IsHazardFn = [TRI](const MachineInstr &I) {
1207  if (SIInstrInfo::isVALU(I))
1208  return false;
1209  return I.readsRegister(AMDGPU::EXEC, TRI);
1210  };
1211 
1212  const SIInstrInfo *TII = ST.getInstrInfo();
1213  auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
1214  if (SIInstrInfo::isVALU(MI)) {
1215  if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst))
1216  return true;
1217  for (auto MO : MI.implicit_operands())
1218  if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg())))
1219  return true;
1220  }
1221  if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1222  (MI.getOperand(0).getImm() & 0xfffe) == 0xfffe)
1223  return true;
1224  return false;
1225  };
1226 
1227  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1229  return false;
1230 
1231  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1232  TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1233  .addImm(0xfffe);
1234  return true;
1235 }
1236 
1238  const GCNSubtarget &ST) {
1239  if (!ST.hasLdsBranchVmemWARHazard())
1240  return false;
1241 
1242  // Check if the necessary condition for the hazard is met: both LDS and VMEM
1243  // instructions need to appear in the same function.
1244  bool HasLds = false;
1245  bool HasVmem = false;
1246  for (auto &MBB : MF) {
1247  for (auto &MI : MBB) {
1248  HasLds |= SIInstrInfo::isDS(MI);
1249  HasVmem |=
1251  if (HasLds && HasVmem)
1252  return true;
1253  }
1254  }
1255  return false;
1256 }
1257 
1258 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
1259  if (!RunLdsBranchVmemWARHazardFixup)
1260  return false;
1261 
1263 
1264  auto IsHazardInst = [](const MachineInstr &MI) {
1265  if (SIInstrInfo::isDS(MI))
1266  return 1;
1268  return 2;
1269  return 0;
1270  };
1271 
1272  auto InstType = IsHazardInst(*MI);
1273  if (!InstType)
1274  return false;
1275 
1276  auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
1277  return IsHazardInst(I) || (I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1278  I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1279  !I.getOperand(1).getImm());
1280  };
1281 
1282  auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
1283  if (!I.isBranch())
1284  return false;
1285 
1286  auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
1287  auto InstType2 = IsHazardInst(I);
1288  return InstType2 && InstType != InstType2;
1289  };
1290 
1291  auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
1292  auto InstType2 = IsHazardInst(I);
1293  if (InstType == InstType2)
1294  return true;
1295 
1296  return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1297  I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1298  !I.getOperand(1).getImm();
1299  };
1300 
1303  };
1304 
1305  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1307  return false;
1308 
1309  const SIInstrInfo *TII = ST.getInstrInfo();
1310  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1311  TII->get(AMDGPU::S_WAITCNT_VSCNT))
1312  .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1313  .addImm(0);
1314 
1315  return true;
1316 }
1317 
1318 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) {
1319  int NSAtoVMEMWaitStates = 1;
1320 
1321  if (!ST.hasNSAtoVMEMBug())
1322  return 0;
1323 
1325  return 0;
1326 
1327  const SIInstrInfo *TII = ST.getInstrInfo();
1328  const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
1329  if (!Offset || (Offset->getImm() & 6) == 0)
1330  return 0;
1331 
1332  auto IsHazardFn = [TII](const MachineInstr &I) {
1333  if (!SIInstrInfo::isMIMG(I))
1334  return false;
1335  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode());
1336  return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
1337  TII->getInstSizeInBytes(I) >= 16;
1338  };
1339 
1340  return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
1341 }
1342 
1343 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) {
1344  int FPAtomicToDenormModeWaitStates = 3;
1345 
1346  if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
1347  return 0;
1348 
1349  auto IsHazardFn = [](const MachineInstr &I) {
1351  return false;
1352  return SIInstrInfo::isFPAtomic(I);
1353  };
1354 
1355  auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
1356  if (WaitStates >= 3 || SIInstrInfo::isVALU(MI))
1357  return true;
1358 
1359  switch (MI.getOpcode()) {
1360  case AMDGPU::S_WAITCNT:
1361  case AMDGPU::S_WAITCNT_VSCNT:
1362  case AMDGPU::S_WAITCNT_VMCNT:
1363  case AMDGPU::S_WAITCNT_EXPCNT:
1364  case AMDGPU::S_WAITCNT_LGKMCNT:
1365  case AMDGPU::S_WAIT_IDLE:
1366  return true;
1367  default:
1368  break;
1369  }
1370 
1371  return false;
1372  };
1373 
1374  return FPAtomicToDenormModeWaitStates -
1375  ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn);
1376 }
1377 
1378 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) {
1380 
1381  return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
1382 }
1383 
1384 int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) {
1385  // Early exit if no padding is requested.
1386  if (MFMAPaddingRatio == 0)
1387  return 0;
1388 
1389  auto IsMFMAFn = [](const MachineInstr &MI) {
1390  return SIInstrInfo::isMAI(MI) &&
1391  MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1392  MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1393  };
1394 
1396  if (!IsMFMAFn(*MI) || MFI->getOccupancy() < 2)
1397  return 0;
1398 
1399  int NeighborMFMALatency = 0;
1400  auto IsNeighboringMFMA = [&IsMFMAFn, &NeighborMFMALatency,
1401  this](const MachineInstr &MI) {
1402  if (!IsMFMAFn(MI))
1403  return false;
1404 
1405  NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
1406  return true;
1407  };
1408 
1409  const int MaxMFMAPipelineWaitStates = 16;
1410  int WaitStatesSinceNeighborMFMA =
1411  getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates);
1412 
1413  int NeighborMFMAPaddingNeeded =
1414  (NeighborMFMALatency * MFMAPaddingRatio / 100) -
1415  WaitStatesSinceNeighborMFMA;
1416 
1417  return std::max(0, NeighborMFMAPaddingNeeded);
1418 }
1419 
1420 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) {
1421  int WaitStatesNeeded = 0;
1422  unsigned Opc = MI->getOpcode();
1423 
1424  auto IsVALUFn = [](const MachineInstr &MI) {
1425  return SIInstrInfo::isVALU(MI);
1426  };
1427 
1428  if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
1429  const int LegacyVALUWritesVGPRWaitStates = 2;
1430  const int VALUWritesExecWaitStates = 4;
1431  const int MaxWaitStates = 4;
1432 
1433  int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1434  getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
1435  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1436 
1437  if (WaitStatesNeeded < MaxWaitStates) {
1438  for (const MachineOperand &Use : MI->explicit_uses()) {
1439  const int MaxWaitStates = 2;
1440 
1441  if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1442  continue;
1443 
1444  int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
1445  getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates);
1446  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1447 
1448  if (WaitStatesNeeded == MaxWaitStates)
1449  break;
1450  }
1451  }
1452  }
1453 
1454  auto IsMFMAFn = [](const MachineInstr &MI) {
1455  return SIInstrInfo::isMAI(MI) &&
1456  MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1457  MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1458  };
1459 
1460  for (const MachineOperand &Op : MI->explicit_operands()) {
1461  if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg()))
1462  continue;
1463 
1464  if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1465  continue;
1466 
1467  const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
1468  const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
1469  const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
1470  const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
1471  const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
1472  const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
1473  const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
1474  const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
1475  const int MaxWaitStates = 18;
1476  Register Reg = Op.getReg();
1477  unsigned HazardDefLatency = 0;
1478 
1479  auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &HazardDefLatency,
1480  this](const MachineInstr &MI) {
1481  if (!IsMFMAFn(MI))
1482  return false;
1483  Register DstReg = MI.getOperand(0).getReg();
1484  if (DstReg == Reg)
1485  return false;
1486  HazardDefLatency =
1487  std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
1488  return TRI.regsOverlap(DstReg, Reg);
1489  };
1490 
1491  int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn,
1492  MaxWaitStates);
1493  int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
1494  int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1495  int OpNo = MI->getOperandNo(&Op);
1496  if (OpNo == SrcCIdx) {
1497  NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
1498  } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
1499  switch (HazardDefLatency) {
1500  case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
1501  break;
1502  case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
1503  break;
1504  case 16: LLVM_FALLTHROUGH;
1505  default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
1506  break;
1507  }
1508  } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1509  switch (HazardDefLatency) {
1510  case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
1511  break;
1512  case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
1513  break;
1514  case 16: LLVM_FALLTHROUGH;
1515  default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
1516  break;
1517  }
1518  }
1519 
1520  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1521  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1522 
1523  if (WaitStatesNeeded == MaxWaitStates)
1524  return WaitStatesNeeded; // Early exit.
1525 
1526  auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
1527  if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1528  return false;
1529  Register DstReg = MI.getOperand(0).getReg();
1530  return TRI.regsOverlap(Reg, DstReg);
1531  };
1532 
1533  const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
1534  const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
1535  const int AccVGPRWriteAccVgprReadWaitStates = 3;
1536  NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
1537  if (OpNo == SrcCIdx)
1538  NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
1539  else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
1540  NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
1541 
1542  WaitStatesNeededForUse = NeedWaitStates -
1543  getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates);
1544  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1545 
1546  if (WaitStatesNeeded == MaxWaitStates)
1547  return WaitStatesNeeded; // Early exit.
1548  }
1549 
1550  if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1551  const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
1552  const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
1553  const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
1554  const int MaxWaitStates = 13;
1555  Register DstReg = MI->getOperand(0).getReg();
1556  unsigned HazardDefLatency = 0;
1557 
1558  auto IsSrcCMFMAFn = [DstReg, &IsMFMAFn, &HazardDefLatency,
1559  this](const MachineInstr &MI) {
1560  if (!IsMFMAFn(MI))
1561  return false;
1562  Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
1563  HazardDefLatency =
1564  std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
1565  return TRI.regsOverlap(Reg, DstReg);
1566  };
1567 
1568  int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
1569  int NeedWaitStates;
1570  switch (HazardDefLatency) {
1571  case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
1572  break;
1573  case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
1574  break;
1575  case 16: LLVM_FALLTHROUGH;
1576  default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
1577  break;
1578  }
1579 
1580  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
1581  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1582  }
1583 
1584  // Pad neighboring MFMA with noops for better inter-wave performance.
1585  WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI));
1586 
1587  return WaitStatesNeeded;
1588 }
1589 
1590 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
1591  int WaitStatesNeeded = 0;
1592  unsigned Opc = MI->getOpcode();
1593 
1594  auto IsMFMAFn = [](const MachineInstr &MI) {
1595  return SIInstrInfo::isMAI(MI) &&
1596  MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1597  MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1598  };
1599 
1600  auto IsLegacyVALUFn = [&IsMFMAFn](const MachineInstr &MI) {
1601  return SIInstrInfo::isVALU(MI) && !IsMFMAFn(MI);
1602  };
1603 
1604  auto IsLegacyVALUNotDotFn = [&IsMFMAFn](const MachineInstr &MI) {
1605  return SIInstrInfo::isVALU(MI) && !IsMFMAFn(MI) && !SIInstrInfo::isDOT(MI);
1606  };
1607 
1608  if (!IsMFMAFn(*MI))
1609  return WaitStatesNeeded;
1610 
1611  const int VALUWritesExecWaitStates = 4;
1612  int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1613  getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
1614  VALUWritesExecWaitStates);
1615  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1616 
1617  int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1618 
1619  // Loop for both DGEMM and S/HGEMM 2nd instruction.
1620  for (const MachineOperand &Use : MI->explicit_uses()) {
1621  const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
1622  const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
1623  const int GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates = 3;
1624  const int GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates = 5;
1625  const int GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates = 4;
1626  const int GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates = 9;
1627  const int GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates = 8;
1628  const int GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates = 17;
1629  const int GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates = 16;
1630  const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
1631  const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
1632  const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
1633  const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
1634  const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
1635  const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
1636  const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
1637  const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
1638  const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
1639  const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
1640  const int GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates = 4;
1641  const int GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates = 6;
1642  const int GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates = 10;
1643  const int GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates = 18;
1644  const int GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates = 5;
1645  const int GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates = 7;
1646  const int GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates = 11;
1647  const int GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates = 19;
1648  const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
1649  const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
1650  const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
1651  const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
1652  const int MaxWaitStates = 19;
1653 
1654  if (!Use.isReg())
1655  continue;
1656  Register Reg = Use.getReg();
1657  bool FullReg;
1658  const MachineInstr *MI1;
1659 
1660  auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &FullReg, &MI1,
1661  this](const MachineInstr &MI) {
1662  if (!IsMFMAFn(MI))
1663  return false;
1664  Register DstReg = MI.getOperand(0).getReg();
1665  FullReg = (DstReg == Reg);
1666  MI1 = &MI;
1667  return TRI.regsOverlap(DstReg, Reg);
1668  };
1669 
1670  WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
1671  getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
1672  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1673 
1674  int NumWaitStates =
1675  getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates);
1676  if (NumWaitStates == std::numeric_limits<int>::max())
1677  continue;
1678 
1679  int OpNo = MI->getOperandNo(&Use);
1680  unsigned Opc1 = MI1->getOpcode();
1681  int NeedWaitStates = 0;
1682  if (OpNo == SrcCIdx) {
1683  if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) {
1684  NeedWaitStates = 0;
1685  } else if (FullReg) {
1686  if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
1687  Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
1688  (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
1689  Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
1690  NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
1691  else if (ST.hasGFX940Insts() &&
1692  TSchedModel.computeInstrLatency(MI1) == 2)
1693  NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
1694  } else {
1695  switch (Opc1) {
1696  case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
1697  case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
1698  case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
1699  case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
1700  if (!isXDL(ST, *MI))
1701  NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
1702  break;
1703  case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
1704  case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
1705  if (!isXDL(ST, *MI))
1706  NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
1707  break;
1708  default:
1709  if (ST.hasGFX940Insts() && isXDL(ST, *MI) && !isXDL(ST, *MI1))
1710  break;
1711  switch (TSchedModel.computeInstrLatency(MI1)) {
1712  case 2:
1713  NeedWaitStates = ST.hasGFX940Insts()
1714  ? isXDL(ST, *MI1)
1715  ? GFX940_XDL2PassWritesVGPROverlappedSMFMASrcCWaitStates
1716  : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates
1717  : isDGEMM(Opc)
1718  ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
1719  : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
1720  break;
1721  case 4:
1722  assert(ST.hasGFX940Insts());
1723  NeedWaitStates = isXDL(ST, *MI1)
1724  ? GFX940_XDL4PassWritesVGPROverlappedSMFMASrcCWaitStates
1725  : GFX940_SMFMA4PassWritesVGPROverlappedSMFMASrcCWaitStates;
1726  break;
1727  case 8:
1728  NeedWaitStates = ST.hasGFX940Insts()
1729  ? isXDL(ST, *MI1)
1730  ? GFX940_XDL8PassWritesVGPROverlappedSMFMASrcCWaitStates
1731  : GFX940_SMFMA8PassWritesVGPROverlappedSMFMASrcCWaitStates
1732  : isDGEMM(Opc)
1733  ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
1734  : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
1735  break;
1736  case 16: LLVM_FALLTHROUGH;
1737  default:
1738  NeedWaitStates = ST.hasGFX940Insts()
1739  ? isXDL(ST, *MI1)
1740  ? GFX940_XDL16PassWritesVGPROverlappedSMFMASrcCWaitStates
1741  : GFX940_SMFMA16PassWritesVGPROverlappedSMFMASrcCWaitStates
1742  : isDGEMM(Opc)
1743  ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
1744  : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
1745  }
1746  }
1747  }
1748  } else {
1749  switch (Opc1) {
1750  case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
1751  case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
1752  case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
1753  case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
1754  NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
1755  break;
1756  case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
1757  case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
1758  NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
1759  break;
1760  default:
1761  switch (TSchedModel.computeInstrLatency(MI1)) {
1762  case 2:
1763  NeedWaitStates = ST.hasGFX940Insts()
1764  ? isXDL(ST, *MI1)
1765  ? GFX940_XDL2PassWritesVGPROverlappedSrcABWaitStates
1766  : GFX940_SMFMA2PassWritesVGPROverlappedSrcABWaitStates
1767  : SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
1768  break;
1769  case 4:
1770  assert(ST.hasGFX940Insts());
1771  NeedWaitStates = isXDL(ST, *MI1)
1772  ? GFX940_XDL4PassWritesVGPROverlappedSrcABWaitStates
1773  : GFX940_SMFMA4PassWritesVGPROverlappedSrcABWaitStates;
1774  break;
1775  case 8:
1776  NeedWaitStates = ST.hasGFX940Insts()
1777  ? isXDL(ST, *MI1)
1778  ? GFX940_XDL8PassWritesVGPROverlappedSrcABWaitStates
1779  : GFX940_SMFMA8PassWritesVGPROverlappedSrcABWaitStates
1780  : SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
1781  break;
1782  case 16: LLVM_FALLTHROUGH;
1783  default:
1784  NeedWaitStates = ST.hasGFX940Insts()
1785  ? isXDL(ST, *MI1)
1786  ? GFX940_XDL16PassWritesVGPROverlappedSrcABWaitStates
1787  : GFX940_SMFMA16PassWritesVGPROverlappedSrcABWaitStates
1788  : SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
1789  }
1790  }
1791  }
1792  if (WaitStatesNeeded >= NeedWaitStates)
1793  continue;
1794 
1795  WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
1796  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1797 
1798  if (WaitStatesNeeded == MaxWaitStates)
1799  break;
1800  }
1801 
1802  return WaitStatesNeeded;
1803 }
1804 
1805 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) {
1806  // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
1807  if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
1808  return 0;
1809 
1810  int WaitStatesNeeded = 0;
1811 
1812  auto IsAccVgprReadFn = [](const MachineInstr &MI) {
1813  return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
1814  };
1815 
1816  for (const MachineOperand &Op : MI->explicit_uses()) {
1817  if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg()))
1818  continue;
1819 
1820  Register Reg = Op.getReg();
1821 
1822  const int AccVgprReadLdStWaitStates = 2;
1823  const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
1824  const int MaxWaitStates = 2;
1825 
1826  int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
1827  getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates);
1828  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1829 
1830  if (WaitStatesNeeded == MaxWaitStates)
1831  return WaitStatesNeeded; // Early exit.
1832 
1833  auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
1834  if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
1835  MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1836  return false;
1837  auto IsVALUFn = [](const MachineInstr &MI) {
1839  };
1840  return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
1842  };
1843 
1844  WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
1845  getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
1846  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1847  }
1848 
1849  return WaitStatesNeeded;
1850 }
1851 
1852 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
1853  if (!ST.hasGFX90AInsts())
1854  return 0;
1855 
1856  auto IsMFMAFn = [](const MachineInstr &MI) -> bool {
1857  return SIInstrInfo::isMAI(MI) &&
1858  MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1859  MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1860  };
1861 
1862  auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
1863  return isDGEMM(MI.getOpcode());
1864  };
1865 
1866  // This is checked in checkMAIHazards90A()
1867  if (IsMFMAFn(*MI))
1868  return 0;
1869 
1870  int WaitStatesNeeded = 0;
1871 
1872  bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) ||
1873  SIInstrInfo::isFLAT(*MI) ||
1874  SIInstrInfo::isDS(*MI) ||
1876  bool IsVALU = SIInstrInfo::isVALU(*MI);
1877 
1878  const MachineInstr *MFMA = nullptr;
1879  unsigned Reg;
1880  auto IsMFMAWriteFn = [&Reg, &IsMFMAFn, &MFMA, this](const MachineInstr &MI) {
1881  if (!IsMFMAFn(MI) || !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
1882  return false;
1883  MFMA = &MI;
1884  return true;
1885  };
1886 
1887  const MachineInstr *DOT = nullptr;
1888  auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
1889  if (!SIInstrInfo::isDOT(MI) ||
1890  !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
1891  return false;
1892  DOT = &MI;
1893  return true;
1894  };
1895 
1896  int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
1897  AMDGPU::OpName::src2);
1898 
1899  if (IsMemOrExport || IsVALU) {
1900  const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
1901  const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
1902  const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
1903  const int GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates = 4;
1904  const int GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates = 6;
1905  const int GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates = 10;
1906  const int GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates = 18;
1907  const int GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates = 5;
1908  const int GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates = 7;
1909  const int GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates = 11;
1910  const int GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates = 19;
1911  const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
1912  const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
1913  const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
1914  const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
1915  const int DotWriteSameDotReadSrcAB = 3;
1916  const int DotWriteDifferentVALURead = 3;
1917  const int MaxWaitStates = 19;
1918 
1919  for (const MachineOperand &Use : MI->explicit_uses()) {
1920  if (!Use.isReg())
1921  continue;
1922  Reg = Use.getReg();
1923 
1924  DOT = nullptr;
1925  int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
1926  MaxWaitStates);
1927  if (DOT) {
1928  int NeedWaitStates = 0;
1929  if (DOT->getOpcode() == MI->getOpcode()) {
1930  if (&Use - &MI->getOperand(0) != SrcCIdx)
1931  NeedWaitStates = DotWriteSameDotReadSrcAB;
1932  } else {
1933  NeedWaitStates = DotWriteDifferentVALURead;
1934  }
1935 
1936  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1937  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1938  }
1939 
1940  MFMA = nullptr;
1941  WaitStatesSinceDef =
1942  getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
1943  if (!MFMA)
1944  continue;
1945 
1946  unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
1947  int NeedWaitStates = MaxWaitStates;
1948  switch (HazardDefLatency) {
1949  case 2:
1950  NeedWaitStates =
1951  ST.hasGFX940Insts()
1952  ? isXDL(ST, *MFMA)
1953  ? GFX940_XDL2PassWriteVgprVALUMemExpReadWaitStates
1954  : GFX940_SMFMA2PassWriteVgprVALUMemExpReadWaitStates
1955  : SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
1956  break;
1957  case 4:
1958  assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts());
1959  NeedWaitStates =
1960  isDGEMM(MFMA->getOpcode())
1961  ? IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
1962  : DMFMA4x4WriteVgprVALUReadWaitStates
1963  : isXDL(ST, *MFMA)
1964  ? GFX940_XDL4PassWriteVgprVALUMemExpReadWaitStates
1965  : GFX940_SMFMA4PassWriteVgprVALUMemExpReadWaitStates;
1966  break;
1967  case 8:
1968  NeedWaitStates =
1969  ST.hasGFX940Insts()
1970  ? isXDL(ST, *MFMA)
1971  ? GFX940_XDL8PassWriteVgprVALUMemExpReadWaitStates
1972  : GFX940_SMFMA8PassWriteVgprVALUMemExpReadWaitStates
1973  : SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
1974  break;
1975  case 16: LLVM_FALLTHROUGH;
1976  default:
1977  NeedWaitStates =
1978  isDGEMM(MFMA->getOpcode())
1979  ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates
1980  : DMFMA16x16WriteVgprVALUReadWaitStates
1981  : ST.hasGFX940Insts()
1982  ? isXDL(ST, *MFMA)
1983  ? GFX940_XDL16PassWriteVgprVALUMemExpReadWaitStates
1984  : GFX940_SMFMA16PassWriteVgprVALUMemExpReadWaitStates
1985  : SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
1986  break;
1987  }
1988 
1989  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1990  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1991 
1992  if (WaitStatesNeeded == MaxWaitStates)
1993  break;
1994  }
1995  }
1996 
1997  unsigned Opc = MI->getOpcode();
1998  const int DMFMAToFMA64WaitStates = 2;
1999  if ((Opc == AMDGPU::V_FMA_F64_e64 ||
2000  Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
2001  Opc == AMDGPU::V_FMAC_F64_dpp) &&
2002  WaitStatesNeeded < DMFMAToFMA64WaitStates) {
2003  int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
2004  getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
2005  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2006  }
2007 
2008  if (!IsVALU && !IsMemOrExport)
2009  return WaitStatesNeeded;
2010 
2011  for (const MachineOperand &Def : MI->defs()) {
2012  const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
2013  const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
2014  const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
2015  const int GFX940_SMFMA2PassWriteVgprVALUWawWaitStates = 4;
2016  const int GFX940_SMFMA4PassWriteVgprVALUWawWaitStates = 6;
2017  const int GFX940_SMFMA8PassWriteVgprVALUWawWaitStates = 10;
2018  const int GFX940_SMFMA16PassWriteVgprVALUWawWaitStates = 18;
2019  const int GFX940_XDL2PassWriteVgprVALUWawWaitStates = 5;
2020  const int GFX940_XDL4PassWriteVgprVALUWawWaitStates = 7;
2021  const int GFX940_XDL8PassWriteVgprVALUWawWaitStates = 11;
2022  const int GFX940_XDL16PassWriteVgprVALUWawWaitStates = 19;
2023  const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
2024  const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
2025  const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
2026  const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
2027  const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
2028  const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
2029  const int DotWriteDifferentVALUWrite = 3;
2030  const int MaxWaitStates = 19;
2031  const int MaxWarWaitStates = 15;
2032 
2033  Reg = Def.getReg();
2034 
2035  DOT = nullptr;
2036  int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
2037  MaxWaitStates);
2038  if (DOT && DOT->getOpcode() != MI->getOpcode())
2039  WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
2040  WaitStatesSinceDef);
2041 
2042  MFMA = nullptr;
2043  WaitStatesSinceDef =
2044  getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
2045  if (MFMA) {
2046  int NeedWaitStates = MaxWaitStates;
2047  switch (TSchedModel.computeInstrLatency(MFMA)) {
2048  case 2:
2049  NeedWaitStates = ST.hasGFX940Insts()
2050  ? isXDL(ST, *MFMA)
2051  ? GFX940_XDL2PassWriteVgprVALUWawWaitStates
2052  : GFX940_SMFMA2PassWriteVgprVALUWawWaitStates
2053  : SMFMA4x4WriteVgprVALUWawWaitStates;
2054  break;
2055  case 4:
2056  assert(isDGEMM(MFMA->getOpcode()) || ST.hasGFX940Insts());
2057  NeedWaitStates = isDGEMM(MFMA->getOpcode())
2058  ? DMFMA4x4WriteVgprVALUWriteWaitStates
2059  : isXDL(ST, *MFMA)
2060  ? GFX940_XDL4PassWriteVgprVALUWawWaitStates
2061  : GFX940_SMFMA4PassWriteVgprVALUWawWaitStates;
2062  break;
2063  case 8:
2064  NeedWaitStates = ST.hasGFX940Insts()
2065  ? isXDL(ST, *MFMA)
2066  ? GFX940_XDL8PassWriteVgprVALUWawWaitStates
2067  : GFX940_SMFMA8PassWriteVgprVALUWawWaitStates
2068  : SMFMA16x16WriteVgprVALUWawWaitStates;
2069  break;
2070  case 16: LLVM_FALLTHROUGH;
2071  default:
2072  NeedWaitStates = isDGEMM(MFMA->getOpcode())
2073  ? DMFMA16x16WriteVgprVALUWriteWaitStates
2074  : ST.hasGFX940Insts()
2075  ? isXDL(ST, *MFMA)
2076  ? GFX940_XDL16PassWriteVgprVALUWawWaitStates
2077  : GFX940_SMFMA16PassWriteVgprVALUWawWaitStates
2078  : SMFMA32x32WriteVgprVALUWawWaitStates;
2079  break;
2080  }
2081 
2082  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
2083  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2084 
2085  if (WaitStatesNeeded == MaxWaitStates)
2086  break;
2087  }
2088 
2089  auto IsSMFMAReadAsCFn = [&Reg, &IsMFMAFn, &MFMA,
2090  this](const MachineInstr &MI) {
2091  if (!IsMFMAFn(MI) || isDGEMM(MI.getOpcode()) ||
2092  !MI.readsRegister(Reg, &TRI))
2093  return false;
2094 
2095  if (ST.hasGFX940Insts() && !isXDL(ST, MI))
2096  return false;
2097 
2098  const MachineOperand *SrcC =
2099  TII.getNamedOperand(MI, AMDGPU::OpName::src2);
2100  assert(SrcC);
2101  if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
2102  return false;
2103 
2104  MFMA = &MI;
2105  return true;
2106  };
2107 
2108  MFMA = nullptr;
2109  int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn,
2110  MaxWarWaitStates);
2111  if (!MFMA)
2112  continue;
2113 
2114  unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
2115  int NeedWaitStates = MaxWaitStates;
2116  switch (HazardDefLatency) {
2117  case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
2118  break;
2119  case 4: assert(ST.hasGFX940Insts());
2120  NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
2121  break;
2122  case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
2123  break;
2124  case 16: LLVM_FALLTHROUGH;
2125  default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
2126  break;
2127  }
2128 
2129  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
2130  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
2131  }
2132 
2133  return WaitStatesNeeded;
2134 }
2135 
2137  if (!SU->isInstr())
2138  return false;
2139 
2140  const MachineInstr *MAI = nullptr;
2141  auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
2142  MAI = nullptr;
2143  if (SIInstrInfo::isMAI(MI) &&
2144  MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
2145  MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64)
2146  MAI = &MI;
2147  return MAI != nullptr;
2148  };
2149 
2150  MachineInstr *MI = SU->getInstr();
2151  if (IsMFMAFn(*MI)) {
2152  int W = getWaitStatesSince(IsMFMAFn, 16);
2153  if (MAI)
2154  return W < (int)TSchedModel.computeInstrLatency(MAI);
2155  }
2156 
2157  return false;
2158 }
i
i
Definition: README.txt:29
llvm::GCNSubtarget::hasReadM0LdsDmaHazard
bool hasReadM0LdsDmaHazard() const
Definition: GCNSubtarget.h:952
llvm::GCNSubtarget::hasVDecCoExecHazard
bool hasVDecCoExecHazard() const
Definition: GCNSubtarget.h:1000
llvm::TargetSchedModel::getWriteProcResBegin
ProcResIter getWriteProcResBegin(const MCSchedClassDesc *SC) const
Definition: TargetSchedule.h:133
ScheduleDAG.h
llvm::MachineInstr::uses
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:668
llvm::ScheduleHazardRecognizer::getMaxLookAhead
unsigned getMaxLookAhead() const
Definition: ScheduleHazardRecognizer.h:43
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:104
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::AMDGPU::getMAIIsDGEMM
bool getMAIIsDGEMM(unsigned Opc)
Returns true if MAI operation is a double precision GEMM.
Definition: AMDGPUBaseInfo.cpp:370
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::MachineRegisterInfo::isPhysRegUsed
bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
Definition: MachineRegisterInfo.cpp:581
llvm::GCNHazardRecognizer::getHazardType
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
Definition: GCNHazardRecognizer.cpp:181
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::GCNSubtarget::hasDstSelForwardingHazard
bool hasDstSelForwardingHazard() const
Definition: GCNSubtarget.h:994
llvm::AMDGPU::getIsaVersion
IsaVersion getIsaVersion(StringRef GPU)
Definition: TargetParser.cpp:193
SIMachineFunctionInfo.h
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
llvm::BitVector::set
BitVector & set()
Definition: BitVector.h:344
llvm::BitVector::none
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:181
llvm::ScheduleHazardRecognizer::MaxLookAhead
unsigned MaxLookAhead
MaxLookAhead - Indicate the number of cycles in the scoreboard state.
Definition: ScheduleHazardRecognizer.h:31
llvm::SIInstrInfo::isBufferSMRD
bool isBufferSMRD(const MachineInstr &MI) const
Definition: SIInstrInfo.cpp:7724
llvm::SIInstrInfo::getNumWaitStates
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
Definition: SIInstrInfo.cpp:1765
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:52
llvm::RISCVFenceField::W
@ W
Definition: RISCVBaseInfo.h:241
llvm::GCNSubtarget::hasVMEMReadSGPRVALUDefHazard
bool hasVMEMReadSGPRVALUDefHazard() const
A read of an SGPR by a VMEM instruction requires 5 wait states when the SGPR was written by a VALU In...
Definition: GCNSubtarget.h:434
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::GCNHazardRecognizer::GCNHazardRecognizer
GCNHazardRecognizer(const MachineFunction &MF)
Definition: GCNHazardRecognizer.cpp:53
llvm::SIInstrInfo::isEXP
static bool isEXP(const MachineInstr &MI)
Definition: SIInstrInfo.h:555
isRWLane
static bool isRWLane(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:98
llvm::GCNSubtarget::getSetRegWaitStates
unsigned getSetRegWaitStates() const
Number of hazard wait states for s_setreg_b32/s_setreg_imm32_b32.
Definition: GCNSubtarget.h:443
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:139
llvm::AMDGPU::Hwreg::ID_MASK_
@ ID_MASK_
Definition: SIDefines.h:406
llvm::ScheduleHazardRecognizer::Hazard
@ Hazard
Definition: ScheduleHazardRecognizer.h:39
isXDL
static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:122
addRegUnits
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg)
Definition: GCNHazardRecognizer.cpp:526
isSGetReg
static bool isSGetReg(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:83
llvm::GCNSubtarget
Definition: GCNSubtarget.h:31
TargetParser.h
llvm::AMDGPU::getNamedOperandIdx
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, DenseMapInfo< ValueT >, detail::DenseSetPair< ValueT > >, DenseMapInfo< ValueT > >::insert
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
llvm::SISrcMods::DST_OP_SEL
@ DST_OP_SEL
Definition: SIDefines.h:219
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1618
getWaitStatesSince
static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock * > &Visited)
Definition: GCNHazardRecognizer.cpp:432
MFMAPaddingRatio
static cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))
llvm::GCNSubtarget::getRegisterInfo
const SIRegisterInfo * getRegisterInfo() const override
Definition: GCNSubtarget.h:226
llvm::GCNSubtarget::getInstrInfo
const SIInstrInfo * getInstrInfo() const override
Definition: GCNSubtarget.h:214
llvm::TargetSchedModel::init
void init(const TargetSubtargetInfo *TSInfo)
Initialize the machine model for instruction scheduling.
Definition: TargetSchedule.cpp:62
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:186
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:650
llvm::SIInstrInfo::isMIMG
static bool isMIMG(const MachineInstr &MI)
Definition: SIInstrInfo.h:502
llvm::PPCISD::SC
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
Definition: PPCISelLowering.h:418
GCNSubtarget.h
getHWReg
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr)
Definition: GCNHazardRecognizer.cpp:174
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:546
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:738
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:501
int
Clang compiles this i1 i64 store i64 i64 store i64 i64 store i64 i64 store i64 align Which gets codegen d xmm0 movaps rbp movaps rbp movaps rbp movaps rbp rbp rbp rbp rbp It would be better to have movq s of instead of the movaps s LLVM produces ret int
Definition: README.txt:536
llvm::SIInstrInfo::isMAI
static bool isMAI(const MachineInstr &MI)
Definition: SIInstrInfo.h:653
llvm::MCWriteProcResEntry::Cycles
uint16_t Cycles
Definition: MCSchedule.h:65
llvm::MCSchedClassDesc
Summarize the scheduling resources required for an instruction of a particular scheduling class.
Definition: MCSchedule.h:109
shouldRunLdsBranchVmemWARHazardFixup
static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF, const GCNSubtarget &ST)
Definition: GCNHazardRecognizer.cpp:1237
llvm::GCNSubtarget::hasSMRDReadVALUDefHazard
bool hasSMRDReadVALUDefHazard() const
A read of an SGPR by SMRD instruction requires 4 wait states when the SGPR was written by a VALU inst...
Definition: GCNSubtarget.h:428
isRFE
static bool isRFE(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:102
llvm::AMDGPU::getMAIIsGFX940XDL
bool getMAIIsGFX940XDL(unsigned Opc)
Definition: AMDGPUBaseInfo.cpp:375
false
Definition: StackSlotColoring.cpp:141
llvm::GCNSubtarget::hasGFX940Insts
bool hasGFX940Insts() const
Definition: GCNSubtarget.h:1047
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:127
llvm::GraphProgram::DOT
@ DOT
Definition: GraphWriter.h:51
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:197
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:48
llvm::M0
unsigned M0(unsigned Val)
Definition: VE.h:370
llvm::cl::parser
Definition: CommandLine.h:797
llvm::AMDGPU::decodeWaitcnt
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt)
Decodes Vmcnt, Expcnt and Lgkmcnt from given Waitcnt for given isa Version, and writes decoded values...
Definition: AMDGPUBaseInfo.cpp:1005
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
llvm::SIRegisterInfo
Definition: SIRegisterInfo.h:30
llvm::TargetSchedModel::getWriteProcResEnd
ProcResIter getWriteProcResEnd(const MCSchedClassDesc *SC) const
Definition: TargetSchedule.h:137
llvm::BitVector
Definition: BitVector.h:75
llvm::MCOperandInfo::RegClass
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition: MCInstrDesc.h:90
llvm::GCNSubtarget::hasTransForwardingHazard
bool hasTransForwardingHazard() const
Definition: GCNSubtarget.h:990
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::GCNSubtarget::hasVcmpxExecWARHazard
bool hasVcmpxExecWARHazard() const
Definition: GCNSubtarget.h:980
llvm::InlineAsm::MIOp_FirstOperand
@ MIOp_FirstOperand
Definition: InlineAsm.h:225
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:50
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::TargetSchedModel::resolveSchedClass
const MCSchedClassDesc * resolveSchedClass(const MachineInstr *MI) const
Return the MCSchedClassDesc for this instruction.
Definition: TargetSchedule.cpp:132
llvm::SIInstrFlags::DPP
@ DPP
Definition: SIDefines.h:50
isSendMsgTraceDataOrGDS
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:137
isPermlane
static bool isPermlane(const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:163
llvm::DenseSet
Implements a dense probed hash-table based set.
Definition: DenseSet.h:268
llvm::cl::opt
Definition: CommandLine.h:1392
llvm::AMDGPU::getRegBitWidth
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
Definition: AMDGPUBaseInfo.cpp:1881
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:239
llvm::cl::Option
Definition: CommandLine.h:242
llvm::SIInstrInfo::isDPP
static bool isDPP(const MachineInstr &MI)
Definition: SIInstrInfo.h:621
llvm::ScheduleHazardRecognizer::HazardType
HazardType
Definition: ScheduleHazardRecognizer.h:37
AMDGPUMCTargetDesc.h
llvm::TargetRegisterInfo::regsOverlap
bool regsOverlap(Register RegA, Register RegB) const
Returns true if the two registers are equal or alias each other.
Definition: TargetRegisterInfo.h:419
llvm::GCNHazardRecognizer::PreEmitNoopsCommon
unsigned PreEmitNoopsCommon(MachineInstr *)
Definition: GCNHazardRecognizer.cpp:310
llvm::AMDGPU::Hwreg::Offset
Offset
Definition: SIDefines.h:409
llvm::MachineBasicBlock::instr_rend
reverse_instr_iterator instr_rend()
Definition: MachineBasicBlock.h:267
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:320
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
UseReg
static Register UseReg(const MachineOperand &MO)
Definition: HexagonCopyToCombine.cpp:254
llvm::GCNSubtarget::hasVMEMtoScalarWriteHazard
bool hasVMEMtoScalarWriteHazard() const
Definition: GCNSubtarget.h:964
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::SIInstrInfo::isDS
static bool isDS(const MachineInstr &MI)
Definition: SIInstrInfo.h:492
llvm::SIInstrInfo::isSMRD
static bool isSMRD(const MachineInstr &MI)
Definition: SIInstrInfo.h:482
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SUnit::getInstr
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:373
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
isLdsDma
static bool isLdsDma(const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:169
llvm::AMDGPU::SDWA::DWORD
@ DWORD
Definition: SIDefines.h:765
addRegsToSet
static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &Set)
Definition: GCNHazardRecognizer.cpp:532
llvm::GCNHazardRecognizer::PreEmitNoops
unsigned PreEmitNoops(MachineInstr *) override
This overload will be used when the hazard recognizer is being used by a non-scheduling pass,...
Definition: GCNHazardRecognizer.cpp:301
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::SIInstrInfo::isFPAtomic
static bool isFPAtomic(const MachineInstr &MI)
Definition: SIInstrInfo.h:743
llvm::GCNSubtarget::hasReadM0MovRelInterpHazard
bool hasReadM0MovRelInterpHazard() const
Definition: GCNSubtarget.h:943
llvm::MCInstrDesc::OpInfo
const MCOperandInfo * OpInfo
Definition: MCInstrDesc.h:208
llvm::GCNSubtarget::hasRFEHazards
bool hasRFEHazards() const
Definition: GCNSubtarget.h:438
llvm::GCNSubtarget::hasGFX90AInsts
bool hasGFX90AInsts() const
Definition: GCNSubtarget.h:1012
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::AMDGPU::getMIMGInfo
const LLVM_READONLY MIMGInfo * getMIMGInfo(unsigned Opc)
llvm::GCNSubtarget::hasNoDataDepHazard
bool hasNoDataDepHazard() const
Definition: GCNSubtarget.h:754
llvm::GCNHazardRecognizer::EmitInstruction
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
Definition: GCNHazardRecognizer.cpp:71
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:359
llvm::MachineBasicBlock::predecessors
iterator_range< pred_iterator > predecessors()
Definition: MachineBasicBlock.h:358
IsExpiredFn
function_ref< bool(const MachineInstr &, int WaitStates)> IsExpiredFn
Definition: GCNHazardRecognizer.cpp:427
llvm::GCNHazardRecognizer::EmitNoop
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
Definition: GCNHazardRecognizer.cpp:377
breaksVMEMSoftClause
static bool breaksVMEMSoftClause(MachineInstr *MI)
Definition: GCNHazardRecognizer.cpp:551
llvm::GCNSubtarget::hasLdsBranchVmemWARHazard
bool hasLdsBranchVmemWARHazard() const
Definition: GCNSubtarget.h:984
llvm::SIMachineFunctionInfo::getOccupancy
unsigned getOccupancy() const
Definition: SIMachineFunctionInfo.h:955
llvm::SIInstrInfo::isSALU
static bool isSALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:354
llvm::MachineBasicBlock::instr_end
instr_iterator instr_end()
Definition: MachineBasicBlock.h:263
llvm::MachineFunction
Definition: MachineFunction.h:241
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
insertNoopsInBundle
static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity)
Definition: GCNHazardRecognizer.cpp:257
cl
http eax xorl edx cl sete al setne dl sall cl
Definition: README.txt:25
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::SIInstrInfo::isSDWA
static bool isSDWA(const MachineInstr &MI)
Definition: SIInstrInfo.h:450
parse
static llvm::Error parse(DataExtractor &Data, uint64_t BaseAddr, LineEntryCallback const &Callback)
Definition: LineTable.cpp:54
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
llvm::SIInstrFlags::SMRD
@ SMRD
Definition: SIDefines.h:56
isSSetReg
static bool isSSetReg(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:87
llvm::SIInstrInfo::isMUBUF
static bool isMUBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:466
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:288
llvm::SIInstrInfo::isDOT
static bool isDOT(const MachineInstr &MI)
Definition: SIInstrInfo.h:661
LLVM_FALLTHROUGH
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:280
llvm::GCNHazardRecognizer::Reset
void Reset() override
Reset - This callback is invoked when a new block of instructions is about to be schedule.
Definition: GCNHazardRecognizer.cpp:67
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
isDGEMM
static bool isDGEMM(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:118
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::ScheduleHazardRecognizer::NoopHazard
@ NoopHazard
Definition: ScheduleHazardRecognizer.h:40
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::GCNSubtarget::hasNSAtoVMEMBug
bool hasNSAtoVMEMBug() const
Definition: GCNSubtarget.h:1004
llvm::GCNHazardRecognizer::RecedeCycle
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
Definition: GCNHazardRecognizer.cpp:419
llvm::ScheduleHazardRecognizer::NoHazard
@ NoHazard
Definition: ScheduleHazardRecognizer.h:38
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:341
llvm::SIInstrInfo::isSegmentSpecificFLAT
static bool isSegmentSpecificFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:524
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::MachineInstr::mayStore
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:1026
llvm::SIInstrInfo::isVMEM
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:370
llvm::SIInstrInfo::isMTBUF
static bool isMTBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:474
llvm::MachineInstr::isBundle
bool isBundle() const
Definition: MachineInstr.h:1289
llvm::SIInstrInfo
Definition: SIInstrInfo.h:43
GCNHazardRecognizer.h
llvm::GCNHazardRecognizer::AdvanceCycle
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
Definition: GCNHazardRecognizer.cpp:381
llvm::MCRegUnitIterator
Definition: MCRegisterInfo.h:680
llvm::GCNSubtarget::isXNACKEnabled
bool isXNACKEnabled() const
Definition: GCNSubtarget.h:533
isDivFMas
static bool isDivFMas(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:79
llvm::SUnit::isInstr
bool isInstr() const
Returns true if this SUnit refers to a machine instruction as opposed to an SDNode.
Definition: ScheduleDAG.h:362
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
llvm::SIInstrInfo::isVALU
static bool isVALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:362
llvm::MCRegisterInfo::DiffListIterator::isValid
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
Definition: MCRegisterInfo.h:224
llvm::MachineInstr::getNumOperands
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:494
llvm::SIMachineFunctionInfo
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Definition: SIMachineFunctionInfo.h:348
llvm::GCNHazardRecognizer::ShouldPreferAnother
bool ShouldPreferAnother(SUnit *SU) override
ShouldPreferAnother - This callback may be invoked if getHazardType returns NoHazard.
Definition: GCNHazardRecognizer.cpp:2136
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:340
llvm::iterator_range
A range adaptor for a pair of iterators.
Definition: iterator_range.h:30
llvm::GCNHazardRecognizer::IsHazardFn
function_ref< bool(const MachineInstr &)> IsHazardFn
Definition: GCNHazardRecognizer.h:34
IV
static const uint32_t IV[8]
Definition: blake3_impl.h:85
llvm::GCNSubtarget::hasReadM0SendMsgHazard
bool hasReadM0SendMsgHazard() const
Definition: GCNSubtarget.h:947
llvm::BitVector::anyCommon
bool anyCommon(const BitVector &RHS) const
Test if any common bits are set.
Definition: BitVector.h:482
llvm::SUnit
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:242
llvm::GCNSubtarget::hasSMEMtoVectorWriteHazard
bool hasSMEMtoVectorWriteHazard() const
Definition: GCNSubtarget.h:968
llvm::AMDGPU::Hwreg::ID_TRAPSTS
@ ID_TRAPSTS
Definition: SIDefines.h:381
llvm::SIInstrFlags::VALU
@ VALU
Definition: SIDefines.h:30
llvm::SIInstrInfo::isTRANS
static bool isTRANS(const MachineInstr &MI)
Definition: SIInstrInfo.h:629
llvm::SIInstrInfo::isFLAT
static bool isFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:518
llvm::cl::desc
Definition: CommandLine.h:405
llvm::RegState::Kill
@ Kill
The last use of a register.
Definition: MachineInstrBuilder.h:48
breaksSMEMSoftClause
static bool breaksSMEMSoftClause(MachineInstr *MI)
Definition: GCNHazardRecognizer.cpp:547
llvm::GCNSubtarget::has12DWordStoreHazard
bool has12DWordStoreHazard() const
Definition: GCNSubtarget.h:934
MachineFunction.h
llvm::GCNSubtarget::hasVcmpxPermlaneHazard
bool hasVcmpxPermlaneHazard() const
Definition: GCNSubtarget.h:960
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
isSMovRel
static bool isSMovRel(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:106
llvm::SIInstrInfo::isVINTRP
static bool isVINTRP(const MachineInstr &MI)
Definition: SIInstrInfo.h:645
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
llvm::GCNSubtarget::hasMAIInsts
bool hasMAIInsts() const
Definition: GCNSubtarget.h:712
llvm::MCRegister
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:24
llvm::GCNSubtarget::hasReadM0LdsDirectHazard
bool hasReadM0LdsDirectHazard() const
Definition: GCNSubtarget.h:956