LLVM  13.0.0git
GCNHazardRecognizer.cpp
Go to the documentation of this file.
1 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "GCNHazardRecognizer.h"
14 #include "GCNSubtarget.h"
19 
20 using namespace llvm;
21 
22 //===----------------------------------------------------------------------===//
23 // Hazard Recoginizer Implementation
24 //===----------------------------------------------------------------------===//
25 
27  IsHazardRecognizerMode(false),
28  CurrCycleInstr(nullptr),
29  MF(MF),
30  ST(MF.getSubtarget<GCNSubtarget>()),
31  TII(*ST.getInstrInfo()),
32  TRI(TII.getRegisterInfo()),
33  ClauseUses(TRI.getNumRegUnits()),
34  ClauseDefs(TRI.getNumRegUnits()) {
35  MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
36  TSchedModel.init(&ST);
37 }
38 
40  EmittedInstrs.clear();
41 }
42 
45 }
46 
48  CurrCycleInstr = MI;
49 }
50 
51 static bool isDivFMas(unsigned Opcode) {
52  return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
53 }
54 
55 static bool isSGetReg(unsigned Opcode) {
56  return Opcode == AMDGPU::S_GETREG_B32;
57 }
58 
59 static bool isSSetReg(unsigned Opcode) {
60  switch (Opcode) {
61  case AMDGPU::S_SETREG_B32:
62  case AMDGPU::S_SETREG_B32_mode:
63  case AMDGPU::S_SETREG_IMM32_B32:
64  case AMDGPU::S_SETREG_IMM32_B32_mode:
65  return true;
66  }
67  return false;
68 }
69 
70 static bool isRWLane(unsigned Opcode) {
71  return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
72 }
73 
74 static bool isRFE(unsigned Opcode) {
75  return Opcode == AMDGPU::S_RFE_B64;
76 }
77 
78 static bool isSMovRel(unsigned Opcode) {
79  switch (Opcode) {
80  case AMDGPU::S_MOVRELS_B32:
81  case AMDGPU::S_MOVRELS_B64:
82  case AMDGPU::S_MOVRELD_B32:
83  case AMDGPU::S_MOVRELD_B64:
84  return true;
85  default:
86  return false;
87  }
88 }
89 
90 static bool isDGEMM(unsigned Opcode) {
91  return Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
92  Opcode == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64 ||
93  Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_e64 ||
94  Opcode == AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64;
95 }
96 
97 static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
98  unsigned Opcode = MI.getOpcode();
99 
100  if (!SIInstrInfo::isMAI(MI) ||
101  isDGEMM(Opcode) ||
102  Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
103  Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
104  return false;
105 
106  return true;
107 }
108 
110  const MachineInstr &MI) {
111  if (TII.isAlwaysGDS(MI.getOpcode()))
112  return true;
113 
114  switch (MI.getOpcode()) {
115  case AMDGPU::S_SENDMSG:
116  case AMDGPU::S_SENDMSGHALT:
117  case AMDGPU::S_TTRACEDATA:
118  return true;
119  // These DS opcodes don't support GDS.
120  case AMDGPU::DS_NOP:
121  case AMDGPU::DS_PERMUTE_B32:
122  case AMDGPU::DS_BPERMUTE_B32:
123  return false;
124  default:
125  if (TII.isDS(MI.getOpcode())) {
126  int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
127  AMDGPU::OpName::gds);
128  if (MI.getOperand(GDS).getImm())
129  return true;
130  }
131  return false;
132  }
133 }
134 
135 static bool isPermlane(const MachineInstr &MI) {
136  unsigned Opcode = MI.getOpcode();
137  return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
138  Opcode == AMDGPU::V_PERMLANEX16_B32_e64;
139 }
140 
141 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
142  const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
143  AMDGPU::OpName::simm16);
144  return RegOp->getImm() & AMDGPU::Hwreg::ID_MASK_;
145 }
146 
149  MachineInstr *MI = SU->getInstr();
150  // If we are not in "HazardRecognizerMode" and therefore not being run from
151  // the scheduler, track possible stalls from hazards but don't insert noops.
152  auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard;
153 
154  if (MI->isBundle())
155  return NoHazard;
156 
157  if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
158  return HazardType;
159 
160  if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
161  return HazardType;
162 
163  if (checkFPAtomicToDenormModeHazard(MI) > 0)
164  return HazardType;
165 
166  if (ST.hasNoDataDepHazard())
167  return NoHazard;
168 
169  // FIXME: Should flat be considered vmem?
170  if ((SIInstrInfo::isVMEM(*MI) ||
172  && checkVMEMHazards(MI) > 0)
173  return HazardType;
174 
175  if (SIInstrInfo::isVALU(*MI) && checkVALUHazards(MI) > 0)
176  return HazardType;
177 
178  if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
179  return HazardType;
180 
181  if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
182  return HazardType;
183 
184  if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
185  return HazardType;
186 
189  SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
190  return HazardType;
191 
192  if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
193  return HazardType;
194 
195  if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
196  return HazardType;
197 
198  if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
199  return HazardType;
200 
201  if (ST.hasReadM0MovRelInterpHazard() &&
202  (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode())) &&
203  checkReadM0Hazards(MI) > 0)
204  return HazardType;
205 
207  checkReadM0Hazards(MI) > 0)
208  return HazardType;
209 
210  if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
211  return HazardType;
212 
213  if ((SIInstrInfo::isVMEM(*MI) ||
215  SIInstrInfo::isDS(*MI)) && checkMAILdStHazards(MI) > 0)
216  return HazardType;
217 
218  if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
219  return HazardType;
220 
221  return NoHazard;
222 }
223 
225  unsigned Quantity) {
226  while (Quantity > 0) {
227  unsigned Arg = std::min(Quantity, 8u);
228  Quantity -= Arg;
229  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
230  .addImm(Arg - 1);
231  }
232 }
233 
234 void GCNHazardRecognizer::processBundle() {
235  MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
237  // Check bundled MachineInstr's for hazards.
238  for (; MI != E && MI->isInsideBundle(); ++MI) {
239  CurrCycleInstr = &*MI;
240  unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
241 
242  if (IsHazardRecognizerMode) {
243  fixHazards(CurrCycleInstr);
244 
245  insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
246  }
247 
248  // It’s unnecessary to track more than MaxLookAhead instructions. Since we
249  // include the bundled MI directly after, only add a maximum of
250  // (MaxLookAhead - 1) noops to EmittedInstrs.
251  for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
252  EmittedInstrs.push_front(nullptr);
253 
254  EmittedInstrs.push_front(CurrCycleInstr);
255  EmittedInstrs.resize(MaxLookAhead);
256  }
257  CurrCycleInstr = nullptr;
258 }
259 
261  IsHazardRecognizerMode = true;
262  CurrCycleInstr = MI;
263  unsigned W = PreEmitNoopsCommon(MI);
264  fixHazards(MI);
265  CurrCycleInstr = nullptr;
266  return W;
267 }
268 
270  if (MI->isBundle())
271  return 0;
272 
273  int WaitStates = 0;
274 
275  if (SIInstrInfo::isSMRD(*MI))
276  return std::max(WaitStates, checkSMRDHazards(MI));
277 
278  if (ST.hasNSAtoVMEMBug())
279  WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
280 
281  WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI));
282 
283  if (ST.hasNoDataDepHazard())
284  return WaitStates;
285 
287  WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
288 
289  if (SIInstrInfo::isVALU(*MI))
290  WaitStates = std::max(WaitStates, checkVALUHazards(MI));
291 
292  if (SIInstrInfo::isDPP(*MI))
293  WaitStates = std::max(WaitStates, checkDPPHazards(MI));
294 
295  if (isDivFMas(MI->getOpcode()))
296  WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
297 
298  if (isRWLane(MI->getOpcode()))
299  WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
300 
303  SIInstrInfo::isEXP(*MI)) && checkMAIVALUHazards(MI) > 0)
304  WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
305 
306  if (MI->isInlineAsm())
307  return std::max(WaitStates, checkInlineAsmHazards(MI));
308 
309  if (isSGetReg(MI->getOpcode()))
310  return std::max(WaitStates, checkGetRegHazards(MI));
311 
312  if (isSSetReg(MI->getOpcode()))
313  return std::max(WaitStates, checkSetRegHazards(MI));
314 
315  if (isRFE(MI->getOpcode()))
316  return std::max(WaitStates, checkRFEHazards(MI));
317 
318  if (ST.hasReadM0MovRelInterpHazard() && (TII.isVINTRP(*MI) ||
319  isSMovRel(MI->getOpcode())))
320  return std::max(WaitStates, checkReadM0Hazards(MI));
321 
323  return std::max(WaitStates, checkReadM0Hazards(MI));
324 
325  if (SIInstrInfo::isMAI(*MI))
326  return std::max(WaitStates, checkMAIHazards(MI));
327 
328  if (SIInstrInfo::isVMEM(*MI) ||
331  return std::max(WaitStates, checkMAILdStHazards(MI));
332 
333  return WaitStates;
334 }
335 
337  EmittedInstrs.push_front(nullptr);
338 }
339 
341  // When the scheduler detects a stall, it will call AdvanceCycle() without
342  // emitting any instructions.
343  if (!CurrCycleInstr) {
344  EmittedInstrs.push_front(nullptr);
345  return;
346  }
347 
348  // Do not track non-instructions which do not affect the wait states.
349  // If included, these instructions can lead to buffer overflow such that
350  // detectable hazards are missed.
351  if (CurrCycleInstr->isImplicitDef() || CurrCycleInstr->isDebugInstr() ||
352  CurrCycleInstr->isKill()) {
353  CurrCycleInstr = nullptr;
354  return;
355  }
356 
357  if (CurrCycleInstr->isBundle()) {
358  processBundle();
359  return;
360  }
361 
362  unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
363 
364  // Keep track of emitted instructions
365  EmittedInstrs.push_front(CurrCycleInstr);
366 
367  // Add a nullptr for each additional wait state after the first. Make sure
368  // not to add more than getMaxLookAhead() items to the list, since we
369  // truncate the list to that size right after this loop.
370  for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
371  i < e; ++i) {
372  EmittedInstrs.push_front(nullptr);
373  }
374 
375  // getMaxLookahead() is the largest number of wait states we will ever need
376  // to insert, so there is no point in keeping track of more than that many
377  // wait states.
378  EmittedInstrs.resize(getMaxLookAhead());
379 
380  CurrCycleInstr = nullptr;
381 }
382 
384  llvm_unreachable("hazard recognizer does not support bottom-up scheduling.");
385 }
386 
387 //===----------------------------------------------------------------------===//
388 // Helper Functions
389 //===----------------------------------------------------------------------===//
390 
391 typedef function_ref<bool(MachineInstr *, int WaitStates)> IsExpiredFn;
392 
393 // Returns a minimum wait states since \p I walking all predecessors.
394 // Only scans until \p IsExpired does not return true.
395 // Can only be run in a hazard recognizer mode.
399  int WaitStates,
400  IsExpiredFn IsExpired,
402  for (auto E = MBB->instr_rend(); I != E; ++I) {
403  // Don't add WaitStates for parent BUNDLE instructions.
404  if (I->isBundle())
405  continue;
406 
407  if (IsHazard(&*I))
408  return WaitStates;
409 
410  if (I->isInlineAsm() || I->isMetaInstruction())
411  continue;
412 
413  WaitStates += SIInstrInfo::getNumWaitStates(*I);
414 
415  if (IsExpired(&*I, WaitStates))
417  }
418 
419  int MinWaitStates = WaitStates;
420  bool Found = false;
421  for (MachineBasicBlock *Pred : MBB->predecessors()) {
422  if (!Visited.insert(Pred).second)
423  continue;
424 
425  int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(),
426  WaitStates, IsExpired, Visited);
427 
429  continue;
430 
431  MinWaitStates = Found ? std::min(MinWaitStates, W) : W;
432  if (IsExpired(nullptr, MinWaitStates))
433  return MinWaitStates;
434 
435  Found = true;
436  }
437 
438  if (Found)
439  return MinWaitStates;
440 
442 }
443 
445  MachineInstr *MI,
446  IsExpiredFn IsExpired) {
448  return getWaitStatesSince(IsHazard, MI->getParent(),
449  std::next(MI->getReverseIterator()),
450  0, IsExpired, Visited);
451 }
452 
453 int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
454  if (IsHazardRecognizerMode) {
455  auto IsExpiredFn = [Limit] (MachineInstr *, int WaitStates) {
456  return WaitStates >= Limit;
457  };
458  return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
459  }
460 
461  int WaitStates = 0;
462  for (MachineInstr *MI : EmittedInstrs) {
463  if (MI) {
464  if (IsHazard(MI))
465  return WaitStates;
466 
467  if (MI->isInlineAsm())
468  continue;
469  }
470  ++WaitStates;
471 
472  if (WaitStates >= Limit)
473  break;
474  }
476 }
477 
478 int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
479  IsHazardFn IsHazardDef,
480  int Limit) {
481  const SIRegisterInfo *TRI = ST.getRegisterInfo();
482 
483  auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) {
484  return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI);
485  };
486 
487  return getWaitStatesSince(IsHazardFn, Limit);
488 }
489 
490 int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
491  int Limit) {
492  auto IsHazardFn = [IsHazard] (MachineInstr *MI) {
493  return isSSetReg(MI->getOpcode()) && IsHazard(MI);
494  };
495 
496  return getWaitStatesSince(IsHazardFn, Limit);
497 }
498 
499 //===----------------------------------------------------------------------===//
500 // No-op Hazard Detection
501 //===----------------------------------------------------------------------===//
502 
503 static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
504  MCRegister Reg) {
505  for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI)
506  BV.set(*RUI);
507 }
508 
509 static void addRegsToSet(const SIRegisterInfo &TRI,
511  BitVector &Set) {
512  for (const MachineOperand &Op : Ops) {
513  if (Op.isReg())
514  addRegUnits(TRI, Set, Op.getReg().asMCReg());
515  }
516 }
517 
518 void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) {
519  // XXX: Do we need to worry about implicit operands
520  addRegsToSet(TRI, MI.defs(), ClauseDefs);
521  addRegsToSet(TRI, MI.uses(), ClauseUses);
522 }
523 
525  return !SIInstrInfo::isSMRD(*MI);
526 }
527 
530 }
531 
532 int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
533  // SMEM soft clause are only present on VI+, and only matter if xnack is
534  // enabled.
535  if (!ST.isXNACKEnabled())
536  return 0;
537 
538  bool IsSMRD = TII.isSMRD(*MEM);
539 
540  resetClause();
541 
542  // A soft-clause is any group of consecutive SMEM instructions. The
543  // instructions in this group may return out of order and/or may be
544  // replayed (i.e. the same instruction issued more than once).
545  //
546  // In order to handle these situations correctly we need to make sure that
547  // when a clause has more than one instruction, no instruction in the clause
548  // writes to a register that is read by another instruction in the clause
549  // (including itself). If we encounter this situaion, we need to break the
550  // clause by inserting a non SMEM instruction.
551 
552  for (MachineInstr *MI : EmittedInstrs) {
553  // When we hit a non-SMEM instruction then we have passed the start of the
554  // clause and we can stop.
555  if (!MI)
556  break;
557 
559  break;
560 
561  addClauseInst(*MI);
562  }
563 
564  if (ClauseDefs.none())
565  return 0;
566 
567  // We need to make sure not to put loads and stores in the same clause if they
568  // use the same address. For now, just start a new clause whenever we see a
569  // store.
570  if (MEM->mayStore())
571  return 1;
572 
573  addClauseInst(*MEM);
574 
575  // If the set of defs and uses intersect then we cannot add this instruction
576  // to the clause, so we have a hazard.
577  return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
578 }
579 
580 int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
581  int WaitStatesNeeded = 0;
582 
583  WaitStatesNeeded = checkSoftClauseHazards(SMRD);
584 
585  // This SMRD hazard only affects SI.
586  if (!ST.hasSMRDReadVALUDefHazard())
587  return WaitStatesNeeded;
588 
589  // A read of an SGPR by SMRD instruction requires 4 wait states when the
590  // SGPR was written by a VALU instruction.
591  int SmrdSgprWaitStates = 4;
592  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
593  auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); };
594 
595  bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
596 
597  for (const MachineOperand &Use : SMRD->uses()) {
598  if (!Use.isReg())
599  continue;
600  int WaitStatesNeededForUse =
601  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
602  SmrdSgprWaitStates);
603  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
604 
605  // This fixes what appears to be undocumented hardware behavior in SI where
606  // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
607  // needs some number of nops in between. We don't know how many we need, but
608  // let's use 4. This wasn't discovered before probably because the only
609  // case when this happens is when we expand a 64-bit pointer into a full
610  // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
611  // probably never encountered in the closed-source land.
612  if (IsBufferSMRD) {
613  int WaitStatesNeededForUse =
614  SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
615  IsBufferHazardDefFn,
616  SmrdSgprWaitStates);
617  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
618  }
619  }
620 
621  return WaitStatesNeeded;
622 }
623 
624 int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
626  return 0;
627 
628  int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
629 
630  // A read of an SGPR by a VMEM instruction requires 5 wait states when the
631  // SGPR was written by a VALU Instruction.
632  const int VmemSgprWaitStates = 5;
633  auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
634  for (const MachineOperand &Use : VMEM->uses()) {
635  if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
636  continue;
637 
638  int WaitStatesNeededForUse =
639  VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
640  VmemSgprWaitStates);
641  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
642  }
643  return WaitStatesNeeded;
644 }
645 
646 int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
647  const SIRegisterInfo *TRI = ST.getRegisterInfo();
648  const SIInstrInfo *TII = ST.getInstrInfo();
649 
650  // Check for DPP VGPR read after VALU VGPR write and EXEC write.
651  int DppVgprWaitStates = 2;
652  int DppExecWaitStates = 5;
653  int WaitStatesNeeded = 0;
654  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
655 
656  for (const MachineOperand &Use : DPP->uses()) {
657  if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
658  continue;
659  int WaitStatesNeededForUse =
660  DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
661  [](MachineInstr *) { return true; },
662  DppVgprWaitStates);
663  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
664  }
665 
666  WaitStatesNeeded = std::max(
667  WaitStatesNeeded,
668  DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
669  DppExecWaitStates));
670 
671  return WaitStatesNeeded;
672 }
673 
674 int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
675  const SIInstrInfo *TII = ST.getInstrInfo();
676 
677  // v_div_fmas requires 4 wait states after a write to vcc from a VALU
678  // instruction.
679  const int DivFMasWaitStates = 4;
680  auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
681  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
682  DivFMasWaitStates);
683 
684  return DivFMasWaitStates - WaitStatesNeeded;
685 }
686 
687 int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
688  const SIInstrInfo *TII = ST.getInstrInfo();
689  unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
690 
691  const int GetRegWaitStates = 2;
692  auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
693  return GetRegHWReg == getHWReg(TII, *MI);
694  };
695  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
696 
697  return GetRegWaitStates - WaitStatesNeeded;
698 }
699 
700 int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
701  const SIInstrInfo *TII = ST.getInstrInfo();
702  unsigned HWReg = getHWReg(TII, *SetRegInstr);
703 
704  const int SetRegWaitStates = ST.getSetRegWaitStates();
705  auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
706  return HWReg == getHWReg(TII, *MI);
707  };
708  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
709  return SetRegWaitStates - WaitStatesNeeded;
710 }
711 
712 int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
713  if (!MI.mayStore())
714  return -1;
715 
716  const SIInstrInfo *TII = ST.getInstrInfo();
717  unsigned Opcode = MI.getOpcode();
718  const MCInstrDesc &Desc = MI.getDesc();
719 
720  int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
721  int VDataRCID = -1;
722  if (VDataIdx != -1)
723  VDataRCID = Desc.OpInfo[VDataIdx].RegClass;
724 
725  if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
726  // There is no hazard if the instruction does not use vector regs
727  // (like wbinvl1)
728  if (VDataIdx == -1)
729  return -1;
730  // For MUBUF/MTBUF instructions this hazard only exists if the
731  // instruction is not using a register in the soffset field.
732  const MachineOperand *SOffset =
733  TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
734  // If we have no soffset operand, then assume this field has been
735  // hardcoded to zero.
736  if (AMDGPU::getRegBitWidth(VDataRCID) > 64 &&
737  (!SOffset || !SOffset->isReg()))
738  return VDataIdx;
739  }
740 
741  // MIMG instructions create a hazard if they don't use a 256-bit T# and
742  // the store size is greater than 8 bytes and they have more than two bits
743  // of their dmask set.
744  // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
745  if (TII->isMIMG(MI)) {
746  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
747  assert(SRsrcIdx != -1 &&
748  AMDGPU::getRegBitWidth(Desc.OpInfo[SRsrcIdx].RegClass) == 256);
749  (void)SRsrcIdx;
750  }
751 
752  if (TII->isFLAT(MI)) {
753  int DataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
754  if (AMDGPU::getRegBitWidth(Desc.OpInfo[DataIdx].RegClass) > 64)
755  return DataIdx;
756  }
757 
758  return -1;
759 }
760 
761 int
762 GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
763  const MachineRegisterInfo &MRI) {
764  // Helper to check for the hazard where VMEM instructions that store more than
765  // 8 bytes can have there store data over written by the next instruction.
766  const SIRegisterInfo *TRI = ST.getRegisterInfo();
767 
768  const int VALUWaitStates = 1;
769  int WaitStatesNeeded = 0;
770 
771  if (!TRI->isVectorRegister(MRI, Def.getReg()))
772  return WaitStatesNeeded;
773  Register Reg = Def.getReg();
774  auto IsHazardFn = [this, Reg, TRI] (MachineInstr *MI) {
775  int DataIdx = createsVALUHazard(*MI);
776  return DataIdx >= 0 &&
777  TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg);
778  };
779  int WaitStatesNeededForDef =
780  VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
781  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
782 
783  return WaitStatesNeeded;
784 }
785 
786 int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) {
787  // This checks for the hazard where VMEM instructions that store more than
788  // 8 bytes can have there store data over written by the next instruction.
789  if (!ST.has12DWordStoreHazard())
790  return 0;
791 
792  const MachineRegisterInfo &MRI = MF.getRegInfo();
793  int WaitStatesNeeded = 0;
794 
795  for (const MachineOperand &Def : VALU->defs()) {
796  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
797  }
798 
799  return WaitStatesNeeded;
800 }
801 
802 int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) {
803  // This checks for hazards associated with inline asm statements.
804  // Since inline asms can contain just about anything, we use this
805  // to call/leverage other check*Hazard routines. Note that
806  // this function doesn't attempt to address all possible inline asm
807  // hazards (good luck), but is a collection of what has been
808  // problematic thus far.
809 
810  // see checkVALUHazards()
811  if (!ST.has12DWordStoreHazard())
812  return 0;
813 
814  const MachineRegisterInfo &MRI = MF.getRegInfo();
815  int WaitStatesNeeded = 0;
816 
817  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = IA->getNumOperands();
818  I != E; ++I) {
819  const MachineOperand &Op = IA->getOperand(I);
820  if (Op.isReg() && Op.isDef()) {
821  WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
822  }
823  }
824 
825  return WaitStatesNeeded;
826 }
827 
828 int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
829  const SIInstrInfo *TII = ST.getInstrInfo();
830  const SIRegisterInfo *TRI = ST.getRegisterInfo();
831  const MachineRegisterInfo &MRI = MF.getRegInfo();
832 
833  const MachineOperand *LaneSelectOp =
834  TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
835 
836  if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
837  return 0;
838 
839  Register LaneSelectReg = LaneSelectOp->getReg();
840  auto IsHazardFn = [TII] (MachineInstr *MI) {
841  return TII->isVALU(*MI);
842  };
843 
844  const int RWLaneWaitStates = 4;
845  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
846  RWLaneWaitStates);
847  return RWLaneWaitStates - WaitStatesSince;
848 }
849 
850 int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
851  if (!ST.hasRFEHazards())
852  return 0;
853 
854  const SIInstrInfo *TII = ST.getInstrInfo();
855 
856  const int RFEWaitStates = 1;
857 
858  auto IsHazardFn = [TII] (MachineInstr *MI) {
859  return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
860  };
861  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
862  return RFEWaitStates - WaitStatesNeeded;
863 }
864 
865 int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
866  const SIInstrInfo *TII = ST.getInstrInfo();
867  const int SMovRelWaitStates = 1;
868  auto IsHazardFn = [TII] (MachineInstr *MI) {
869  return TII->isSALU(*MI);
870  };
871  return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn,
872  SMovRelWaitStates);
873 }
874 
875 void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
876  fixVMEMtoScalarWriteHazards(MI);
877  fixVcmpxPermlaneHazards(MI);
878  fixSMEMtoVectorWriteHazards(MI);
879  fixVcmpxExecWARHazard(MI);
880  fixLdsBranchVmemWARHazard(MI);
881 }
882 
883 bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
884  if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI))
885  return false;
886 
887  const SIInstrInfo *TII = ST.getInstrInfo();
888  auto IsHazardFn = [TII] (MachineInstr *MI) {
889  return TII->isVOPC(*MI);
890  };
891 
892  auto IsExpiredFn = [] (MachineInstr *MI, int) {
893  if (!MI)
894  return false;
895  unsigned Opc = MI->getOpcode();
896  return SIInstrInfo::isVALU(*MI) &&
897  Opc != AMDGPU::V_NOP_e32 &&
898  Opc != AMDGPU::V_NOP_e64 &&
899  Opc != AMDGPU::V_NOP_sdwa;
900  };
901 
902  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
904  return false;
905 
906  // V_NOP will be discarded by SQ.
907  // Use V_MOB_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
908  // which is always a VGPR and available.
909  auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
910  Register Reg = Src0->getReg();
911  bool IsUndef = Src0->isUndef();
912  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
913  TII->get(AMDGPU::V_MOV_B32_e32))
914  .addReg(Reg, RegState::Define | (IsUndef ? RegState::Dead : 0))
915  .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill);
916 
917  return true;
918 }
919 
920 bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
921  if (!ST.hasVMEMtoScalarWriteHazard())
922  return false;
923 
925  return false;
926 
927  if (MI->getNumDefs() == 0)
928  return false;
929 
930  const SIRegisterInfo *TRI = ST.getRegisterInfo();
931 
932  auto IsHazardFn = [TRI, MI] (MachineInstr *I) {
933  if (!SIInstrInfo::isVMEM(*I) && !SIInstrInfo::isDS(*I) &&
935  return false;
936 
937  for (const MachineOperand &Def : MI->defs()) {
938  MachineOperand *Op = I->findRegisterUseOperand(Def.getReg(), false, TRI);
939  if (!Op)
940  continue;
941  return true;
942  }
943  return false;
944  };
945 
946  auto IsExpiredFn = [](MachineInstr *MI, int) {
947  return MI && (SIInstrInfo::isVALU(*MI) ||
948  (MI->getOpcode() == AMDGPU::S_WAITCNT &&
949  !MI->getOperand(0).getImm()) ||
950  (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
951  MI->getOperand(0).getImm() == 0xffe3));
952  };
953 
954  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
956  return false;
957 
958  const SIInstrInfo *TII = ST.getInstrInfo();
959  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
960  TII->get(AMDGPU::S_WAITCNT_DEPCTR))
961  .addImm(0xffe3);
962  return true;
963 }
964 
965 bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
966  if (!ST.hasSMEMtoVectorWriteHazard())
967  return false;
968 
969  if (!SIInstrInfo::isVALU(*MI))
970  return false;
971 
972  unsigned SDSTName;
973  switch (MI->getOpcode()) {
974  case AMDGPU::V_READLANE_B32:
975  case AMDGPU::V_READFIRSTLANE_B32:
976  SDSTName = AMDGPU::OpName::vdst;
977  break;
978  default:
979  SDSTName = AMDGPU::OpName::sdst;
980  break;
981  }
982 
983  const SIInstrInfo *TII = ST.getInstrInfo();
984  const SIRegisterInfo *TRI = ST.getRegisterInfo();
985  const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
986  const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
987  if (!SDST) {
988  for (const auto &MO : MI->implicit_operands()) {
989  if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg()))) {
990  SDST = &MO;
991  break;
992  }
993  }
994  }
995 
996  if (!SDST)
997  return false;
998 
999  const Register SDSTReg = SDST->getReg();
1000  auto IsHazardFn = [SDSTReg, TRI] (MachineInstr *I) {
1001  return SIInstrInfo::isSMRD(*I) && I->readsRegister(SDSTReg, TRI);
1002  };
1003 
1004  auto IsExpiredFn = [TII, IV] (MachineInstr *MI, int) {
1005  if (MI) {
1006  if (TII->isSALU(*MI)) {
1007  switch (MI->getOpcode()) {
1008  case AMDGPU::S_SETVSKIP:
1009  case AMDGPU::S_VERSION:
1010  case AMDGPU::S_WAITCNT_VSCNT:
1011  case AMDGPU::S_WAITCNT_VMCNT:
1012  case AMDGPU::S_WAITCNT_EXPCNT:
1013  // These instructions cannot not mitigate the hazard.
1014  return false;
1015  case AMDGPU::S_WAITCNT_LGKMCNT:
1016  // Reducing lgkmcnt count to 0 always mitigates the hazard.
1017  return (MI->getOperand(1).getImm() == 0) &&
1018  (MI->getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1019  case AMDGPU::S_WAITCNT: {
1020  const int64_t Imm = MI->getOperand(0).getImm();
1021  AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm);
1022  return (Decoded.LgkmCnt == 0);
1023  }
1024  default:
1025  // SOPP instructions cannot mitigate the hazard.
1026  if (TII->isSOPP(*MI))
1027  return false;
1028  // At this point the SALU can be assumed to mitigate the hazard
1029  // because either:
1030  // (a) it is independent of the at risk SMEM (breaking chain),
1031  // or
1032  // (b) it is dependent on the SMEM, in which case an appropriate
1033  // s_waitcnt lgkmcnt _must_ exist between it and the at risk
1034  // SMEM instruction.
1035  return true;
1036  }
1037  }
1038  }
1039  return false;
1040  };
1041 
1042  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1044  return false;
1045 
1046  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1047  TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
1048  .addImm(0);
1049  return true;
1050 }
1051 
1052 bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
1054  return false;
1055 
1056  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1057  if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
1058  return false;
1059 
1060  auto IsHazardFn = [TRI] (MachineInstr *I) {
1061  if (SIInstrInfo::isVALU(*I))
1062  return false;
1063  return I->readsRegister(AMDGPU::EXEC, TRI);
1064  };
1065 
1066  const SIInstrInfo *TII = ST.getInstrInfo();
1067  auto IsExpiredFn = [TII, TRI] (MachineInstr *MI, int) {
1068  if (!MI)
1069  return false;
1070  if (SIInstrInfo::isVALU(*MI)) {
1071  if (TII->getNamedOperand(*MI, AMDGPU::OpName::sdst))
1072  return true;
1073  for (auto MO : MI->implicit_operands())
1074  if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegClass(MO.getReg())))
1075  return true;
1076  }
1077  if (MI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
1078  (MI->getOperand(0).getImm() & 0xfffe) == 0xfffe)
1079  return true;
1080  return false;
1081  };
1082 
1083  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1085  return false;
1086 
1087  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1088  TII->get(AMDGPU::S_WAITCNT_DEPCTR))
1089  .addImm(0xfffe);
1090  return true;
1091 }
1092 
1093 bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
1094  if (!ST.hasLdsBranchVmemWARHazard())
1095  return false;
1096 
1097  auto IsHazardInst = [] (const MachineInstr *MI) {
1098  if (SIInstrInfo::isDS(*MI))
1099  return 1;
1101  return 2;
1102  return 0;
1103  };
1104 
1105  auto InstType = IsHazardInst(MI);
1106  if (!InstType)
1107  return false;
1108 
1109  auto IsExpiredFn = [&IsHazardInst] (MachineInstr *I, int) {
1110  return I && (IsHazardInst(I) ||
1111  (I->getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1112  I->getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1113  !I->getOperand(1).getImm()));
1114  };
1115 
1116  auto IsHazardFn = [InstType, &IsHazardInst] (MachineInstr *I) {
1117  if (!I->isBranch())
1118  return false;
1119 
1120  auto IsHazardFn = [InstType, IsHazardInst] (MachineInstr *I) {
1121  auto InstType2 = IsHazardInst(I);
1122  return InstType2 && InstType != InstType2;
1123  };
1124 
1125  auto IsExpiredFn = [InstType, &IsHazardInst] (MachineInstr *I, int) {
1126  if (!I)
1127  return false;
1128 
1129  auto InstType2 = IsHazardInst(I);
1130  if (InstType == InstType2)
1131  return true;
1132 
1133  return I->getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
1134  I->getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
1135  !I->getOperand(1).getImm();
1136  };
1137 
1140  };
1141 
1142  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
1144  return false;
1145 
1146  const SIInstrInfo *TII = ST.getInstrInfo();
1147  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
1148  TII->get(AMDGPU::S_WAITCNT_VSCNT))
1149  .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1150  .addImm(0);
1151 
1152  return true;
1153 }
1154 
1155 int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) {
1156  int NSAtoVMEMWaitStates = 1;
1157 
1158  if (!ST.hasNSAtoVMEMBug())
1159  return 0;
1160 
1162  return 0;
1163 
1164  const SIInstrInfo *TII = ST.getInstrInfo();
1165  const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
1166  if (!Offset || (Offset->getImm() & 6) == 0)
1167  return 0;
1168 
1169  auto IsHazardFn = [TII] (MachineInstr *I) {
1170  if (!SIInstrInfo::isMIMG(*I))
1171  return false;
1172  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I->getOpcode());
1173  return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
1174  TII->getInstSizeInBytes(*I) >= 16;
1175  };
1176 
1177  return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
1178 }
1179 
1180 int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) {
1181  int FPAtomicToDenormModeWaitStates = 3;
1182 
1183  if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
1184  return 0;
1185 
1186  auto IsHazardFn = [] (MachineInstr *I) {
1188  return false;
1189  return SIInstrInfo::isFPAtomic(*I);
1190  };
1191 
1192  auto IsExpiredFn = [] (MachineInstr *MI, int WaitStates) {
1193  if (WaitStates >= 3 || SIInstrInfo::isVALU(*MI))
1194  return true;
1195 
1196  switch (MI->getOpcode()) {
1197  case AMDGPU::S_WAITCNT:
1198  case AMDGPU::S_WAITCNT_VSCNT:
1199  case AMDGPU::S_WAITCNT_VMCNT:
1200  case AMDGPU::S_WAITCNT_EXPCNT:
1201  case AMDGPU::S_WAITCNT_LGKMCNT:
1202  case AMDGPU::S_WAIT_IDLE:
1203  return true;
1204  default:
1205  break;
1206  }
1207 
1208  return false;
1209  };
1210 
1211 
1212  return FPAtomicToDenormModeWaitStates -
1213  ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn);
1214 }
1215 
1216 int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) {
1218 
1219  return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
1220 }
1221 
1222 int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) {
1223  int WaitStatesNeeded = 0;
1224  unsigned Opc = MI->getOpcode();
1225 
1226  auto IsVALUFn = [] (MachineInstr *MI) {
1227  return SIInstrInfo::isVALU(*MI);
1228  };
1229 
1230  if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
1231  const int LegacyVALUWritesVGPRWaitStates = 2;
1232  const int VALUWritesExecWaitStates = 4;
1233  const int MaxWaitStates = 4;
1234 
1235  int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1236  getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
1237  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1238 
1239  if (WaitStatesNeeded < MaxWaitStates) {
1240  for (const MachineOperand &Use : MI->explicit_uses()) {
1241  const int MaxWaitStates = 2;
1242 
1243  if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
1244  continue;
1245 
1246  int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
1247  getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates);
1248  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1249 
1250  if (WaitStatesNeeded == MaxWaitStates)
1251  break;
1252  }
1253  }
1254  }
1255 
1256  auto IsMFMAFn = [] (MachineInstr *MI) {
1257  return SIInstrInfo::isMAI(*MI) &&
1258  MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1259  MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1260  };
1261 
1262  for (const MachineOperand &Op : MI->explicit_operands()) {
1263  if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg()))
1264  continue;
1265 
1266  if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1267  continue;
1268 
1269  const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
1270  const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
1271  const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
1272  const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
1273  const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
1274  const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
1275  const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
1276  const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
1277  const int MaxWaitStates = 18;
1278  Register Reg = Op.getReg();
1279  unsigned HazardDefLatency = 0;
1280 
1281  auto IsOverlappedMFMAFn = [Reg, &IsMFMAFn, &HazardDefLatency, this]
1282  (MachineInstr *MI) {
1283  if (!IsMFMAFn(MI))
1284  return false;
1285  Register DstReg = MI->getOperand(0).getReg();
1286  if (DstReg == Reg)
1287  return false;
1288  HazardDefLatency = std::max(HazardDefLatency,
1289  TSchedModel.computeInstrLatency(MI));
1290  return TRI.regsOverlap(DstReg, Reg);
1291  };
1292 
1293  int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn,
1294  MaxWaitStates);
1295  int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
1296  int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1297  int OpNo = MI->getOperandNo(&Op);
1298  if (OpNo == SrcCIdx) {
1299  NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
1300  } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
1301  switch (HazardDefLatency) {
1302  case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
1303  break;
1304  case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
1305  break;
1306  case 16: LLVM_FALLTHROUGH;
1307  default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
1308  break;
1309  }
1310  } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1311  switch (HazardDefLatency) {
1312  case 2: NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
1313  break;
1314  case 8: NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
1315  break;
1316  case 16: LLVM_FALLTHROUGH;
1317  default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
1318  break;
1319  }
1320  }
1321 
1322  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1323  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1324 
1325  if (WaitStatesNeeded == MaxWaitStates)
1326  return WaitStatesNeeded; // Early exit.
1327 
1328  auto IsAccVgprWriteFn = [Reg, this] (MachineInstr *MI) {
1329  if (MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1330  return false;
1331  Register DstReg = MI->getOperand(0).getReg();
1332  return TRI.regsOverlap(Reg, DstReg);
1333  };
1334 
1335  const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
1336  const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
1337  const int AccVGPRWriteAccVgprReadWaitStates = 3;
1338  NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
1339  if (OpNo == SrcCIdx)
1340  NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
1341  else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
1342  NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
1343 
1344  WaitStatesNeededForUse = NeedWaitStates -
1345  getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates);
1346  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1347 
1348  if (WaitStatesNeeded == MaxWaitStates)
1349  return WaitStatesNeeded; // Early exit.
1350  }
1351 
1352  if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
1353  const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
1354  const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
1355  const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
1356  const int MaxWaitStates = 13;
1357  Register DstReg = MI->getOperand(0).getReg();
1358  unsigned HazardDefLatency = 0;
1359 
1360  auto IsSrcCMFMAFn = [DstReg, &IsMFMAFn, &HazardDefLatency, this]
1361  (MachineInstr *MI) {
1362  if (!IsMFMAFn(MI))
1363  return false;
1364  Register Reg = TII.getNamedOperand(*MI, AMDGPU::OpName::src2)->getReg();
1365  HazardDefLatency = std::max(HazardDefLatency,
1366  TSchedModel.computeInstrLatency(MI));
1367  return TRI.regsOverlap(Reg, DstReg);
1368  };
1369 
1370  int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
1371  int NeedWaitStates;
1372  switch (HazardDefLatency) {
1373  case 2: NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
1374  break;
1375  case 8: NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
1376  break;
1377  case 16: LLVM_FALLTHROUGH;
1378  default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
1379  break;
1380  }
1381 
1382  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
1383  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1384  }
1385 
1386  return WaitStatesNeeded;
1387 }
1388 
1389 int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
1390  int WaitStatesNeeded = 0;
1391  unsigned Opc = MI->getOpcode();
1392 
1393  auto IsMFMAFn = [] (MachineInstr *MI) {
1394  return SIInstrInfo::isMAI(*MI) &&
1395  MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1396  MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1397  };
1398 
1399  auto IsLegacyVALUFn = [&IsMFMAFn] (MachineInstr *MI) {
1400  return SIInstrInfo::isVALU(*MI) && !IsMFMAFn(MI);
1401  };
1402 
1403  auto IsLegacyVALUNotDotFn = [&IsMFMAFn] (MachineInstr *MI) {
1404  return SIInstrInfo::isVALU(*MI) &&
1405  !IsMFMAFn(MI) && !SIInstrInfo::isDOT(*MI);
1406  };
1407 
1408  if (!IsMFMAFn(MI))
1409  return WaitStatesNeeded;
1410 
1411  const int VALUWritesExecWaitStates = 4;
1412  int WaitStatesNeededForUse = VALUWritesExecWaitStates -
1413  getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
1414  VALUWritesExecWaitStates);
1415  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1416 
1417  int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1418 
1419  // Loop for both DGEMM and S/HGEMM 2nd instruction.
1420  for (const MachineOperand &Use : MI->explicit_uses()) {
1421  const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
1422  const int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
1423  const int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
1424  const int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
1425  const int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
1426  const int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
1427  const int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
1428  const int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
1429  const int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
1430  const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
1431  const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
1432  const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
1433  const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
1434  const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
1435  const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
1436  const int MaxWaitStates = 19;
1437 
1438  if (!Use.isReg())
1439  continue;
1440  unsigned Reg = Use.getReg();
1441  bool FullReg;
1442  MachineInstr *MI1;
1443 
1444  auto IsOverlappedDGEMMorXDLFn = [Reg, &IsMFMAFn, &FullReg, &MI1, this]
1445  (MachineInstr *MI) {
1446  if (!IsMFMAFn(MI))
1447  return false;
1448  if (!isDGEMM(MI->getOpcode()) && !isXDL(ST, *MI))
1449  return false;
1450  Register DstReg = MI->getOperand(0).getReg();
1451  FullReg = (DstReg == Reg);
1452  MI1 = MI;
1453  return TRI.regsOverlap(DstReg, Reg);
1454  };
1455 
1456  WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
1457  getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
1458  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1459 
1460  int NumWaitStates = getWaitStatesSinceDef(Reg, IsOverlappedDGEMMorXDLFn,
1461  MaxWaitStates);
1462  if (NumWaitStates == std::numeric_limits<int>::max())
1463  continue;
1464 
1465  int OpNo = MI->getOperandNo(&Use);
1466  unsigned Opc1 = MI1->getOpcode();
1467  int NeedWaitStates = 0;
1468  if (OpNo == SrcCIdx) {
1469  if (!isDGEMM(Opc) && isDGEMM(Opc1)) {
1470  NeedWaitStates = 0;
1471  } else if (FullReg) {
1472  if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
1473  Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
1474  (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
1475  Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
1476  NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
1477  } else {
1478  switch (Opc1) {
1479  case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
1480  case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
1481  if (!isXDL(ST, *MI))
1482  NeedWaitStates = DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
1483  break;
1484  case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
1485  case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
1486  if (!isXDL(ST, *MI))
1487  NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
1488  break;
1489  default:
1490  switch (TSchedModel.computeInstrLatency(MI1)) {
1491  case 2:
1492  NeedWaitStates = isDGEMM(Opc)
1493  ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
1494  : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
1495  break;
1496  case 8:
1497  NeedWaitStates = isDGEMM(Opc)
1498  ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
1499  : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
1500  break;
1501  case 16: LLVM_FALLTHROUGH;
1502  default:
1503  NeedWaitStates = isDGEMM(Opc)
1504  ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
1505  : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
1506  }
1507  }
1508  }
1509  } else {
1510  switch (Opc1) {
1511  case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
1512  case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
1513  NeedWaitStates = DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
1514  break;
1515  case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
1516  case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
1517  NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
1518  break;
1519  default:
1520  switch (TSchedModel.computeInstrLatency(MI1)) {
1521  case 2:
1522  NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
1523  break;
1524  case 8:
1525  NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
1526  break;
1527  case 16: LLVM_FALLTHROUGH;
1528  default:
1529  NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
1530  }
1531  }
1532  }
1533  if (WaitStatesNeeded >= NeedWaitStates)
1534  continue;
1535 
1536  WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
1537  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1538 
1539  if (WaitStatesNeeded == MaxWaitStates)
1540  break;
1541  }
1542 
1543  return WaitStatesNeeded;
1544 }
1545 
1546 int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) {
1547  // On gfx90a+ releveant hazards are checked in checkMAIVALUHazards()
1548  if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
1549  return 0;
1550 
1551  int WaitStatesNeeded = 0;
1552 
1553  auto IsAccVgprReadFn = [] (MachineInstr *MI) {
1554  return MI->getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
1555  };
1556 
1557  for (const MachineOperand &Op : MI->explicit_uses()) {
1558  if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg()))
1559  continue;
1560 
1561  Register Reg = Op.getReg();
1562 
1563  const int AccVgprReadLdStWaitStates = 2;
1564  const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
1565  const int MaxWaitStates = 2;
1566 
1567  int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
1568  getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates);
1569  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1570 
1571  if (WaitStatesNeeded == MaxWaitStates)
1572  return WaitStatesNeeded; // Early exit.
1573 
1574  auto IsVALUAccVgprRdWrCheckFn = [Reg, this](MachineInstr *MI) {
1575  if (MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
1576  MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
1577  return false;
1578  auto IsVALUFn = [] (MachineInstr *MI) {
1580  };
1581  return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
1583  };
1584 
1585  WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
1586  getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
1587  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1588  }
1589 
1590  return WaitStatesNeeded;
1591 }
1592 
1593 int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
1594  if (!ST.hasGFX90AInsts())
1595  return 0;
1596 
1597  auto IsMFMAFn = [] (MachineInstr *MI) -> bool {
1598  return SIInstrInfo::isMAI(*MI) &&
1599  MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1600  MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1601  };
1602 
1603  auto IsDGEMMFn = [] (MachineInstr *MI) -> bool {
1604  return isDGEMM(MI->getOpcode());
1605  };
1606 
1607  // This is checked in checkMAIHazards90A()
1608  if (IsMFMAFn(MI))
1609  return 0;
1610 
1611  int WaitStatesNeeded = 0;
1612 
1613  bool IsMemOrExport = SIInstrInfo::isVMEM(*MI) ||
1614  SIInstrInfo::isFLAT(*MI) ||
1615  SIInstrInfo::isDS(*MI) ||
1617  bool IsVALU = SIInstrInfo::isVALU(*MI);
1618 
1619  MachineInstr *MFMA = nullptr;
1620  unsigned Reg;
1621  auto IsDGEMMorXDLWriteFn = [&Reg, &IsMFMAFn, &MFMA, this] (MachineInstr *MI) {
1622  if (!IsMFMAFn(MI) || !TRI.regsOverlap(MI->getOperand(0).getReg(), Reg))
1623  return false;
1624  if (!isDGEMM(MI->getOpcode()) && !isXDL(ST, *MI))
1625  return false;
1626  MFMA = MI;
1627  return true;
1628  };
1629 
1630  MachineInstr *DOT = nullptr;
1631  auto IsDotWriteFn = [&Reg, &DOT, this] (MachineInstr *MI) {
1632  if (!SIInstrInfo::isDOT(*MI) ||
1633  !TRI.regsOverlap(MI->getOperand(0).getReg(), Reg))
1634  return false;
1635  DOT = MI;
1636  return true;
1637  };
1638 
1639  int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
1640  AMDGPU::OpName::src2);
1641 
1642  if (IsMemOrExport || IsVALU) {
1643  const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
1644  const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
1645  const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
1646  const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
1647  const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
1648  const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
1649  const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
1650  const int DotWriteSameDotReadSrcAB = 3;
1651  const int DotWriteDifferentVALURead = 3;
1652  const int MaxWaitStates = 19;
1653 
1654  for (const MachineOperand &Use : MI->explicit_uses()) {
1655  if (!Use.isReg())
1656  continue;
1657  Reg = Use.getReg();
1658 
1659  DOT = nullptr;
1660  int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
1661  MaxWaitStates);
1662  if (DOT) {
1663  int NeedWaitStates = 0;
1664  if (DOT->getOpcode() == MI->getOpcode()) {
1665  if (&Use - &MI->getOperand(0) != SrcCIdx)
1666  NeedWaitStates = DotWriteSameDotReadSrcAB;
1667  } else {
1668  NeedWaitStates = DotWriteDifferentVALURead;
1669  }
1670 
1671  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1672  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1673  }
1674 
1675  MFMA = nullptr;
1676  WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDGEMMorXDLWriteFn,
1677  MaxWaitStates);
1678  if (!MFMA)
1679  continue;
1680 
1681  unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
1682  int NeedWaitStates = MaxWaitStates;
1683  switch (HazardDefLatency) {
1684  case 2:
1685  NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
1686  break;
1687  case 4:
1688  assert(isDGEMM(MFMA->getOpcode()));
1689  NeedWaitStates =
1690  IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
1691  : DMFMA4x4WriteVgprVALUReadWaitStates;
1692  break;
1693  case 8:
1694  NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
1695  break;
1696  case 16: LLVM_FALLTHROUGH;
1697  default:
1698  NeedWaitStates =
1699  isDGEMM(MFMA->getOpcode())
1700  ? IsMemOrExport ? DMFMA16x16WriteVgprMemExpReadWaitStates
1701  : DMFMA16x16WriteVgprVALUReadWaitStates
1702  : SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
1703  break;
1704  }
1705 
1706  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1707  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1708 
1709  if (WaitStatesNeeded == MaxWaitStates)
1710  break;
1711  }
1712  }
1713 
1714  unsigned Opc = MI->getOpcode();
1715  const int DMFMAToFMA64WaitStates = 2;
1716  if ((Opc == AMDGPU::V_FMA_F64_e64 ||
1717  Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
1718  Opc == AMDGPU::V_FMAC_F64_dpp) &&
1719  WaitStatesNeeded < DMFMAToFMA64WaitStates) {
1720  int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
1721  getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
1722  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1723  }
1724 
1725  if (!IsVALU && !IsMemOrExport)
1726  return WaitStatesNeeded;
1727 
1728  for (const MachineOperand &Def : MI->defs()) {
1729  const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
1730  const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
1731  const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
1732  const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
1733  const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
1734  const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
1735  const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
1736  const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
1737  const int DotWriteDifferentVALUWrite = 3;
1738  const int MaxWaitStates = 19;
1739  const int MaxWarWaitStates = 15;
1740 
1741  Reg = Def.getReg();
1742 
1743  DOT = nullptr;
1744  int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
1745  MaxWaitStates);
1746  if (DOT && DOT->getOpcode() != MI->getOpcode())
1747  WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
1748  WaitStatesSinceDef);
1749 
1750  MFMA = nullptr;
1751  WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDGEMMorXDLWriteFn,
1752  MaxWaitStates);
1753  if (MFMA) {
1754  int NeedWaitStates = MaxWaitStates;
1755  switch (TSchedModel.computeInstrLatency(MFMA)) {
1756  case 2:
1757  NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
1758  break;
1759  case 4:
1760  assert(isDGEMM(MFMA->getOpcode()));
1761  NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
1762  break;
1763  case 8:
1764  NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
1765  break;
1766  case 16: LLVM_FALLTHROUGH;
1767  default:
1768  NeedWaitStates = isDGEMM(MFMA->getOpcode())
1769  ? DMFMA16x16WriteVgprVALUWriteWaitStates
1770  : SMFMA32x32WriteVgprVALUWawWaitStates;
1771  break;
1772  }
1773 
1774  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
1775  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1776 
1777  if (WaitStatesNeeded == MaxWaitStates)
1778  break;
1779  }
1780 
1781  auto IsSMFMAReadAsCFn = [&Reg, &IsMFMAFn, &MFMA, this]
1782  (MachineInstr *MI) {
1783  if (!IsMFMAFn(MI) || isDGEMM(MI->getOpcode()) ||
1784  !MI->readsRegister(Reg, &TRI))
1785  return false;
1786 
1787  MachineOperand *SrcC = TII.getNamedOperand(*MI, AMDGPU::OpName::src2);
1788  assert(SrcC);
1789  if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
1790  return false;
1791 
1792  MFMA = MI;
1793  return true;
1794  };
1795 
1796  MFMA = nullptr;
1797  int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn,
1798  MaxWarWaitStates);
1799  if (!MFMA)
1800  continue;
1801 
1802  unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
1803  int NeedWaitStates = MaxWaitStates;
1804  switch (HazardDefLatency) {
1805  case 2: NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
1806  break;
1807  case 8: NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
1808  break;
1809  case 16: LLVM_FALLTHROUGH;
1810  default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
1811  break;
1812  }
1813 
1814  int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
1815  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
1816  }
1817 
1818  return WaitStatesNeeded;
1819 }
1820 
1822  if (!SU->isInstr())
1823  return false;
1824 
1825  MachineInstr *MAI = nullptr;
1826  auto IsMFMAFn = [&MAI] (MachineInstr *MI) {
1827  MAI = nullptr;
1828  if (SIInstrInfo::isMAI(*MI) &&
1829  MI->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1830  MI->getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64)
1831  MAI = MI;
1832  return MAI != nullptr;
1833  };
1834 
1835  MachineInstr *MI = SU->getInstr();
1836  if (IsMFMAFn(MI)) {
1837  int W = getWaitStatesSince(IsMFMAFn, 16);
1838  if (MAI)
1839  return W < (int)TSchedModel.computeInstrLatency(MAI);
1840  }
1841 
1842  return false;
1843 }
i
i
Definition: README.txt:29
ScheduleDAG.h
llvm::MachineInstr::uses
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:655
llvm::ScheduleHazardRecognizer::getMaxLookAhead
unsigned getMaxLookAhead() const
Definition: ScheduleHazardRecognizer.h:43
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:100
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:132
llvm
Definition: AllocatorList.h:23
llvm::MachineInstr::isImplicitDef
bool isImplicitDef() const
Definition: MachineInstr.h:1248
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
llvm::GCNHazardRecognizer::getHazardType
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
Definition: GCNHazardRecognizer.cpp:148
llvm::AMDGPU::getIsaVersion
IsaVersion getIsaVersion(StringRef GPU)
Definition: TargetParser.cpp:186
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
llvm::BitVector::set
BitVector & set()
Definition: BitVector.h:343
llvm::BitVector::none
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:180
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:51
llvm::ScheduleHazardRecognizer::MaxLookAhead
unsigned MaxLookAhead
MaxLookAhead - Indicate the number of cycles in the scoreboard state.
Definition: ScheduleHazardRecognizer.h:31
llvm::SIInstrInfo::isBufferSMRD
bool isBufferSMRD(const MachineInstr &MI) const
Definition: SIInstrInfo.cpp:7289
llvm::SIInstrInfo::getNumWaitStates
static unsigned getNumWaitStates(const MachineInstr &MI)
Return the number of wait states that result from executing this instruction.
Definition: SIInstrInfo.cpp:1639
llvm::RISCVFenceField::W
@ W
Definition: RISCVBaseInfo.h:130
llvm::GCNSubtarget::hasVMEMReadSGPRVALUDefHazard
bool hasVMEMReadSGPRVALUDefHazard() const
A read of an SGPR by a VMEM instruction requires 5 wait states when the SGPR was written by a VALU In...
Definition: GCNSubtarget.h:429
llvm::GCNHazardRecognizer::GCNHazardRecognizer
GCNHazardRecognizer(const MachineFunction &MF)
Definition: GCNHazardRecognizer.cpp:26
llvm::SIInstrInfo::isEXP
static bool isEXP(const MachineInstr &MI)
Definition: SIInstrInfo.h:538
isRWLane
static bool isRWLane(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:70
llvm::GCNSubtarget::getSetRegWaitStates
unsigned getSetRegWaitStates() const
Number of hazard wait states for s_setreg_b32/s_setreg_imm32_b32.
Definition: GCNSubtarget.h:438
llvm::AMDGPU::Hwreg::ID_MASK_
@ ID_MASK_
Definition: SIDefines.h:377
llvm::ScheduleHazardRecognizer::Hazard
@ Hazard
Definition: ScheduleHazardRecognizer.h:39
isXDL
static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:97
addRegUnits
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg)
Definition: GCNHazardRecognizer.cpp:503
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
isSGetReg
static bool isSGetReg(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:55
llvm::GCNSubtarget
Definition: GCNSubtarget.h:38
TargetParser.h
llvm::AMDGPU::getNamedOperandIdx
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, DenseMapInfo< ValueT >, detail::DenseSetPair< ValueT > >, DenseMapInfo< ValueT > >::insert
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1567
IsExpiredFn
function_ref< bool(MachineInstr *, int WaitStates)> IsExpiredFn
Definition: GCNHazardRecognizer.cpp:391
llvm::GCNSubtarget::getRegisterInfo
const SIRegisterInfo * getRegisterInfo() const override
Definition: GCNSubtarget.h:226
llvm::GCNSubtarget::getInstrInfo
const SIInstrInfo * getInstrInfo() const override
Definition: GCNSubtarget.h:214
llvm::TargetSchedModel::init
void init(const TargetSubtargetInfo *TSInfo)
Initialize the machine model for instruction scheduling.
Definition: TargetSchedule.cpp:63
llvm::AMDGPU::IsaVersion
Instruction set architecture version.
Definition: TargetParser.h:102
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:205
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:565
llvm::SIInstrInfo::isMIMG
static bool isMIMG(const MachineInstr &MI)
Definition: SIInstrInfo.h:485
GCNSubtarget.h
getHWReg
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr)
Definition: GCNHazardRecognizer.cpp:141
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:534
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:488
int
Clang compiles this i1 i64 store i64 i64 store i64 i64 store i64 i64 store i64 align Which gets codegen d xmm0 movaps rbp movaps rbp movaps rbp movaps rbp rbp rbp rbp rbp It would be better to have movq s of instead of the movaps s LLVM produces ret int
Definition: README.txt:536
llvm::SIInstrInfo::isMAI
static bool isMAI(const MachineInstr &MI)
Definition: SIInstrInfo.h:636
llvm::GCNSubtarget::hasSMRDReadVALUDefHazard
bool hasSMRDReadVALUDefHazard() const
A read of an SGPR by SMRD instruction requires 4 wait states when the SGPR was written by a VALU inst...
Definition: GCNSubtarget.h:423
isRFE
static bool isRFE(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:74
llvm::MachineInstr::isDebugInstr
bool isDebugInstr() const
Definition: MachineInstr.h:1210
false
Definition: StackSlotColoring.cpp:142
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:129
llvm::GraphProgram::DOT
@ DOT
Definition: GraphWriter.h:53
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:196
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:49
llvm::M0
unsigned M0(unsigned Val)
Definition: VE.h:371
llvm::TargetRegisterInfo::regsOverlap
bool regsOverlap(Register regA, Register regB) const
Returns true if the two registers are equal or alias each other.
Definition: TargetRegisterInfo.h:402
llvm::AMDGPU::decodeWaitcnt
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt)
Decodes Vmcnt, Expcnt and Lgkmcnt from given Waitcnt for given isa Version, and writes decoded values...
Definition: AMDGPUBaseInfo.cpp:909
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:45
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:26
llvm::SIRegisterInfo
Definition: SIRegisterInfo.h:29
llvm::BitVector
Definition: BitVector.h:74
llvm::MCOperandInfo::RegClass
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition: MCInstrDesc.h:90
llvm::RegState::Kill
@ Kill
The last use of a register.
Definition: MachineInstrBuilder.h:49
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
llvm::GCNSubtarget::hasVcmpxExecWARHazard
bool hasVcmpxExecWARHazard() const
Definition: GCNSubtarget.h:935
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLExtras.h:176
isSendMsgTraceDataOrGDS
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII, const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:109
isPermlane
static bool isPermlane(const MachineInstr &MI)
Definition: GCNHazardRecognizer.cpp:135
llvm::DenseSet
Implements a dense probed hash-table based set.
Definition: DenseSet.h:268
llvm::AMDGPU::getRegBitWidth
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
Definition: AMDGPUBaseInfo.cpp:1553
llvm::SIInstrInfo::isDPP
static bool isDPP(const MachineInstr &MI)
Definition: SIInstrInfo.h:604
llvm::ScheduleHazardRecognizer::HazardType
HazardType
Definition: ScheduleHazardRecognizer.h:37
AMDGPUMCTargetDesc.h
llvm::GCNHazardRecognizer::PreEmitNoopsCommon
unsigned PreEmitNoopsCommon(MachineInstr *)
Definition: GCNHazardRecognizer.cpp:269
llvm::InlineAsm::MIOp_FirstOperand
@ MIOp_FirstOperand
Definition: InlineAsm.h:218
llvm::MachineBasicBlock::instr_rend
reverse_instr_iterator instr_rend()
Definition: MachineBasicBlock.h:258
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:318
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
getWaitStatesSince
static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard, MachineBasicBlock *MBB, MachineBasicBlock::reverse_instr_iterator I, int WaitStates, IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock * > &Visited)
Definition: GCNHazardRecognizer.cpp:396
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::SIInstrFlags::DPP
@ DPP
Definition: SIDefines.h:42
llvm::GCNSubtarget::hasVMEMtoScalarWriteHazard
bool hasVMEMtoScalarWriteHazard() const
Definition: GCNSubtarget.h:919
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::SIInstrInfo::isDS
static bool isDS(const MachineInstr &MI)
Definition: SIInstrInfo.h:475
llvm::SIInstrInfo::isSMRD
static bool isSMRD(const MachineInstr &MI)
Definition: SIInstrInfo.h:465
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::SUnit::getInstr
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:373
addRegsToSet
static void addRegsToSet(const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &Set)
Definition: GCNHazardRecognizer.cpp:509
llvm::GCNHazardRecognizer::PreEmitNoops
unsigned PreEmitNoops(MachineInstr *) override
This overload will be used when the hazard recognizer is being used by a non-scheduling pass,...
Definition: GCNHazardRecognizer.cpp:260
llvm::GCNHazardRecognizer::IsHazardFn
function_ref< bool(MachineInstr *)> IsHazardFn
Definition: GCNHazardRecognizer.h:35
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::SIInstrFlags::VALU
@ VALU
Definition: SIDefines.h:22
llvm::SIInstrInfo::isFPAtomic
static bool isFPAtomic(const MachineInstr &MI)
Definition: SIInstrInfo.h:718
llvm::GCNSubtarget::hasReadM0MovRelInterpHazard
bool hasReadM0MovRelInterpHazard() const
Definition: GCNSubtarget.h:906
llvm::MCInstrDesc::OpInfo
const MCOperandInfo * OpInfo
Definition: MCInstrDesc.h:207
llvm::GCNSubtarget::hasRFEHazards
bool hasRFEHazards() const
Definition: GCNSubtarget.h:433
llvm::AMDGPU::Waitcnt::LgkmCnt
unsigned LgkmCnt
Definition: AMDGPUBaseInfo.h:464
llvm::GCNSubtarget::hasGFX90AInsts
bool hasGFX90AInsts() const
Definition: GCNSubtarget.h:949
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:98
llvm::AMDGPU::getMIMGInfo
const LLVM_READONLY MIMGInfo * getMIMGInfo(unsigned Opc)
llvm::GCNSubtarget::hasNoDataDepHazard
bool hasNoDataDepHazard() const
Definition: GCNSubtarget.h:736
llvm::GCNHazardRecognizer::EmitInstruction
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
Definition: GCNHazardRecognizer.cpp:43
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:357
llvm::MachineBasicBlock::predecessors
iterator_range< pred_iterator > predecessors()
Definition: MachineBasicBlock.h:349
llvm::GCNHazardRecognizer::EmitNoop
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
Definition: GCNHazardRecognizer.cpp:336
breaksVMEMSoftClause
static bool breaksVMEMSoftClause(MachineInstr *MI)
Definition: GCNHazardRecognizer.cpp:528
llvm::GCNSubtarget::hasLdsBranchVmemWARHazard
bool hasLdsBranchVmemWARHazard() const
Definition: GCNSubtarget.h:939
llvm::SIInstrInfo::isSALU
static bool isSALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:337
llvm::MachineBasicBlock::instr_end
instr_iterator instr_end()
Definition: MachineBasicBlock.h:254
llvm::MachineFunction
Definition: MachineFunction.h:227
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
insertNoopsInBundle
static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity)
Definition: GCNHazardRecognizer.cpp:224
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
isSSetReg
static bool isSSetReg(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:59
llvm::SIInstrInfo::isMUBUF
static bool isMUBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:449
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:286
llvm::SIInstrInfo::isDOT
static bool isDOT(const MachineInstr &MI)
Definition: SIInstrInfo.h:644
LLVM_FALLTHROUGH
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:281
llvm::GCNHazardRecognizer::Reset
void Reset() override
Reset - This callback is invoked when a new block of instructions is about to be schedule.
Definition: GCNHazardRecognizer.cpp:39
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
isDGEMM
static bool isDGEMM(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:90
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::ScheduleHazardRecognizer::NoopHazard
@ NoopHazard
Definition: ScheduleHazardRecognizer.h:40
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::GCNSubtarget::hasNSAtoVMEMBug
bool hasNSAtoVMEMBug() const
Definition: GCNSubtarget.h:943
llvm::AMDGPU::Waitcnt
Represents the counter values to wait for in an s_waitcnt instruction.
Definition: AMDGPUBaseInfo.h:461
llvm::GCNHazardRecognizer::RecedeCycle
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
Definition: GCNHazardRecognizer.cpp:383
llvm::ScheduleHazardRecognizer::NoHazard
@ NoHazard
Definition: ScheduleHazardRecognizer.h:38
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:314
llvm::SIInstrInfo::isSegmentSpecificFLAT
static bool isSegmentSpecificFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:507
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::SIInstrFlags::SMRD
@ SMRD
Definition: SIDefines.h:48
llvm::MachineInstr::mayStore
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:1007
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:53
llvm::SIInstrInfo::isVMEM
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:353
llvm::AMDGPU::MIMGInfo
Definition: AMDGPUBaseInfo.h:350
llvm::SIInstrInfo::isMTBUF
static bool isMTBUF(const MachineInstr &MI)
Definition: SIInstrInfo.h:457
llvm::MachineRegisterInfo::isPhysRegUsed
bool isPhysRegUsed(MCRegister PhysReg) const
Return true if the specified register is modified or read in this function.
Definition: MachineRegisterInfo.cpp:585
llvm::MachineInstr::isBundle
bool isBundle() const
Definition: MachineInstr.h:1275
llvm::SIInstrInfo
Definition: SIInstrInfo.h:38
GCNHazardRecognizer.h
llvm::GCNHazardRecognizer::AdvanceCycle
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
Definition: GCNHazardRecognizer.cpp:340
llvm::MCRegUnitIterator
Definition: MCRegisterInfo.h:677
llvm::GCNSubtarget::isXNACKEnabled
bool isXNACKEnabled() const
Definition: GCNSubtarget.h:528
isDivFMas
static bool isDivFMas(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:51
llvm::SUnit::isInstr
bool isInstr() const
Returns true if this SUnit refers to a machine instruction as opposed to an SDNode.
Definition: ScheduleDAG.h:362
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:329
llvm::SIInstrInfo::isVALU
static bool isVALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:345
llvm::MCRegisterInfo::DiffListIterator::isValid
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
Definition: MCRegisterInfo.h:224
llvm::MachineInstr::getNumOperands
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:481
llvm::GCNHazardRecognizer::ShouldPreferAnother
bool ShouldPreferAnother(SUnit *SU) override
ShouldPreferAnother - This callback may be invoked if getHazardType returns NoHazard.
Definition: GCNHazardRecognizer.cpp:1821
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:350
llvm::iterator_range
A range adaptor for a pair of iterators.
Definition: iterator_range.h:30
llvm::GCNSubtarget::hasReadM0SendMsgHazard
bool hasReadM0SendMsgHazard() const
Definition: GCNSubtarget.h:910
llvm::BitVector::anyCommon
bool anyCommon(const BitVector &RHS) const
Test if any common bits are set.
Definition: BitVector.h:469
llvm::SUnit
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:242
llvm::GCNSubtarget::hasSMEMtoVectorWriteHazard
bool hasSMEMtoVectorWriteHazard() const
Definition: GCNSubtarget.h:923
llvm::AMDGPU::Hwreg::ID_TRAPSTS
@ ID_TRAPSTS
Definition: SIDefines.h:356
llvm::MachineInstr::isKill
bool isKill() const
Definition: MachineInstr.h:1247
llvm::SIInstrInfo::isFLAT
static bool isFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:501
breaksSMEMSoftClause
static bool breaksSMEMSoftClause(MachineInstr *MI)
Definition: GCNHazardRecognizer.cpp:524
llvm::GCNSubtarget::has12DWordStoreHazard
bool has12DWordStoreHazard() const
Definition: GCNSubtarget.h:897
MachineFunction.h
llvm::GCNSubtarget::hasVcmpxPermlaneHazard
bool hasVcmpxPermlaneHazard() const
Definition: GCNSubtarget.h:915
isSMovRel
static bool isSMovRel(unsigned Opcode)
Definition: GCNHazardRecognizer.cpp:78
llvm::SIInstrInfo::isVINTRP
static bool isVINTRP(const MachineInstr &MI)
Definition: SIInstrInfo.h:628
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:44
llvm::GCNSubtarget::hasMAIInsts
bool hasMAIInsts() const
Definition: GCNSubtarget.h:696
llvm::MCRegister
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:22