GCNHazardRecognizer.cpp

Go to the documentation of this file.

//===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements hazard recognizers for scheduling on GCN processors.
//
//===----------------------------------------------------------------------===//
 
#include "GCNHazardRecognizer.h"
#include "AMDGPUTargetMachine.h"
#include "AMDGPUWaitcntUtils.h"
#include "GCNSubtarget.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/Support/Debug.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
 
using namespace llvm;
 
#define DEBUG_TYPE "gcn-hazard-recognizer"
// Opt-in debug type for the per-candidate co-execution slot traces, which are
// far too noisy for the normal debug output. Pass both types to get everything.
#define DEBUG_TYPE_VERBOSE "gcn-hazard-recognizer-verbose"
 
STATISTIC(NumWMMANopsHoisted,
          "Number of WMMA hazard V_NOPs hoisted from loops");
STATISTIC(NumWMMAHoistingBailed,
          "Number of WMMA hazards where V_NOP hoisting was not possible");
 
namespace {
 
struct MFMAPaddingRatioParser : public cl::parser<unsigned> {
  MFMAPaddingRatioParser(cl::Option &O) : cl::parser<unsigned>(O) {}
 
  bool parse(cl::Option &O, StringRef ArgName, StringRef Arg, unsigned &Value) {
    if (Arg.getAsInteger(0, Value))
      return O.error("'" + Arg + "' value invalid for uint argument!");
 
    if (Value > 100)
      return O.error("'" + Arg + "' value must be in the range [0, 100]!");
 
    return false;
  }
};
 
} // end anonymous namespace
 
static cl::opt<unsigned, false, MFMAPaddingRatioParser>
    MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden,
                     cl::desc("Fill a percentage of the latency between "
                              "neighboring MFMA with s_nops."));
 
// This is intended for debugging purposes only.
static cl::opt<unsigned>
    NopPadding("amdgpu-snop-padding", cl::init(0), cl::Hidden,
               cl::desc("Insert a s_nop x before every instruction"));
 
static cl::opt<bool> EnableWMMAVnopHoisting(
    "amdgpu-wmma-vnop-hoisting", cl::init(true), cl::Hidden,
    cl::desc("Hoist WMMA hazard V_NOPs from loops to preheaders"));
 
//===----------------------------------------------------------------------===//
// Hazard Recognizer Implementation
//===----------------------------------------------------------------------===//
 
static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
                                                 const GCNSubtarget &ST);
 
GCNHazardRecognizer::GCNHazardRecognizer(
    const MachineFunction &MF, GCNHazardRecognizer::OperatingMode Mode,
    MachineLoopInfo *MLI)
    : Mode(Mode), CurrCycleInstr(nullptr), MF(MF),
      ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
      TRI(TII.getRegisterInfo()), TSchedModel(TII.getSchedModel()), MLI(MLI),
      ClauseUses(TRI.getNumRegUnits()), ClauseDefs(TRI.getNumRegUnits()) {
  MaxLookAhead = MF.getRegInfo().isPhysRegUsed(AMDGPU::AGPR0) ? 19 : 5;
  RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup(MF, ST);
  LLVM_DEBUG({
    if (isPreRA())
      dbgs() << "  PreRA hazard recognizer: " << MF.getName() << "\n";
  });
}
 
GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF,
                                         MachineLoopInfo *MLI)
    : GCNHazardRecognizer(MF, OperatingMode::PostRA, MLI) {}
 
GCNHazardRecognizer::~GCNHazardRecognizer() {
  // Dump any active co-execution window that did not complete naturally
  // (e.g. region ended before the window expired).
  LLVM_DEBUG({
    if (CurrentCoExecStage.has_value()) {
      unsigned Stage = *CurrentCoExecStage;
      if (Stage < AMDGPU::MaxCoExecStages)
        CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
      dbgs() << "  CoExec window ended at stage " << Stage << ":\n";
      dumpCoExecWindow();
    }
  });
}
 
void GCNHazardRecognizer::Reset() {
  EmittedInstrs.clear();
  EmittedVALUInstrs.clear();
  HasPendingWMMACoexecHazard = false;
  if (isSchedulerMode())
    schedulerReset();
}
 
void GCNHazardRecognizer::schedulerReset() {
  LLVM_DEBUG({
    if (CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
        CyclesUntilVALU > 0)
      dbgs() << "  Scheduler Reset: clearing co-exec window, TRANS="
             << CyclesUntilTRANS << ", VALU=" << CyclesUntilVALU << "\n";
  });
  CurrentCoExecStage = std::nullopt;
  CoExecWindowStartCycle = 0;
  CyclesUntilTRANS = 0;
  CyclesUntilVALU = 0;
  ActiveCoExecInfo = AMDGPU::CoExecInfo();
  CoExecWindowLog.fill('.');
}
 
void GCNHazardRecognizer::dumpCoExecWindow() const {
  unsigned W = ActiveCoExecInfo.TotalWindow;
  if (W == 0)
    return;
 
  // Print the stage numbers row.
  dbgs() << "    Stages:    ";
  for (unsigned I = 0; I < W; ++I)
    dbgs() << I % 10 << ' ';
  dbgs() << '\n';
 
  // Print the pattern row.
  dbgs() << "    Slots:     ";
  for (unsigned I = 0; I < W; ++I)
    dbgs() << ActiveCoExecInfo.Pattern[I] << ' ';
  dbgs() << '\n';
 
  // Print the scheduled row.
  dbgs() << "    Scheduled: ";
  for (unsigned I = 0; I < W; ++I)
    dbgs() << CoExecWindowLog[I] << ' ';
  dbgs() << '\n';
}
 
void GCNHazardRecognizer::schedulerAdvanceCycle() {
  // Record what happened at the current stage of the co-exec window.
  if (CurrentCoExecStage.has_value()) {
    unsigned Stage = *CurrentCoExecStage;
    if (Stage < AMDGPU::MaxCoExecStages) {
      if (CurrCycleInstr)
        CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
      else
        CoExecWindowLog[Stage] = '-';
    }
  }
 
  LLVM_DEBUG({
    bool HasState = CurrentCoExecStage.has_value() || CyclesUntilTRANS > 0 ||
                    CyclesUntilVALU > 0;
    if (HasState) {
      dbgs() << "  Scheduler AdvanceCycle:";
      if (CurrentCoExecStage.has_value()) {
        unsigned Stage = *CurrentCoExecStage;
        unsigned Next = Stage + 1;
        if (Next >= ActiveCoExecInfo.TotalWindow)
          dbgs() << " stage " << Stage << "->expired";
        else
          dbgs() << " stage " << Stage << "->" << Next;
      }
      if (CyclesUntilTRANS > 0)
        dbgs() << " TRANS=" << CyclesUntilTRANS << "->"
               << (CyclesUntilTRANS - 1);
      if (CyclesUntilVALU > 0)
        dbgs() << " VALU=" << CyclesUntilVALU << "->" << (CyclesUntilVALU - 1);
      dbgs() << "\n";
    }
  });
 
  // Decrement hazard counters.
  if (CyclesUntilTRANS > 0)
    --CyclesUntilTRANS;
  if (CyclesUntilVALU > 0)
    --CyclesUntilVALU;
 
  // Advance WMMA co-execution window.
  if (CurrentCoExecStage.has_value()) {
    unsigned Stage = *CurrentCoExecStage + 1;
    if (Stage >= ActiveCoExecInfo.TotalWindow) {
      // Window expired.
      LLVM_DEBUG({
        dbgs() << "  CoExec window complete:\n";
        dumpCoExecWindow();
      });
      CurrentCoExecStage = std::nullopt;
    } else {
      CurrentCoExecStage = Stage;
    }
  }
}
 
bool GCNHazardRecognizer::hasCoExecWindowModel() const {
  // The co-execution slot patterns returned by getCoExecInfo() are derived from
  // gfx1250 timings, so the window model is restricted to gfx1250 for now.
  // gfx1251 and gfx12.5-generic report the same co-execution hazard features
  // but have different WMMA latencies, so they need their own slot patterns
  // before they can be modeled here.
  if (ST.hasWMMACoexecutionHazards() && ST.hasTransCoexecutionHazard() &&
      AMDGPU::isGFX1250(ST))
    return true;
 
  if (ST.hasGFX950Insts() &&
      AMDGPU::getSchedStrategy(MF.getFunction()) == "coexec")
    return true;
 
  return false;
}
 
void GCNHazardRecognizer::updateWMMAWindowState(const MachineInstr &MI) {
  if (!hasCoExecWindowModel())
    return;
 
  if (!SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
      !SIInstrInfo::isMFMA(MI))
    return;
 
  // If a previous window was still active, dump it before starting a new one.
  // Record the current stage (filled by this new WMMA) before dumping.
  LLVM_DEBUG({
    if (CurrentCoExecStage.has_value()) {
      unsigned Stage = *CurrentCoExecStage;
      if (Stage < AMDGPU::MaxCoExecStages)
        CoExecWindowLog[Stage] = ActiveCoExecInfo.Pattern[Stage];
      dbgs() << "  CoExec window interrupted at stage " << Stage << ":\n";
      dumpCoExecWindow();
    }
  });
 
  // Start a new co-execution window.
  ActiveCoExecInfo = AMDGPU::getCoExecInfo(MI, TII);
  CurrentCoExecStage = 0;
  CoExecWindowLog.fill('.');
 
  LLVM_DEBUG(dbgs() << "  WMMA window started: " << ActiveCoExecInfo.Pattern
                    << " (window=" << ActiveCoExecInfo.TotalWindow << ")\n"
                    << "    " << MI);
}
 
void GCNHazardRecognizer::updateTRANSState(const MachineInstr &MI) {
  if (!hasCoExecWindowModel())
    return;
  if (!SIInstrInfo::isTRANS(MI))
    return;
 
  // Back-to-back TRANS instructions have a 1-cycle hazard.
  // This is checked via checkTRANSHazard() and does not create a co-exec
  // window. The TRANS shadow slot allows anything except TRANS and
  // multi-cycle VALU.
  // Set to 2: bumpCycle advances to the next pick's cycle (decrementing
  // by 1 via AdvanceCycle) before the next instruction's hazard check, so
  // the counter is observed at 1 there. That 1-cycle stall lets the
  // strategy pick a non-TRANS, non-multi-cycle-VALU candidate to fill the
  // shadow slot.
  CyclesUntilTRANS = 2;
  LLVM_DEBUG(dbgs() << "    TRANS hazard set: CyclesUntilTRANS=2\n");
}
 
void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) {
  if (!hasCoExecWindowModel())
    return;
  // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for repeat rate cycles.
  if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
    return;
 
  // Skip WMMA, MFMA, and TRANS - they have their own tracking.
  if (SIInstrInfo::isWMMA(MI) || SIInstrInfo::isSWMMAC(MI) ||
      SIInstrInfo::isMFMA(MI) || SIInstrInfo::isTRANS(MI))
    return;
 
  unsigned RepeatRate = TII.getRepeatRate(MI);
  if (RepeatRate > 1) {
    // bumpCycle's AdvanceCycle decrements once before the next pick's
    // hazard check (same convention as CyclesUntilTRANS), so to expose
    // RepeatRate-1 cycles of shadow we must seed with RepeatRate.
    CyclesUntilVALU = RepeatRate;
    LLVM_DEBUG(dbgs() << "    Multi-cycle VALU: repeat=" << RepeatRate
                      << ", CyclesUntilVALU=" << CyclesUntilVALU << "\n");
  }
}
 
AMDGPU::CoExecMaskT
GCNHazardRecognizer::getCoExecMaskForMI(const MachineInstr &MI,
                                        const SIInstrInfo &TII) {
  return AMDGPU::getCoExecMask(AMDGPU::classifyFlavor(MI, TII));
}
 
unsigned GCNHazardRecognizer::checkTRANSHazard(const MachineInstr &MI) const {
  if (!CyclesUntilTRANS)
    return 0;
 
  // Only TRANS and multi-cycle VALU are blocked by the TRANS shadow.
  if (SIInstrInfo::isTRANS(MI))
    return CyclesUntilTRANS;
 
  if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
      !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
      TII.getRepeatRate(MI) > 1)
    return CyclesUntilTRANS;
 
  return 0;
}
 
unsigned
GCNHazardRecognizer::checkMultiCycleVALUHazard(const MachineInstr &MI) const {
  if (!CyclesUntilVALU)
    return 0;
 
  // Multi-cycle VALU blocks anything on the VALU pipe - VALU, WMMA, SWMMAC,
  // and TRANS - for RepeatRate-1 cycles. Only off-pipe instructions (MEM,
  // SALU, control) can fill the shadow.
  if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
      !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) &&
      !SIInstrInfo::isTRANS(MI))
    return 0;
 
  return CyclesUntilVALU;
}
 
unsigned
GCNHazardRecognizer::checkWMMACoexecSlot(const MachineInstr &MI) const {
  // No hazard if not in a WMMA window.
  if (!CurrentCoExecStage.has_value())
    return 0;
 
  unsigned Stage = *CurrentCoExecStage;
  AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
  unsigned StallCycles = ActiveCoExecInfo.getStallCycles(InstMask, Stage);
 
  // No stall required if the instruction can co-execute at the current stage.
  if (StallCycles == 0)
    return 0;
 
  // Stall for the required number of cycles until the next allowed stage.
  unsigned NextStage = Stage + StallCycles;
  if (NextStage < ActiveCoExecInfo.TotalWindow) {
    DEBUG_WITH_TYPE(
        DEBUG_TYPE_VERBOSE,
        dbgs() << "    CoExec stall: stage=" << Stage << "("
               << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
               << ") mask=" << AMDGPU::getCoExecMaskName(InstMask)
               << " -> stall " << StallCycles << " (next allowed=" << NextStage
               << ")\n"
               << "      " << MI);
    return StallCycles;
  }
 
  // No compatible slot in window - stall until window ends.
  DEBUG_WITH_TYPE(
      DEBUG_TYPE_VERBOSE,
      dbgs() << "    CoExec stall: stage=" << Stage << "("
             << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
             << ") mask=" << AMDGPU::getCoExecMaskName(InstMask) << " -> stall "
             << StallCycles << " (window ends)\n"
             << "      " << MI);
  return StallCycles;
}
 
unsigned
GCNHazardRecognizer::checkMultiShadowHazard(const MachineInstr &MI) const {
  // This models a VALU caught in both a WMMA and a TRANS shadow.
  if (!hasCoExecWindowModel())
    return 0;
 
  // No hazard if not in a WMMA window.
  if (!CurrentCoExecStage.has_value())
    return 0;
 
  if (!CyclesUntilTRANS)
    return 0;
 
  if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
      SIInstrInfo::isLDSDMA(MI))
    return 0;
 
  // We have a VALU instruction that is under both a TRANS and WMMA shadow.
  // We need to wait for at least one to clear.
 
  unsigned LookAheadStage = *CurrentCoExecStage + CyclesUntilTRANS;
  AMDGPU::CoExecMaskT InstMask = getCoExecMaskForMI(MI, TII);
  return CyclesUntilTRANS +
         ActiveCoExecInfo.getStallCycles(InstMask, LookAheadStage);
}
 
void GCNHazardRecognizer::schedulerEmitInstruction(MachineInstr *MI) {
  LLVM_DEBUG({
    bool InWindow = CurrentCoExecStage.has_value();
    bool HasActiveState =
        InWindow || CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
    if (HasActiveState) {
      if (InWindow) {
        unsigned Stage = *CurrentCoExecStage;
        dbgs() << "  Stage " << Stage << "("
               << AMDGPU::getStageTypeName(ActiveCoExecInfo.getType(Stage))
               << ") Emit ["
               << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
               << "]: " << *MI;
      } else {
        dbgs() << "  Emit ["
               << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
               << "]: " << *MI;
      }
    }
  });
  DEBUG_WITH_TYPE(DEBUG_TYPE_VERBOSE, {
    bool HasActiveState = CurrentCoExecStage.has_value() ||
                          CyclesUntilTRANS > 0 || CyclesUntilVALU > 0;
    if (!HasActiveState)
      dbgs() << "  Emit ["
             << AMDGPU::getCoExecMaskName(getCoExecMaskForMI(*MI, TII))
             << "]: " << *MI;
  });
  updateWMMAWindowState(*MI);
  updateTRANSState(*MI);
  updateMultiCycleVALUState(*MI);
}
 
void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
  EmitInstruction(SU->getInstr());
}
 
void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) {
  CurrCycleInstr = MI;
  if (isSchedulerMode())
    schedulerEmitInstruction(MI);
}
 
static bool isDivFMas(unsigned Opcode) {
  return Opcode == AMDGPU::V_DIV_FMAS_F32_e64 || Opcode == AMDGPU::V_DIV_FMAS_F64_e64;
}
 
static bool isSGetReg(unsigned Opcode) {
  return Opcode == AMDGPU::S_GETREG_B32 || Opcode == AMDGPU::S_GETREG_B32_const;
}
 
static bool isSSetReg(unsigned Opcode) {
  switch (Opcode) {
  case AMDGPU::S_SETREG_B32:
  case AMDGPU::S_SETREG_B32_mode:
  case AMDGPU::S_SETREG_IMM32_B32:
  case AMDGPU::S_SETREG_IMM32_B32_mode:
    return true;
  }
  return false;
}
 
static bool isRWLane(unsigned Opcode) {
  return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
}
 
static bool isRFE(unsigned Opcode) {
  return Opcode == AMDGPU::S_RFE_B64;
}
 
static bool isSMovRel(unsigned Opcode) {
  switch (Opcode) {
  case AMDGPU::S_MOVRELS_B32:
  case AMDGPU::S_MOVRELS_B64:
  case AMDGPU::S_MOVRELD_B32:
  case AMDGPU::S_MOVRELD_B64:
    return true;
  default:
    return false;
  }
}
 
static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
                                    const MachineInstr &MI) {
  if (TII.isAlwaysGDS(MI.getOpcode()))
    return true;
 
  switch (MI.getOpcode()) {
  case AMDGPU::S_SENDMSG:
  case AMDGPU::S_SENDMSGHALT:
  case AMDGPU::S_TTRACEDATA:
    return true;
  // These DS opcodes don't support GDS.
  case AMDGPU::DS_NOP:
  case AMDGPU::DS_PERMUTE_B32:
  case AMDGPU::DS_BPERMUTE_B32:
    return false;
  default:
    if (TII.isDS(MI.getOpcode())) {
      int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
                                           AMDGPU::OpName::gds);
      if (MI.getOperand(GDS).getImm())
        return true;
    }
    return false;
  }
}
 
static bool isPermlane(const MachineInstr &MI) {
  unsigned Opcode = MI.getOpcode();
  return Opcode == AMDGPU::V_PERMLANE16_B32_e64 ||
         Opcode == AMDGPU::V_PERMLANE64_B32 ||
         Opcode == AMDGPU::V_PERMLANEX16_B32_e64 ||
         Opcode == AMDGPU::V_PERMLANE16_VAR_B32_e64 ||
         Opcode == AMDGPU::V_PERMLANEX16_VAR_B32_e64 ||
         Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e32 ||
         Opcode == AMDGPU::V_PERMLANE16_SWAP_B32_e64 ||
         Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e32 ||
         Opcode == AMDGPU::V_PERMLANE32_SWAP_B32_e64 ||
         Opcode == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
         Opcode == AMDGPU::V_PERMLANE_UP_B32_e64 ||
         Opcode == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
         Opcode == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
         Opcode == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64;
}
 
static bool isLdsDma(const MachineInstr &MI) {
  return SIInstrInfo::isLDSDMA(MI);
}
 
static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
  const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
                                                     AMDGPU::OpName::simm16);
  return std::get<0>(AMDGPU::Hwreg::HwregEncoding::decode(RegOp->getImm()));
}
 
ScheduleHazardRecognizer::HazardType
GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
  MachineInstr *MI = SU->getInstr();
  // If we are not in "HazardRecognizerMode" and therefore not being run from
  // the scheduler, track possible stalls from hazards but don't insert noops.
  auto HazardType = isHazardRecognizerMode() ? NoopHazard : Hazard;
 
  if (MI->isBundle())
   return NoHazard;
 
  // Check co-execution slot hazards and pipeline stalls in scheduler modes.
  if (isSchedulerMode()) {
    if (checkMultiShadowHazard(*MI) > 0)
      return Hazard;
    if (checkWMMACoexecSlot(*MI) > 0)
      return Hazard;
    if (checkTRANSHazard(*MI) > 0)
      return Hazard;
    if (checkMultiCycleVALUHazard(*MI) > 0)
      return Hazard;
    // The remaining checks are all defined by register dependences.
    if (!hasPhysRegs())
      return NoHazard;
  }
 
  if (SIInstrInfo::isSMRD(*MI) && checkSMRDHazards(MI) > 0)
    return HazardType;
 
  if (ST.hasNSAtoVMEMBug() && checkNSAtoVMEMHazard(MI) > 0)
    return HazardType;
 
  if (checkFPAtomicToDenormModeHazard(MI) > 0)
    return HazardType;
 
  // Hazards which cannot be mitigated with S_NOPs.
  if (!isHazardRecognizerMode()) {
    if (checkWMMACoexecutionHazards(MI) > 0) {
      HasPendingWMMACoexecHazard = true;
      return Hazard;
    }
  }
 
  if (ST.hasNoDataDepHazard())
    return NoHazard;
 
  if (SIInstrInfo::isVMEM(*MI) && checkVMEMHazards(MI) > 0)
    return HazardType;
 
  if (SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) &&
      checkVALUHazards(MI) > 0)
    return HazardType;
 
  if (SIInstrInfo::isDPP(*MI) && checkDPPHazards(MI) > 0)
    return HazardType;
 
  if (isDivFMas(MI->getOpcode()) && checkDivFMasHazards(MI) > 0)
    return HazardType;
 
  if (isRWLane(MI->getOpcode()) && checkRWLaneHazards(MI) > 0)
    return HazardType;
 
  if ((SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) ||
       SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI) ||
       SIInstrInfo::isEXP(*MI)) &&
      checkMAIVALUHazards(MI) > 0)
    return HazardType;
 
  if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
    return HazardType;
 
  if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
    return HazardType;
 
  if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
    return HazardType;
 
  if (((ST.hasReadM0MovRelInterpHazard() &&
        (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) ||
         MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
         MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
       (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) ||
       (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
       (ST.hasReadM0LdsDirectHazard() &&
        MI->readsRegister(AMDGPU::LDS_DIRECT, /*TRI=*/nullptr))) &&
      checkReadM0Hazards(MI) > 0)
    return HazardType;
 
  if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0)
    return HazardType;
 
  if ((SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI)) &&
      checkMAILdStHazards(MI) > 0)
    return HazardType;
 
  if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0)
    return HazardType;
 
  return NoHazard;
}
 
static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
                                unsigned Quantity) {
  while (Quantity > 0) {
    unsigned Arg = std::min(Quantity, 8u);
    Quantity -= Arg;
    BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::S_NOP))
        .addImm(Arg - 1);
  }
}
 
unsigned
GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const {
  const MCSchedClassDesc *SC = TSchedModel.resolveSchedClass(&MI);
  assert(TSchedModel.getWriteProcResBegin(SC) !=
         TSchedModel.getWriteProcResEnd(SC));
  return TSchedModel.getWriteProcResBegin(SC)->ReleaseAtCycle;
}
 
void GCNHazardRecognizer::processBundle() {
  MachineBasicBlock::instr_iterator MI = std::next(CurrCycleInstr->getIterator());
  MachineBasicBlock::instr_iterator E = CurrCycleInstr->getParent()->instr_end();
  // Check bundled MachineInstr's for hazards.
  for (; MI != E && MI->isInsideBundle(); ++MI) {
    CurrCycleInstr = &*MI;
    unsigned WaitStates = PreEmitNoopsCommon(CurrCycleInstr);
 
    if (isHazardRecognizerMode()) {
      fixHazards(CurrCycleInstr);
 
      insertNoopsInBundle(CurrCycleInstr, TII, WaitStates);
    }
 
    // It’s unnecessary to track more than MaxLookAhead instructions. Since we
    // include the bundled MI directly after, only add a maximum of
    // (MaxLookAhead - 1) noops to EmittedInstrs.
    for (unsigned i = 0, e = std::min(WaitStates, MaxLookAhead - 1); i < e; ++i)
      EmittedInstrs.push_front(nullptr);
 
    EmittedInstrs.push_front(CurrCycleInstr);
    EmittedInstrs.resize(MaxLookAhead);
  }
  CurrCycleInstr = nullptr;
}
 
void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) {
  assert(isHazardRecognizerMode());
 
  unsigned NumPreNoops = PreEmitNoops(MI);
  EmitNoops(NumPreNoops);
  if (MI->isInsideBundle())
    insertNoopsInBundle(MI, TII, NumPreNoops);
  else
    TII.insertNoops(*MI->getParent(), MachineBasicBlock::iterator(MI),
                    NumPreNoops);
  EmitInstruction(MI);
  AdvanceCycle();
}
 
unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
  assert(isHazardRecognizerMode());
  CurrCycleInstr = MI;
  unsigned W = PreEmitNoopsCommon(MI);
  fixHazards(MI);
  CurrCycleInstr = nullptr;
  return std::max(W, NopPadding.getValue());
}
 
unsigned GCNHazardRecognizer::getHazardWaitStates(MachineInstr *MI) const {
  unsigned W = 0;
 
  // Check co-execution slot hazards and pipeline stalls in scheduler modes.
  if (isSchedulerMode()) {
    W = checkWMMACoexecSlot(*MI);
    W = std::max(W, checkTRANSHazard(*MI));
    W = std::max(W, checkMultiCycleVALUHazard(*MI));
    W = std::max(W, checkMultiShadowHazard(*MI));
    // The remaining checks are all defined by register dependences.
    if (!hasPhysRegs())
      return W;
  }
 
  return std::max(W, PreEmitNoopsCommon(MI));
}
 
unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
  if (MI->isBundle())
    return 0;
 
  int WaitStates = 0;
 
  if (SIInstrInfo::isSMRD(*MI))
    return std::max(WaitStates, checkSMRDHazards(MI));
 
  if (ST.hasNSAtoVMEMBug())
    WaitStates = std::max(WaitStates, checkNSAtoVMEMHazard(MI));
 
  WaitStates = std::max(WaitStates, checkFPAtomicToDenormModeHazard(MI));
 
  if (ST.hasNoDataDepHazard())
    return WaitStates;
 
  if (SIInstrInfo::isVMEM(*MI))
    WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
 
  if (SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
    WaitStates = std::max(WaitStates, checkVALUHazards(MI));
 
  if (SIInstrInfo::isDPP(*MI))
    WaitStates = std::max(WaitStates, checkDPPHazards(MI));
 
  if (isDivFMas(MI->getOpcode()))
    WaitStates = std::max(WaitStates, checkDivFMasHazards(MI));
 
  if (isRWLane(MI->getOpcode()))
    WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
 
  if ((SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) ||
       SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI) ||
       SIInstrInfo::isEXP(*MI)) &&
      checkMAIVALUHazards(MI) > 0)
    WaitStates = std::max(WaitStates, checkMAIVALUHazards(MI));
 
  if (MI->isInlineAsm())
    return std::max(WaitStates, checkInlineAsmHazards(MI));
 
  if (isSGetReg(MI->getOpcode()))
    return std::max(WaitStates, checkGetRegHazards(MI));
 
  if (isSSetReg(MI->getOpcode()))
    return std::max(WaitStates, checkSetRegHazards(MI));
 
  if (isRFE(MI->getOpcode()))
    return std::max(WaitStates, checkRFEHazards(MI));
 
  if ((ST.hasReadM0MovRelInterpHazard() &&
       (TII.isVINTRP(*MI) || isSMovRel(MI->getOpcode()) ||
        MI->getOpcode() == AMDGPU::DS_WRITE_ADDTID_B32 ||
        MI->getOpcode() == AMDGPU::DS_READ_ADDTID_B32)) ||
      (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI)) ||
      (ST.hasReadM0LdsDmaHazard() && isLdsDma(*MI)) ||
      (ST.hasReadM0LdsDirectHazard() &&
       MI->readsRegister(AMDGPU::LDS_DIRECT, /*TRI=*/nullptr)))
    return std::max(WaitStates, checkReadM0Hazards(MI));
 
  if (SIInstrInfo::isMAI(*MI))
    return std::max(WaitStates, checkMAIHazards(MI));
 
  if (SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI))
    return std::max(WaitStates, checkMAILdStHazards(MI));
 
  if (ST.hasGFX950Insts() && isPermlane(*MI))
    return std::max(WaitStates, checkPermlaneHazards(MI));
 
  return WaitStates;
}
 
void GCNHazardRecognizer::EmitNoop() {
  EmittedInstrs.push_front(nullptr);
}
 
void GCNHazardRecognizer::AdvanceCycle() {
  if (isSchedulerMode())
    schedulerAdvanceCycle();
 
  // When the scheduler detects a stall, it will call AdvanceCycle() without
  // emitting any instructions.
  if (!CurrCycleInstr) {
    EmittedInstrs.push_front(nullptr);
 
    if (HasPendingWMMACoexecHazard)
      EmittedVALUInstrs.push_front(nullptr);
    return;
  }
 
  HasPendingWMMACoexecHazard = false;
 
  if (CurrCycleInstr->isBundle()) {
    processBundle();
    return;
  }
 
  unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr);
  if (!NumWaitStates) {
    CurrCycleInstr = nullptr;
    return;
  }
 
  // Keep track of emitted instructions
  EmittedInstrs.push_front(CurrCycleInstr);
 
  bool IsVALUOrWMMA =
      SIInstrInfo::isVALU(*CurrCycleInstr, /*AllowLDSDMA=*/true) ||
      SIInstrInfo::isWMMA(*CurrCycleInstr) ||
      SIInstrInfo::isSWMMAC(*CurrCycleInstr);
  if (IsVALUOrWMMA) {
    EmittedVALUInstrs.push_front(CurrCycleInstr);
  } else {
    // A pending WMMA co-execution hazard optimistically records stall cycles as
    // future V_NOPs. If the scheduler instead stalls for a different
    // (S_NOP-resolvable) hazard and schedules a non-VALU into those cycles,
    // they will not resolve the VALU-pipe hazard, so drop them here.
    while (!EmittedVALUInstrs.empty() && EmittedVALUInstrs.front() == nullptr)
      EmittedVALUInstrs.pop_front();
  }
 
  // Add a nullptr for each additional wait state after the first.  Make sure
  // not to add more than getMaxLookAhead() items to the list, since we
  // truncate the list to that size right after this loop.
  for (unsigned i = 1, e = std::min(NumWaitStates, getMaxLookAhead());
       i < e; ++i) {
    EmittedInstrs.push_front(nullptr);
  }
 
  // getMaxLookahead() is the largest number of wait states we will ever need
  // to insert, so there is no point in keeping track of more than that many
  // wait states.
  EmittedInstrs.resize(getMaxLookAhead());
  if (EmittedVALUInstrs.size() > MaxVALULookAhead)
    EmittedVALUInstrs.resize(MaxVALULookAhead);
 
  CurrCycleInstr = nullptr;
}
 
void GCNHazardRecognizer::RecedeCycle() {
  assert(!isHazardRecognizerMode() &&
         "Bottom-up scheduling shouldn't run in hazard recognizer mode");
}
 
//===----------------------------------------------------------------------===//
// Helper Functions
//===----------------------------------------------------------------------===//
 
enum HazardFnResult { HazardFound, HazardExpired, NoHazardFound };
 
// Search for a hazard in a block and its predecessors.
template <typename StateT>
static bool
hasHazard(StateT InitialState,
          function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
          function_ref<void(StateT &, const MachineInstr &)> UpdateState,
          const MachineBasicBlock *InitialMBB,
          MachineBasicBlock::const_reverse_instr_iterator InitialI) {
  struct StateMapKey {
    SmallVectorImpl<StateT> *States;
    unsigned Idx;
    static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
      return LHS.States == RHS.States && LHS.Idx == RHS.Idx;
    }
  };
  struct StateMapKeyTraits : DenseMapInfo<StateMapKey> {
    static unsigned getHashValue(const StateMapKey &Key) {
      return StateT::getHashValue((*Key.States)[Key.Idx]);
    }
    static unsigned getHashValue(const StateT &State) {
      return StateT::getHashValue(State);
    }
    static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
      return StateT::isEqual((*LHS.States)[LHS.Idx], (*RHS.States)[RHS.Idx]);
    }
    static bool isEqual(const StateT &LHS, const StateMapKey &RHS) {
      return StateT::isEqual(LHS, (*RHS.States)[RHS.Idx]);
    }
  };
 
  SmallDenseMap<StateMapKey, unsigned, 8, StateMapKeyTraits> StateMap;
  SmallVector<StateT, 8> States;
 
  MachineBasicBlock::const_reverse_instr_iterator I = InitialI;
  const MachineBasicBlock *MBB = InitialMBB;
  StateT State = InitialState;
 
  SmallSetVector<std::pair<const MachineBasicBlock *, unsigned>, 16> Worklist;
  unsigned WorkIdx = 0;
  for (;;) {
    bool Expired = false;
    for (auto E = MBB->instr_rend(); I != E; ++I) {
      // No need to look at parent BUNDLE instructions.
      if (I->isBundle())
        continue;
 
      auto Result = IsHazard(State, *I);
      if (Result == HazardFound)
        return true;
      if (Result == HazardExpired) {
        Expired = true;
        break;
      }
 
      if (I->isInlineAsm() || I->isMetaInstruction())
        continue;
 
      UpdateState(State, *I);
    }
 
    if (!Expired) {
      unsigned StateIdx = States.size();
      StateMapKey Key = {&States, StateIdx};
      auto Insertion = StateMap.insert_as(std::pair(Key, StateIdx), State);
      if (Insertion.second) {
        States.emplace_back(State);
      } else {
        StateIdx = Insertion.first->second;
      }
      for (MachineBasicBlock *Pred : MBB->predecessors())
        Worklist.insert(std::pair(Pred, StateIdx));
    }
 
    if (WorkIdx == Worklist.size())
      break;
 
    unsigned StateIdx;
    std::tie(MBB, StateIdx) = Worklist[WorkIdx++];
    State = States[StateIdx];
    I = MBB->instr_rbegin();
  }
 
  return false;
}
 
// Returns a minimum wait states since \p I walking all predecessors.
// Only scans until \p IsExpired does not return true.
// Can only be run in a hazard recognizer mode.
static int
getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
                   const MachineBasicBlock *MBB,
                   MachineBasicBlock::const_reverse_instr_iterator I,
                   int WaitStates, GCNHazardRecognizer::IsExpiredFn IsExpired,
                   DenseSet<const MachineBasicBlock *> &Visited,
                   GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
                       SIInstrInfo::getNumWaitStates) {
  for (auto E = MBB->instr_rend(); I != E; ++I) {
    // Don't add WaitStates for parent BUNDLE instructions.
    if (I->isBundle())
      continue;
 
    if (IsHazard(*I))
      return WaitStates;
 
    if (I->isInlineAsm())
      continue;
 
    WaitStates += GetNumWaitStates(*I);
 
    if (IsExpired(*I, WaitStates))
      return std::numeric_limits<int>::max();
  }
 
  int MinWaitStates = std::numeric_limits<int>::max();
  for (MachineBasicBlock *Pred : MBB->predecessors()) {
    if (!Visited.insert(Pred).second)
      continue;
 
    int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(), WaitStates,
                               IsExpired, Visited, GetNumWaitStates);
 
    MinWaitStates = std::min(MinWaitStates, W);
  }
 
  return MinWaitStates;
}
 
static int
getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
                   const MachineInstr *MI,
                   GCNHazardRecognizer::IsExpiredFn IsExpired,
                   GCNHazardRecognizer::GetNumWaitStatesFn GetNumWaitStates =
                       SIInstrInfo::getNumWaitStates) {
  DenseSet<const MachineBasicBlock *> Visited;
  return getWaitStatesSince(IsHazard, MI->getParent(),
                            std::next(MI->getReverseIterator()), 0, IsExpired,
                            Visited, GetNumWaitStates);
}
 
int GCNHazardRecognizer::getWaitStatesSince(
    IsHazardFn IsHazard, int Limit, GetNumWaitStatesFn GetNumWaitStates) const {
  if (isHazardRecognizerMode()) {
    auto IsExpiredFn = [Limit](const MachineInstr &, int WaitStates) {
      return WaitStates >= Limit;
    };
    return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn,
                                GetNumWaitStates);
  }
 
  int WaitStates = 0;
  for (MachineInstr *MI : EmittedInstrs) {
    if (MI) {
      if (IsHazard(*MI))
        return WaitStates;
 
      if (MI->isInlineAsm())
        continue;
    }
    WaitStates += MI ? GetNumWaitStates(*MI) : 1;
 
    if (WaitStates >= Limit)
      break;
  }
  return std::numeric_limits<int>::max();
}
 
int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard,
                                            int Limit) const {
  return getWaitStatesSince(IsHazard, Limit, SIInstrInfo::getNumWaitStates);
}
 
int GCNHazardRecognizer::getWaitStatesSinceVALU(IsHazardFn IsHazard,
                                                int Limit) const {
  if (isHazardRecognizerMode()) {
    auto GetVALUWaitStates = [](const MachineInstr &MI) -> unsigned {
      return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
    };
    return getWaitStatesSince(IsHazard, Limit, GetVALUWaitStates);
  }
 
  // EmittedVALUInstrs is capped at MaxVALULookAhead, so a Limit beyond that
  // window could miss a hazard. Keep the cap in sync with the wait-state
  // tables.
  assert(Limit <= (int)MaxVALULookAhead &&
         "Limit exceeds the EmittedVALUInstrs lookahead window");
  int WaitStates = 0;
  for (MachineInstr *MI : EmittedVALUInstrs) {
    if (MI) {
      if (IsHazard(*MI))
        return WaitStates;
    }
 
    ++WaitStates;
 
    if (WaitStates >= Limit)
      break;
  }
  return std::numeric_limits<int>::max();
}
 
int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
                                               IsHazardFn IsHazardDef,
                                               int Limit) const {
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
 
  auto IsHazardFn = [IsHazardDef, TRI, Reg](const MachineInstr &MI) {
    return IsHazardDef(MI) && MI.modifiesRegister(Reg, TRI);
  };
 
  return getWaitStatesSince(IsHazardFn, Limit);
}
 
int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
                                                  int Limit) const {
  auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
    return isSSetReg(MI.getOpcode()) && IsHazard(MI);
  };
 
  return getWaitStatesSince(IsHazardFn, Limit);
}
 
//===----------------------------------------------------------------------===//
// No-op Hazard Detection
//===----------------------------------------------------------------------===//
 
static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
                        MCRegister Reg) {
  for (MCRegUnit Unit : TRI.regunits(Reg))
    BV.set(static_cast<unsigned>(Unit));
}
 
static void addRegsToSet(const SIRegisterInfo &TRI,
                         iterator_range<MachineInstr::const_mop_iterator> Ops,
                         BitVector &DefSet, BitVector &UseSet) {
  for (const MachineOperand &Op : Ops) {
    if (Op.isReg())
      addRegUnits(TRI, Op.isDef() ? DefSet : UseSet, Op.getReg().asMCReg());
  }
}
 
void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) const {
  addRegsToSet(TRI, MI.operands(), ClauseDefs, ClauseUses);
}
 
static bool breaksSMEMSoftClause(MachineInstr *MI) {
  return !SIInstrInfo::isSMRD(*MI);
}
 
static bool breaksVMEMSoftClause(MachineInstr *MI) {
  return !SIInstrInfo::isVMEM(*MI);
}
 
int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) const {
  // SMEM soft clause are only present on VI+, and only matter if xnack is
  // enabled.
  if (!ST.isXNACKEnabled())
    return 0;
 
  bool IsSMRD = TII.isSMRD(*MEM);
 
  resetClause();
 
  // A soft-clause is any group of consecutive SMEM instructions.  The
  // instructions in this group may return out of order and/or may be
  // replayed (i.e. the same instruction issued more than once).
  //
  // In order to handle these situations correctly we need to make sure that
  // when a clause has more than one instruction, no instruction in the clause
  // writes to a register that is read by another instruction in the clause
  // (including itself). If we encounter this situation, we need to break the
  // clause by inserting a non SMEM instruction.
 
  for (MachineInstr *MI : EmittedInstrs) {
    // When we hit a non-SMEM instruction then we have passed the start of the
    // clause and we can stop.
    if (!MI)
      break;
 
    if (IsSMRD ? breaksSMEMSoftClause(MI) : breaksVMEMSoftClause(MI))
      break;
 
    addClauseInst(*MI);
  }
 
  if (ClauseDefs.none())
    return 0;
 
  // We need to make sure not to put loads and stores in the same clause if they
  // use the same address. For now, just start a new clause whenever we see a
  // store.
  if (MEM->mayStore())
    return 1;
 
  addClauseInst(*MEM);
 
  // If the set of defs and uses intersect then we cannot add this instruction
  // to the clause, so we have a hazard.
  return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
}
 
int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) const {
  int WaitStatesNeeded = 0;
 
  WaitStatesNeeded = checkSoftClauseHazards(SMRD);
 
  // This SMRD hazard only affects SI.
  if (!ST.hasSMRDReadVALUDefHazard())
    return WaitStatesNeeded;
 
  // A read of an SGPR by SMRD instruction requires 4 wait states when the
  // SGPR was written by a VALU instruction.
  int SmrdSgprWaitStates = 4;
  auto IsHazardDefFn = [this](const MachineInstr &MI) {
    return TII.isVALU(MI, /*AllowLDSDMA=*/true);
  };
  auto IsBufferHazardDefFn = [this](const MachineInstr &MI) {
    return TII.isSALU(MI);
  };
 
  bool IsBufferSMRD = TII.isBufferSMRD(*SMRD);
 
  for (const MachineOperand &Use : SMRD->uses()) {
    if (!Use.isReg())
      continue;
    int WaitStatesNeededForUse =
        SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
                                                   SmrdSgprWaitStates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
    // This fixes what appears to be undocumented hardware behavior in SI where
    // s_mov writing a descriptor and s_buffer_load_dword reading the descriptor
    // needs some number of nops in between. We don't know how many we need, but
    // let's use 4. This wasn't discovered before probably because the only
    // case when this happens is when we expand a 64-bit pointer into a full
    // descriptor and use s_buffer_load_dword instead of s_load_dword, which was
    // probably never encountered in the closed-source land.
    if (IsBufferSMRD) {
      int WaitStatesNeededForUse =
        SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
                                                   IsBufferHazardDefFn,
                                                   SmrdSgprWaitStates);
      WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
    }
  }
 
  return WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkVMEMHazards(MachineInstr *VMEM) const {
  if (!ST.hasVMEMReadSGPRVALUDefHazard())
    return 0;
 
  int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
 
  // A read of an SGPR by a VMEM instruction requires 5 wait states when the
  // SGPR was written by a VALU Instruction.
  const int VmemSgprWaitStates = 5;
  auto IsHazardDefFn = [this](const MachineInstr &MI) {
    return TII.isVALU(MI, /*AllowLDSDMA=*/true);
  };
  for (const MachineOperand &Use : VMEM->uses()) {
    if (!Use.isReg() || TRI.isVectorRegister(MF.getRegInfo(), Use.getReg()))
      continue;
 
    int WaitStatesNeededForUse =
        VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
                                                   VmemSgprWaitStates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
  }
  return WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) const {
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const SIInstrInfo *TII = ST.getInstrInfo();
 
  // Check for DPP VGPR read after VALU VGPR write and EXEC write.
  int DppVgprWaitStates = 2;
  int DppExecWaitStates = 5;
  int WaitStatesNeeded = 0;
  auto IsHazardDefFn = [TII](const MachineInstr &MI) {
    return TII->isVALU(MI, /*AllowLDSDMA=*/true);
  };
 
  for (const MachineOperand &Use : DPP->uses()) {
    if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
      continue;
    int WaitStatesNeededForUse =
        DppVgprWaitStates - getWaitStatesSinceDef(
                                Use.getReg(),
                                [](const MachineInstr &) { return true; },
                                DppVgprWaitStates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
  }
 
  WaitStatesNeeded = std::max(
      WaitStatesNeeded,
      DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
                                                DppExecWaitStates));
 
  return WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) const {
  const SIInstrInfo *TII = ST.getInstrInfo();
 
  // v_div_fmas requires 4 wait states after a write to vcc from a VALU
  // instruction.
  const int DivFMasWaitStates = 4;
  auto IsHazardDefFn = [TII](const MachineInstr &MI) {
    return TII->isVALU(MI, /*AllowLDSDMA=*/true);
  };
  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
                                               DivFMasWaitStates);
 
  return DivFMasWaitStates - WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) const {
  const SIInstrInfo *TII = ST.getInstrInfo();
  unsigned GetRegHWReg = getHWReg(TII, *GetRegInstr);
 
  const int GetRegWaitStates = 2;
  auto IsHazardFn = [TII, GetRegHWReg](const MachineInstr &MI) {
    return GetRegHWReg == getHWReg(TII, MI);
  };
  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
 
  return GetRegWaitStates - WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) const {
  const SIInstrInfo *TII = ST.getInstrInfo();
  unsigned HWReg = getHWReg(TII, *SetRegInstr);
 
  const int SetRegWaitStates = ST.getSetRegWaitStates();
  auto IsHazardFn = [TII, HWReg](const MachineInstr &MI) {
    return HWReg == getHWReg(TII, MI);
  };
  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
  return SetRegWaitStates - WaitStatesNeeded;
}
 
int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) const {
  if (!MI.mayStore())
    return -1;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  unsigned Opcode = MI.getOpcode();
  const MCInstrDesc &Desc = MI.getDesc();
 
  int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdata);
  int VDataRCID = -1;
  if (VDataIdx != -1)
    VDataRCID = TII->getOpRegClassID(Desc.operands()[VDataIdx]);
 
  if (TII->isMUBUF(MI) || TII->isMTBUF(MI)) {
    // There is no hazard if the instruction does not use vector regs
    // (like wbinvl1)
    if (VDataIdx == -1)
      return -1;
    if (AMDGPU::getRegBitWidth(VDataRCID) > 64) {
      // When SOFFSET-dependent wide-store windows apply, the BUFFER_STORE
      // source-vgpr WAR hazard exists for every SOFFSET shape; the wait-state
      // count differs by SOFFSET and is computed in checkVALUHazardsHelper.
      // Otherwise the hazard only exists if soffset is not an SGPR.
      if (ST.hasVDecCoExecHazard())
        return VDataIdx;
      const MachineOperand *SOffset =
          TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
      if (!SOffset || !SOffset->isReg())
        return VDataIdx;
    }
  }
 
  // MIMG instructions create a hazard if they don't use a 256-bit T# and
  // the store size is greater than 8 bytes and they have more than two bits
  // of their dmask set.
  // All our MIMG definitions use a 256-bit T#, so we can skip checking for them.
  if (TII->isMIMG(MI)) {
    int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
    assert(SRsrcIdx != -1 && AMDGPU::getRegBitWidth(TII->getOpRegClassID(
                                 Desc.operands()[SRsrcIdx])) == 256);
    (void)SRsrcIdx;
  }
 
  if (TII->isFLAT(MI)) {
    // There is no hazard if the instruction does not use vector regs
    if (VDataIdx == -1)
      return -1;
 
    if (AMDGPU::getRegBitWidth(VDataRCID) > 64)
      return VDataIdx;
  }
 
  return -1;
}
 
int GCNHazardRecognizer::checkUniformWindowVALUHazardsHelper(
    Register Reg) const {
  // Wide stores need a single wait-state bubble before a VALU that overwrites
  // store data. createsVALUHazard already excludes MUBUF/MTBUF stores with an
  // SGPR SOFFSET.
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
 
  auto IsHazard = [&](const MachineInstr &MI) {
    int DataIdx = createsVALUHazard(MI);
    return DataIdx >= 0 &&
           TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg);
  };
 
  return std::max(0, 1 - getWaitStatesSince(IsHazard, /*Limit=*/1));
}
 
int GCNHazardRecognizer::checkSOFFSETWindowVALUHazardsHelper(
    Register Reg) const {
  // The required wait-state window depends on the producer's SOFFSET shape:
  //   - MUBUF/MTBUF wide store with sgpr SOFFSET: 1 wait state.
  //   - MUBUF/MTBUF wide store with literal/absent SOFFSET, and FLAT wide
  //     store: 2 wait states.
  // The 1-cycle sgpr-SOFFSET window was measured on gfx950.
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const SIInstrInfo *TII = ST.getInstrInfo();
 
  int WaitStatesNeeded = 0;
 
  // Scan each wait-state window separately and take the max padding needed.
  // getWaitStatesSince supplies the minimum distance to a producer over paths.
  for (int Window = 1; Window <= 2; ++Window) {
    auto IsHazard = [&](const MachineInstr &MI) {
      int DataIdx = createsVALUHazard(MI);
      if (DataIdx < 0 ||
          !TRI->regsOverlap(MI.getOperand(DataIdx).getReg(), Reg))
        return false;
 
      // Window 1 matches every hazard producer. Window 2 excludes BUF stores
      // with an SGPR SOFFSET, which only require a single wait state.
      if (Window == 1 || !TII->isBUF(MI))
        return true;
 
      const MachineOperand *SOffset =
          TII->getNamedOperand(MI, AMDGPU::OpName::soffset);
      return !SOffset || !SOffset->isReg();
    };
    WaitStatesNeeded = std::max(WaitStatesNeeded,
                                Window - getWaitStatesSince(IsHazard, Window));
  }
 
  return WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkVALUHazardsHelper(
    const MachineOperand &Def, const MachineRegisterInfo &MRI) const {
  // Helper to check for the hazard where VMEM instructions that store more
  // than 8 bytes can have their store data overwritten by the next
  // instruction.
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
 
  if (!TRI->isVectorRegister(MRI, Def.getReg()))
    return 0;
 
  if (ST.hasVDecCoExecHazard())
    return checkSOFFSETWindowVALUHazardsHelper(Def.getReg());
 
  return checkUniformWindowVALUHazardsHelper(Def.getReg());
}
 
/// Dest sel forwarding issue occurs if additional logic is needed to swizzle /
/// pack the computed value into correct bit position of the dest register. This
/// occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with
/// dst_sel that is not aligned to the register. This function analayzes the \p
/// MI and \returns an operand with dst forwarding issue, or nullptr if
/// none exists.
static const MachineOperand *
getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) {
  if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
    return nullptr;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
 
  unsigned Opcode = MI.getOpcode();
 
  // There are three different types of instructions
  // which produce forwarded dest: 1. SDWA with dst_sel != DWORD, 2. VOP3
  // which write hi bits (e.g. op_sel[3] == 1), and 3. FP8DstSelInst
  // (instructions with dest byte sel, e.g. CVT_SR_BF8_F32) and
  // op_sel[3:2]
  // != 0
  if (SIInstrInfo::isSDWA(MI)) {
    // Type 1: SDWA with dst_sel != DWORD
    if (auto *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel))
      if (DstSel->getImm() != AMDGPU::SDWA::DWORD)
        return TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
  }
 
  AMDGPU::FPType IsFP4OrFP8ConvOpc = AMDGPU::getFPDstSelType(Opcode);
  if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::op_sel)) {
    // Type 2: VOP3 which write the hi bits
    if (TII->getNamedImmOperand(MI, AMDGPU::OpName::src0_modifiers) &
        SISrcMods::DST_OP_SEL)
      return TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 
    // Type 3: FP8DstSelInst with op_sel[3:2] != 0)
    if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP8 &&
        (TII->getNamedImmOperand(MI, AMDGPU::OpName::src2_modifiers) &
         SISrcMods::OP_SEL_0))
      return TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
  }
 
  // Special case: nop is required for all the opsel values for fp4 sr variant
  // cvt scale instructions
  if (IsFP4OrFP8ConvOpc == AMDGPU::FPType::FP4)
    return TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 
  return nullptr;
}
 
/// Checks whether the provided \p MI "consumes" the operand with a Dest sel
/// fowarding issue \p Dst . We may "consume" the Dst via a standard explicit
/// RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)
static bool consumesDstSelForwardingOperand(const MachineInstr *VALU,
                                            const MachineOperand *Dst,
                                            const SIRegisterInfo *TRI) {
  // We must consider implicit reads of the VALU. SDWA with dst_sel and
  // UNUSED_PRESERVE will implicitly read the result from forwarded dest,
  // and we must account for that hazard.
  // We also must account for WAW hazards. In particular, WAW with dest
  // preserve semantics (e.g. VOP3 with op_sel, VOP2 &&
  // !zeroesHigh16BitsOfDest) will read the forwarded dest for parity
  // check for ECC. Without accounting for this hazard, the ECC will be
  // wrong.
  // TODO: limit to RAW (including implicit reads) + problematic WAW (i.e.
  // complete zeroesHigh16BitsOfDest)
  for (auto &Operand : VALU->operands()) {
    if (Operand.isReg() && TRI->regsOverlap(Dst->getReg(), Operand.getReg())) {
      return true;
    }
  }
  return false;
}
 
int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) const {
  int WaitStatesNeeded = 0;
 
  if (ST.hasTransForwardingHazard() && !SIInstrInfo::isTRANS(*VALU)) {
    const int TransDefWaitstates = 1;
 
    auto IsTransDefFn = [this, VALU](const MachineInstr &MI) {
      if (!SIInstrInfo::isTRANS(MI))
        return false;
      const SIRegisterInfo *TRI = ST.getRegisterInfo();
      const SIInstrInfo *TII = ST.getInstrInfo();
      Register Def = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)->getReg();
 
      for (const MachineOperand &Use : VALU->explicit_uses()) {
        if (Use.isReg() && TRI->regsOverlap(Def, Use.getReg()))
          return true;
      }
 
      return false;
    };
 
    int WaitStatesNeededForDef =
        TransDefWaitstates -
        getWaitStatesSince(IsTransDefFn, TransDefWaitstates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
  }
 
  if (ST.hasDstSelForwardingHazard() || ST.hasCvtScaleForwardingHazard()) {
    const int Shift16DefWaitstates = 1;
 
    auto IsShift16BitDefFn = [this, VALU](const MachineInstr &ProducerMI) {
      const SIRegisterInfo *TRI = ST.getRegisterInfo();
      const MachineOperand *ForwardedDst =
          getDstSelForwardingOperand(ProducerMI, ST);
      if (ForwardedDst) {
        return consumesDstSelForwardingOperand(VALU, ForwardedDst, TRI);
      }
 
      if (ProducerMI.isInlineAsm()) {
        // Assume inline asm has dst forwarding hazard
        for (auto &Def : ProducerMI.all_defs()) {
          if (consumesDstSelForwardingOperand(VALU, &Def, TRI))
            return true;
        }
      }
 
      return false;
    };
 
    int WaitStatesNeededForDef =
        Shift16DefWaitstates -
        getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
  }
 
  if (ST.hasVDecCoExecHazard()) {
    const int VALUWriteSGPRVALUReadWaitstates = 2;
    const int VALUWriteEXECRWLane = 4;
    const int VALUWriteVGPRReadlaneRead = 1;
 
    const SIRegisterInfo *TRI = ST.getRegisterInfo();
    const MachineRegisterInfo &MRI = MF.getRegInfo();
    Register UseReg;
    auto IsVALUDefSGPRFn = [&UseReg, TRI](const MachineInstr &MI) {
      if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
        return false;
      return MI.modifiesRegister(UseReg, TRI);
    };
 
    for (const MachineOperand &Use : VALU->explicit_uses()) {
      if (!Use.isReg())
        continue;
 
      UseReg = Use.getReg();
      if (TRI->isSGPRReg(MRI, UseReg)) {
        int WaitStatesNeededForDef =
            VALUWriteSGPRVALUReadWaitstates -
            getWaitStatesSince(IsVALUDefSGPRFn,
                               VALUWriteSGPRVALUReadWaitstates);
        WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
      }
    }
 
    if (VALU->readsRegister(AMDGPU::VCC, TRI)) {
      UseReg = AMDGPU::VCC;
      int WaitStatesNeededForDef =
          VALUWriteSGPRVALUReadWaitstates -
          getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteSGPRVALUReadWaitstates);
      WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
    }
 
    switch (VALU->getOpcode()) {
    case AMDGPU::V_READLANE_B32:
    case AMDGPU::V_READFIRSTLANE_B32: {
      MachineOperand *Src = TII.getNamedOperand(*VALU, AMDGPU::OpName::src0);
      UseReg = Src->getReg();
      int WaitStatesNeededForDef =
          VALUWriteVGPRReadlaneRead -
          getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteVGPRReadlaneRead);
      WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
    }
      [[fallthrough]];
    case AMDGPU::V_WRITELANE_B32: {
      UseReg = AMDGPU::EXEC;
      int WaitStatesNeededForDef =
          VALUWriteEXECRWLane -
          getWaitStatesSince(IsVALUDefSGPRFn, VALUWriteEXECRWLane);
      WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
      break;
    }
    default:
      break;
    }
  }
 
  // This checks for the hazard where VMEM instructions that store more than
  // 8 bytes can have there store data over written by the next instruction.
  if (!ST.has12DWordStoreHazard())
    return WaitStatesNeeded;
 
  const MachineRegisterInfo &MRI = MF.getRegInfo();
 
  for (const MachineOperand &Def : VALU->defs()) {
    WaitStatesNeeded = std::max(WaitStatesNeeded, checkVALUHazardsHelper(Def, MRI));
  }
 
  return WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) const {
  // This checks for hazards associated with inline asm statements.
  // Since inline asms can contain just about anything, we use this
  // to call/leverage other check*Hazard routines. Note that
  // this function doesn't attempt to address all possible inline asm
  // hazards (good luck), but is a collection of what has been
  // problematic thus far.
 
  // see checkVALUHazards()
  if (!ST.has12DWordStoreHazard() && !ST.hasDstSelForwardingHazard() &&
      !ST.hasCvtScaleForwardingHazard())
    return 0;
 
  const MachineRegisterInfo &MRI = MF.getRegInfo();
  int WaitStatesNeeded = 0;
 
  for (const MachineOperand &Op :
       llvm::drop_begin(IA->operands(), InlineAsm::MIOp_FirstOperand)) {
    if (Op.isReg() && Op.isDef()) {
      if (!TRI.isVectorRegister(MRI, Op.getReg()))
        continue;
 
      if (ST.has12DWordStoreHazard()) {
        WaitStatesNeeded =
            std::max(WaitStatesNeeded, checkVALUHazardsHelper(Op, MRI));
      }
    }
  }
 
  if (ST.hasDstSelForwardingHazard()) {
    const int Shift16DefWaitstates = 1;
 
    auto IsShift16BitDefFn = [this, &IA](const MachineInstr &ProducerMI) {
      const MachineOperand *Dst = getDstSelForwardingOperand(ProducerMI, ST);
      // Assume inline asm reads the dst
      if (Dst)
        return IA->modifiesRegister(Dst->getReg(), &TRI) ||
               IA->readsRegister(Dst->getReg(), &TRI);
 
      if (ProducerMI.isInlineAsm()) {
        // If MI is inline asm, assume it has dst forwarding hazard
        for (auto &Def : ProducerMI.all_defs()) {
          if (IA->modifiesRegister(Def.getReg(), &TRI) ||
              IA->readsRegister(Def.getReg(), &TRI)) {
            return true;
          }
        }
      }
 
      return false;
    };
 
    int WaitStatesNeededForDef =
        Shift16DefWaitstates -
        getWaitStatesSince(IsShift16BitDefFn, Shift16DefWaitstates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
  }
 
  return WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) const {
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const MachineRegisterInfo &MRI = MF.getRegInfo();
 
  const MachineOperand *LaneSelectOp =
      TII->getNamedOperand(*RWLane, AMDGPU::OpName::src1);
 
  if (!LaneSelectOp->isReg() || !TRI->isSGPRReg(MRI, LaneSelectOp->getReg()))
    return 0;
 
  Register LaneSelectReg = LaneSelectOp->getReg();
  auto IsHazardFn = [TII](const MachineInstr &MI) {
    return TII->isVALU(MI, /*AllowLDSDMA=*/true);
  };
 
  const int RWLaneWaitStates = 4;
  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
                                              RWLaneWaitStates);
  return RWLaneWaitStates - WaitStatesSince;
}
 
int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) const {
  if (!ST.hasRFEHazards())
    return 0;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
 
  const int RFEWaitStates = 1;
 
  auto IsHazardFn = [TII](const MachineInstr &MI) {
    return getHWReg(TII, MI) == AMDGPU::Hwreg::ID_TRAPSTS;
  };
  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
  return RFEWaitStates - WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) const {
  const SIInstrInfo *TII = ST.getInstrInfo();
  const int ReadM0WaitStates = 1;
  auto IsHazardFn = [TII](const MachineInstr &MI) { return TII->isSALU(MI); };
  return ReadM0WaitStates -
         getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn, ReadM0WaitStates);
}
 
void GCNHazardRecognizer::emitVNops(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator InsertPt,
                                    int WaitStatesNeeded, bool IsHoisting) {
  const DebugLoc &DL = IsHoisting ? DebugLoc() : InsertPt->getDebugLoc();
  for (int I = 0; I < WaitStatesNeeded; ++I)
    BuildMI(MBB, InsertPt, DL, TII.get(AMDGPU::V_NOP_e32));
}
 
void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
  fixVMEMtoScalarWriteHazards(MI);
  fixVcmpxPermlaneHazards(MI);
  fixSMEMtoVectorWriteHazards(MI);
  fixVcmpxExecWARHazard(MI);
  fixLdsBranchVmemWARHazard(MI);
  if (ST.hasLdsDirect()) {
    fixLdsDirectVALUHazard(MI);
    fixLdsDirectVMEMHazard(MI);
  }
  fixVALUPartialForwardingHazard(MI);
  fixVALUTransUseHazard(MI);
  fixVALUTransCoexecutionHazards(MI);
  fixWMMAHazards(MI); // fall-through if co-execution is enabled.
  fixWMMACoexecutionHazards(MI);
  fixShift64HighRegBug(MI);
  fixVALUMaskWriteHazard(MI);
  fixRequiredExportPriority(MI);
  if (ST.hasVPermPk16Hazard())
    fixVPermPk16Hazard(MI);
  if (ST.requiresWaitIdleBeforeGetReg())
    fixGetRegWaitIdle(MI);
  if (ST.hasDsAtomicAsyncBarrierArriveB64PipeBug())
    fixDsAtomicAsyncBarrierArriveB64(MI);
  if (ST.hasScratchBaseForwardingHazard())
    fixScratchBaseForwardingHazard(MI);
  if (ST.setRegModeNeedsVNOPs())
    fixSetRegMode(MI);
  if (ST.hasNeedsTDMDrain())
    fixTDM(MI);
}
 
static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI,
                              const MachineInstr &MI) {
  return (TII.isVOPC(MI) ||
          (MI.isCompare() && (TII.isVOP3(MI) || TII.isSDWA(MI)))) &&
         MI.modifiesRegister(AMDGPU::EXEC, &TRI);
}
 
bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) {
  if (!ST.hasVcmpxPermlaneHazard() || !isPermlane(*MI))
    return false;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  auto IsHazardFn = [TII, TRI](const MachineInstr &MI) {
    return isVCmpXWritesExec(*TII, *TRI, MI);
  };
 
  auto IsExpiredFn = [](const MachineInstr &MI, int) {
    unsigned Opc = MI.getOpcode();
    return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
           Opc != AMDGPU::V_NOP_e32 && Opc != AMDGPU::V_NOP_e64 &&
           Opc != AMDGPU::V_NOP_sdwa;
  };
 
  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
      std::numeric_limits<int>::max())
    return false;
 
  // V_NOP will be discarded by SQ.
  // Use V_MOV_B32 v?, v?. Register must be alive so use src0 of V_PERMLANE*
  // which is always a VGPR and available.
  auto *Src0 = TII->getNamedOperand(*MI, AMDGPU::OpName::src0);
  Register Reg = Src0->getReg();
  bool IsUndef = Src0->isUndef();
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII->get(AMDGPU::V_MOV_B32_e32))
      .addReg(Reg, RegState::Define | getDeadRegState(IsUndef))
      .addReg(Reg, IsUndef ? RegState::Undef : RegState::Kill);
 
  return true;
}
 
bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) {
  if (!ST.hasVMEMtoScalarWriteHazard())
    return false;
  assert(!ST.hasExtendedWaitCounts());
 
  if (!SIInstrInfo::isSALU(*MI) && !SIInstrInfo::isSMRD(*MI))
    return false;
 
  if (MI->getNumDefs() == 0)
    return false;
 
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
 
  auto IsHazardFn = [TRI, MI](const MachineInstr &I) {
    if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I))
      return false;
 
    for (const MachineOperand &Def : MI->defs()) {
      const MachineOperand *Op =
          I.findRegisterUseOperand(Def.getReg(), TRI, false);
      if (!Op)
        continue;
      return true;
    }
    return false;
  };
 
  auto IsExpiredFn = [](const MachineInstr &MI, int) {
    return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ||
           (MI.getOpcode() == AMDGPU::S_WAITCNT &&
            !MI.getOperand(0).getImm()) ||
           (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
            AMDGPU::DepCtr::decodeFieldVmVsrc(MI.getOperand(0).getImm()) == 0);
  };
 
  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
      std::numeric_limits<int>::max())
    return false;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII->get(AMDGPU::S_WAITCNT_DEPCTR))
      .addImm(AMDGPU::DepCtr::encodeFieldVmVsrc(0, ST));
  return true;
}
 
bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
  if (!ST.hasSMEMtoVectorWriteHazard())
    return false;
  assert(!ST.hasExtendedWaitCounts());
 
  if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
    return false;
 
  AMDGPU::OpName SDSTName;
  switch (MI->getOpcode()) {
  case AMDGPU::V_READLANE_B32:
  case AMDGPU::V_READFIRSTLANE_B32:
    SDSTName = AMDGPU::OpName::vdst;
    break;
  default:
    SDSTName = AMDGPU::OpName::sdst;
    break;
  }
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(ST.getCPU());
  const MachineOperand *SDST = TII->getNamedOperand(*MI, SDSTName);
  if (!SDST) {
    for (const auto &MO : MI->implicit_operands()) {
      if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg()))) {
        SDST = &MO;
        break;
      }
    }
  }
 
  if (!SDST)
    return false;
 
  const Register SDSTReg = SDST->getReg();
  auto IsHazardFn = [SDSTReg, TRI](const MachineInstr &I) {
    return SIInstrInfo::isSMRD(I) && I.readsRegister(SDSTReg, TRI);
  };
 
  auto IsExpiredFn = [TII, IV](const MachineInstr &MI, int) {
    if (TII->isSALU(MI)) {
      switch (MI.getOpcode()) {
      case AMDGPU::S_SETVSKIP:
      case AMDGPU::S_VERSION:
      case AMDGPU::S_WAITCNT_VSCNT:
      case AMDGPU::S_WAITCNT_VMCNT:
      case AMDGPU::S_WAITCNT_EXPCNT:
        // These instructions cannot not mitigate the hazard.
        return false;
      case AMDGPU::S_WAITCNT_LGKMCNT:
        // Reducing lgkmcnt count to 0 always mitigates the hazard.
        return (MI.getOperand(1).getImm() == 0) &&
               (MI.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
      case AMDGPU::S_WAITCNT: {
        const int64_t Imm = MI.getOperand(0).getImm();
        AMDGPU::Waitcnt Decoded = AMDGPU::decodeWaitcnt(IV, Imm);
        // DsCnt corresponds to LGKMCnt here.
        return Decoded.get(AMDGPU::DS_CNT) == 0;
      }
      default:
        assert((!SIInstrInfo::isWaitcnt(MI.getOpcode()) ||
                MI.getOpcode() == AMDGPU::S_WAIT_IDLE) &&
               "unexpected wait count instruction");
        // SOPP instructions cannot mitigate the hazard.
        if (TII->isSOPP(MI))
          return false;
        // At this point the SALU can be assumed to mitigate the hazard
        // because either:
        // (a) it is independent of the at risk SMEM (breaking chain),
        // or
        // (b) it is dependent on the SMEM, in which case an appropriate
        //     s_waitcnt lgkmcnt _must_ exist between it and the at risk
        //     SMEM instruction.
        return true;
      }
    }
    return false;
  };
 
  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
      std::numeric_limits<int>::max())
    return false;
 
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII->get(AMDGPU::S_MOV_B32), AMDGPU::SGPR_NULL)
      .addImm(0);
  return true;
}
 
bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) {
  if (!ST.hasVcmpxExecWARHazard())
    return false;
  assert(!ST.hasExtendedWaitCounts());
 
  if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
    return false;
 
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  if (!MI->modifiesRegister(AMDGPU::EXEC, TRI))
    return false;
 
  auto IsHazardFn = [TRI](const MachineInstr &I) {
    if (SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true))
      return false;
    return I.readsRegister(AMDGPU::EXEC, TRI);
  };
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  auto IsExpiredFn = [TII, TRI](const MachineInstr &MI, int) {
    if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
      if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst))
        return true;
      for (auto MO : MI.implicit_operands())
        if (MO.isDef() && TRI->isSGPRClass(TRI->getPhysRegBaseClass(MO.getReg())))
          return true;
    }
    if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
        AMDGPU::DepCtr::decodeFieldSaSdst(MI.getOperand(0).getImm()) == 0)
      return true;
    return false;
  };
 
  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
      std::numeric_limits<int>::max())
    return false;
 
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII->get(AMDGPU::S_WAITCNT_DEPCTR))
      .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0, ST));
  return true;
}
 
static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
                                                 const GCNSubtarget &ST) {
  if (!ST.hasLdsBranchVmemWARHazard())
    return false;
 
  // Check if the necessary condition for the hazard is met: both LDS and VMEM
  // instructions need to appear in the same function.
  bool HasLds = false;
  bool HasVmem = false;
  for (auto &MBB : MF) {
    for (auto &MI : MBB) {
      HasLds |= SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI);
      HasVmem |= SIInstrInfo::isVMEM(MI);
      if (HasLds && HasVmem)
        return true;
    }
  }
  return false;
}
 
static bool isStoreCountWaitZero(const MachineInstr &I) {
  return I.getOpcode() == AMDGPU::S_WAITCNT_VSCNT &&
         I.getOperand(0).getReg() == AMDGPU::SGPR_NULL &&
         !I.getOperand(1).getImm();
}
 
bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) {
  if (!RunLdsBranchVmemWARHazardFixup)
    return false;
 
  assert(ST.hasLdsBranchVmemWARHazard());
  assert(!ST.hasExtendedWaitCounts());
 
  auto IsHazardInst = [](const MachineInstr &MI) {
    if (SIInstrInfo::isDS(MI) || SIInstrInfo::isLDSDMA(MI))
      return 1;
    if (SIInstrInfo::isVMEM(MI))
      return 2;
    return 0;
  };
 
  auto InstType = IsHazardInst(*MI);
  if (!InstType)
    return false;
 
  auto IsExpiredFn = [&IsHazardInst](const MachineInstr &I, int) {
    return IsHazardInst(I) || isStoreCountWaitZero(I);
  };
 
  auto IsHazardFn = [InstType, &IsHazardInst](const MachineInstr &I) {
    if (!I.isBranch())
      return false;
 
    auto IsHazardFn = [InstType, IsHazardInst](const MachineInstr &I) {
      auto InstType2 = IsHazardInst(I);
      return InstType2 && InstType != InstType2;
    };
 
    auto IsExpiredFn = [InstType, &IsHazardInst](const MachineInstr &I, int) {
      auto InstType2 = IsHazardInst(I);
      if (InstType == InstType2)
        return true;
 
      return isStoreCountWaitZero(I);
    };
 
    return ::getWaitStatesSince(IsHazardFn, &I, IsExpiredFn) !=
           std::numeric_limits<int>::max();
  };
 
  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
      std::numeric_limits<int>::max())
    return false;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII->get(AMDGPU::S_WAITCNT_VSCNT))
    .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
    .addImm(0);
 
  return true;
}
 
bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) {
  if (!SIInstrInfo::isLDSDIR(*MI))
    return false;
 
  const int NoHazardWaitStates = 15;
  const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
  const Register VDSTReg = VDST->getReg();
 
  bool VisitedTrans = false;
  auto IsHazardFn = [this, VDSTReg, &VisitedTrans](const MachineInstr &I) {
    if (!SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true))
      return false;
    VisitedTrans = VisitedTrans || SIInstrInfo::isTRANS(I);
    // Cover both WAR and WAW
    return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
  };
  auto IsExpiredFn = [&](const MachineInstr &I, int WaitStates) {
    if (WaitStates >= NoHazardWaitStates)
      return true;
    // Instructions which cause va_vdst==0 expire hazard
    return SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) ||
           SIInstrInfo::isEXP(I);
  };
  auto GetWaitStatesFn = [](const MachineInstr &MI) {
    return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) ? 1 : 0;
  };
 
  DenseSet<const MachineBasicBlock *> Visited;
  auto Count = ::getWaitStatesSince(IsHazardFn, MI->getParent(),
                                    std::next(MI->getReverseIterator()), 0,
                                    IsExpiredFn, Visited, GetWaitStatesFn);
 
  // Transcendentals can execute in parallel to other VALUs.
  // This makes va_vdst count unusable with a mixture of VALU and TRANS.
  if (VisitedTrans)
    Count = 0;
 
  MachineOperand *WaitVdstOp =
      TII.getNamedOperand(*MI, AMDGPU::OpName::waitvdst);
  WaitVdstOp->setImm(std::min(Count, NoHazardWaitStates));
 
  return true;
}
 
bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) {
  if (!SIInstrInfo::isLDSDIR(*MI))
    return false;
 
  const MachineOperand *VDST = TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
  const Register VDSTReg = VDST->getReg();
 
  auto IsHazardFn = [this, VDSTReg](const MachineInstr &I) {
    if (!SIInstrInfo::isVMEM(I) && !SIInstrInfo::isDS(I))
      return false;
    return I.readsRegister(VDSTReg, &TRI) || I.modifiesRegister(VDSTReg, &TRI);
  };
  bool LdsdirCanWait = ST.hasLdsWaitVMSRC();
  // TODO: On GFX12 the hazard should expire on S_WAIT_LOADCNT/SAMPLECNT/BVHCNT
  // according to the type of VMEM instruction.
  auto IsExpiredFn = [this, LdsdirCanWait](const MachineInstr &I, int) {
    return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true) ||
           SIInstrInfo::isEXP(I) ||
           (I.getOpcode() == AMDGPU::S_WAITCNT && !I.getOperand(0).getImm()) ||
           (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
            AMDGPU::DepCtr::decodeFieldVmVsrc(I.getOperand(0).getImm()) == 0) ||
           (LdsdirCanWait && SIInstrInfo::isLDSDIR(I) &&
            !TII.getNamedOperand(I, AMDGPU::OpName::waitvsrc)->getImm());
  };
 
  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
      std::numeric_limits<int>::max())
    return false;
 
  if (LdsdirCanWait) {
    TII.getNamedOperand(*MI, AMDGPU::OpName::waitvsrc)->setImm(0);
  } else {
    BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
            TII.get(AMDGPU::S_WAITCNT_DEPCTR))
        .addImm(AMDGPU::DepCtr::encodeFieldVmVsrc(0, ST));
  }
 
  return true;
}
 
bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
  if (!ST.hasVALUPartialForwardingHazard())
    return false;
  assert(!ST.hasExtendedWaitCounts());
 
  if (!ST.isWave64() || !SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
    return false;
 
  SmallSetVector<Register, 4> SrcVGPRs;
 
  for (const MachineOperand &Use : MI->explicit_uses()) {
    if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
      SrcVGPRs.insert(Use.getReg());
  }
 
  // Only applies with >= 2 unique VGPR sources
  if (SrcVGPRs.size() <= 1)
    return false;
 
  // Look for the following pattern:
  //   Va <- VALU [PreExecPos]
  //   intv1
  //   Exec <- SALU [ExecPos]
  //   intv2
  //   Vb <- VALU [PostExecPos]
  //   intv3
  //   MI Va, Vb (WaitState = 0)
  //
  // Where:
  // intv1 + intv2 <= 2 VALUs
  // intv3 <= 4 VALUs
  //
  // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
 
  const int Intv1plus2MaxVALUs = 2;
  const int Intv3MaxVALUs = 4;
  const int IntvMaxVALUs = 6;
  const int NoHazardVALUWaitStates = IntvMaxVALUs + 2;
 
  struct StateType {
    SmallDenseMap<Register, int, 4> DefPos;
    int ExecPos = std::numeric_limits<int>::max();
    int VALUs = 0;
 
    static unsigned getHashValue(const StateType &State) {
      hash_code H = hash_combine(State.ExecPos, State.VALUs);
      for (const auto &[Reg, Pos] : State.DefPos)
        H = hash_combine(H, Reg, Pos);
      return H;
    }
    static bool isEqual(const StateType &LHS, const StateType &RHS) {
      return LHS.DefPos == RHS.DefPos && LHS.ExecPos == RHS.ExecPos &&
             LHS.VALUs == RHS.VALUs;
    }
  };
 
  StateType State;
 
  // This overloads expiry testing with all the hazard detection
  auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
    // Too many VALU states have passed
    if (State.VALUs > NoHazardVALUWaitStates)
      return HazardExpired;
 
    // Instructions which cause va_vdst==0 expire hazard
    if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) ||
        SIInstrInfo::isEXP(I) ||
        (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
         AMDGPU::DepCtr::decodeFieldVaVdst(I.getOperand(0).getImm()) == 0))
      return HazardExpired;
 
    // Track registers writes
    bool Changed = false;
    if (SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true)) {
      for (Register Src : SrcVGPRs) {
        if (!State.DefPos.count(Src) && I.modifiesRegister(Src, &TRI)) {
          State.DefPos[Src] = State.VALUs;
          Changed = true;
        }
      }
    } else if (SIInstrInfo::isSALU(I)) {
      if (State.ExecPos == std::numeric_limits<int>::max()) {
        if (!State.DefPos.empty() && I.modifiesRegister(AMDGPU::EXEC, &TRI)) {
          State.ExecPos = State.VALUs;
          Changed = true;
        }
      }
    }
 
    // Early expiration: too many VALUs in intv3
    if (State.VALUs > Intv3MaxVALUs && State.DefPos.empty())
      return HazardExpired;
 
    // Only evaluate state if something changed
    if (!Changed)
      return NoHazardFound;
 
    // Determine positions of VALUs pre/post exec change
    if (State.ExecPos == std::numeric_limits<int>::max())
      return NoHazardFound;
 
    int PreExecPos = std::numeric_limits<int>::max();
    int PostExecPos = std::numeric_limits<int>::max();
 
    for (auto Entry : State.DefPos) {
      int DefVALUs = Entry.second;
      if (DefVALUs != std::numeric_limits<int>::max()) {
        if (DefVALUs >= State.ExecPos)
          PreExecPos = std::min(PreExecPos, DefVALUs);
        else
          PostExecPos = std::min(PostExecPos, DefVALUs);
      }
    }
 
    // Need a VALUs post exec change
    if (PostExecPos == std::numeric_limits<int>::max())
      return NoHazardFound;
 
    // Too many VALUs in intv3?
    int Intv3VALUs = PostExecPos;
    if (Intv3VALUs > Intv3MaxVALUs)
      return HazardExpired;
 
    // Too many VALUs in intv2?
    int Intv2VALUs = (State.ExecPos - PostExecPos) - 1;
    if (Intv2VALUs > Intv1plus2MaxVALUs)
      return HazardExpired;
 
    // Need a VALUs pre exec change
    if (PreExecPos == std::numeric_limits<int>::max())
      return NoHazardFound;
 
    // Too many VALUs in intv1?
    int Intv1VALUs = PreExecPos - State.ExecPos;
    if (Intv1VALUs > Intv1plus2MaxVALUs)
      return HazardExpired;
 
    // Too many VALUs in intv1 + intv2
    if (Intv1VALUs + Intv2VALUs > Intv1plus2MaxVALUs)
      return HazardExpired;
 
    return HazardFound;
  };
  auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
    if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
      State.VALUs += 1;
  };
 
  if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
                            std::next(MI->getReverseIterator())))
    return false;
 
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII.get(AMDGPU::S_WAITCNT_DEPCTR))
      .addImm(AMDGPU::DepCtr::encodeFieldVaVdst(0, ST));
 
  return true;
}
 
bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
  if (!ST.hasVALUTransUseHazard())
    return false;
  assert(!ST.hasExtendedWaitCounts());
 
  if (!SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true))
    return false;
 
  SmallSet<Register, 4> SrcVGPRs;
 
  for (const MachineOperand &Use : MI->explicit_uses()) {
    if (Use.isReg() && TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
      SrcVGPRs.insert(Use.getReg());
  }
 
  // Look for the following pattern:
  //   Va <- TRANS VALU
  //   intv
  //   MI Va (WaitState = 0)
  //
  // Where:
  // intv <= 5 VALUs / 1 TRANS
  //
  // If found, insert an appropriate S_WAITCNT_DEPCTR before MI.
 
  const int IntvMaxVALUs = 5;
  const int IntvMaxTRANS = 1;
 
  struct StateType {
    int VALUs = 0;
    int TRANS = 0;
 
    static unsigned getHashValue(const StateType &State) {
      return hash_combine(State.VALUs, State.TRANS);
    }
    static bool isEqual(const StateType &LHS, const StateType &RHS) {
      return LHS.VALUs == RHS.VALUs && LHS.TRANS == RHS.TRANS;
    }
  };
 
  StateType State;
 
  // This overloads expiry testing with all the hazard detection
  auto IsHazardFn = [&, this](StateType &State, const MachineInstr &I) {
    // Too many VALU states have passed
    if (State.VALUs > IntvMaxVALUs || State.TRANS > IntvMaxTRANS)
      return HazardExpired;
 
    // Instructions which cause va_vdst==0 expire hazard
    if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) ||
        SIInstrInfo::isEXP(I) ||
        (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
         AMDGPU::DepCtr::decodeFieldVaVdst(I.getOperand(0).getImm()) == 0))
      return HazardExpired;
 
    // Track registers writes
    if (SIInstrInfo::isTRANS(I)) {
      for (Register Src : SrcVGPRs) {
        if (I.modifiesRegister(Src, &TRI)) {
          return HazardFound;
        }
      }
    }
 
    return NoHazardFound;
  };
  auto UpdateStateFn = [](StateType &State, const MachineInstr &MI) {
    if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
      State.VALUs += 1;
    if (SIInstrInfo::isTRANS(MI))
      State.TRANS += 1;
  };
 
  if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
                            std::next(MI->getReverseIterator())))
    return false;
 
  // Hazard is observed - insert a wait on va_dst counter to ensure hazard is
  // avoided.
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII.get(AMDGPU::S_WAITCNT_DEPCTR))
      .addImm(AMDGPU::DepCtr::encodeFieldVaVdst(0, ST));
 
  return true;
}
 
bool GCNHazardRecognizer::fixVALUTransCoexecutionHazards(MachineInstr *MI) {
  if (!ST.hasTransCoexecutionHazard() || // Coexecution disabled.
      !SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true) ||
      SIInstrInfo::isTRANS(*MI))
    return false;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
 
  auto IsTransHazardFn = [MI, TII, TRI](const MachineInstr &I) {
    if (!SIInstrInfo::isTRANS(I))
      return false;
 
    // RAW: Trans(I) writes, VALU(MI) reads.
    Register TransDef = TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg();
    for (const MachineOperand &ValuUse : MI->explicit_uses()) {
      if (ValuUse.isReg() && TRI->regsOverlap(TransDef, ValuUse.getReg()))
        return true;
    }
 
    auto *ValuDst = TII->getNamedOperand(*MI, AMDGPU::OpName::vdst);
    if (!ValuDst || !ValuDst->isReg())
      return false;
 
    // WAR: Trans(I) reads, VALU(MI) writes.
    Register ValuDef = ValuDst->getReg();
    for (const MachineOperand &TransUse : I.explicit_uses()) {
      if (TransUse.isReg() && TRI->regsOverlap(ValuDef, TransUse.getReg()))
        return true;
    }
 
    return false;
  };
 
  auto IsExpiredFn = [](const MachineInstr &I, int) {
    return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true);
  };
 
  const int HasVALU = std::numeric_limits<int>::max();
  if (::getWaitStatesSince(IsTransHazardFn, MI, IsExpiredFn) == HasVALU)
    return false;
 
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32));
  return true;
}
 
bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
  if (!SIInstrInfo::isWMMA(*MI) && !SIInstrInfo::isSWMMAC(*MI))
    return false;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
 
  auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) {
    if (!SIInstrInfo::isWMMA(I) && !SIInstrInfo::isSWMMAC(I))
      return false;
 
    // Src0(matrix A) or Src1(matrix B) of the current wmma instruction overlaps
    // with the dest(matrix D) of the previous wmma.
    const Register CurSrc0Reg =
        TII->getNamedOperand(*MI, AMDGPU::OpName::src0)->getReg();
    const Register CurSrc1Reg =
        TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg();
 
    const Register PrevDstReg =
        TII->getNamedOperand(I, AMDGPU::OpName::vdst)->getReg();
 
    if (TRI->regsOverlap(PrevDstReg, CurSrc0Reg) ||
        TRI->regsOverlap(PrevDstReg, CurSrc1Reg)) {
      return true;
    }
 
    // GFX12+ allows overlap of matrix C with PrevDstReg (hardware will stall)
    // but Index can't overlap with PrevDstReg.
    if (AMDGPU::isGFX12Plus(ST)) {
      if (SIInstrInfo::isSWMMAC(*MI)) {
        const Register CurIndex =
            TII->getNamedOperand(*MI, AMDGPU::OpName::src2)->getReg();
        if (TRI->regsOverlap(PrevDstReg, CurIndex))
          return true;
      }
      return false;
    }
 
    return false;
  };
 
  auto IsExpiredFn = [](const MachineInstr &I, int) {
    return SIInstrInfo::isVALU(I, /*AllowLDSDMA=*/true);
  };
 
  if (::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn) ==
      std::numeric_limits<int>::max())
    return false;
 
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::V_NOP_e32));
 
  return true;
}
 
static bool isCoexecutableVALUInst(const MachineInstr &MI) {
  return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false) &&
         !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI);
}
 
// Classify XDL WMMA instructions into co-execution hazard categories
// (Refer to SPG 4.6.12.1), mainly based on instruction latency.
//
// Category 0: WMMA with Latency 8
//   WMMA_*F16, WMMA_*BF16
//   WMMA_*_16X16X128_{FP8,BF8}
//   WMMA_*F8F6F4 if SRCA & SRCB are not both F4
//
// Category 1: WMMA Latency 16
//   WMMA_IU8
//
// Category 2: SWMMAC with Latency 8
//   SWMMAC_*F16, SWMMAC_*BF16,
//   SWMMAC_*FP8FP8
//   SWMMAC_*BF8FP8
//   SWMMAC_*FP8BF8
//   SWMMAC_*BF8BF8
//
// Category 3: SWMMAC with Latency 16
//   SWMMAC_IU8
//
// Category 4: 16 Pass GFX1251 WMMA with latency 16
//   V_WMMA_*_16X16X32_{F16,BF16}
//   V_WMMA_{F32,F16}_16X16X64_{FP8,BF8}*
//   V_WMMA_F32_16x16x128_F8F6F4 (F4 only)
//   V_SWMMAC_*_16X16X64_{F16,BF16}
//   V_SWMMAC_{F32,F16}_16X16X128_{FP8,BF8}*
//
// Category 5: 32 Pass GFX1251 WMMA with latency 32
//   V_WMMA_F32_16x16x128_F8F6F4 (not all F4)
//   V_WMMA_{F32,F16}_16X16X128_{FP8,BF8}*
//   V_WMMA_F32_32X16X128_F4
//   V_WMMA_I32_16X16X64_IU8
//   V_WMMA_I32_16X16X64_IU8
//
// Category 6: gfx1250 WMMA with Latency 4 (one co-execution slot)
//   WMMA_*_16X16X64_{FP8,BF8}
//   WMMA_*F8F6F4 if SRCA & SRCB are both F4
static unsigned getWMMAHazardInstInCategory(const MachineInstr &MI,
                                            const SIInstrInfo *TII,
                                            const TargetSchedModel &SchedModel,
                                            const GCNSubtarget &ST) {
  assert(TII->isXDLWMMA(MI) && "must be xdl wmma");
  bool IsSWMMAC = SIInstrInfo::isSWMMAC(MI);
  bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
  unsigned Category = 0;
 
  unsigned Latency = SchedModel.computeInstrLatency(&MI);
  switch (Latency) {
  case 4:
    // Dense 4-cycle WMMA (gfx1250 16x16x64 FP8/BF8 and f8f6f4 with both
    // inputs F4). One co-execution slot; there is no 4-cycle SWMMAC.
    assert(!IsSWMMAC && "no 4-cycle SWMMAC expected");
    Category = 6;
    break;
  case 8:
    Category = IsSWMMAC ? 2 : 0;
    break;
  case 16:
    Category = IsLowestRateWMMA ? 4 : (IsSWMMAC ? 3 : 1);
    break;
  case 32:
    assert(IsLowestRateWMMA && "latency 32 is not expected");
    Category = 5;
    break;
  default:
    llvm_unreachable("unexpected xdl wmma latency");
  } // end switch.
 
  return Category;
}
 
int GCNHazardRecognizer::checkWMMACoexecutionHazards(MachineInstr *MI) const {
  if (!ST.hasWMMACoexecutionHazards())
    return 0;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  if (!TII->isXDLWMMA(*MI) && !isCoexecutableVALUInst(*MI))
    return 0;
 
  // WaitStates here is the number of V_NOPs or unrelated VALU instructions must
  // be in between the first WMMA and the second instruction to cover the hazard
  // (WMMAWaitStates if the second is also a WMMA, VALUWaitStates if the second
  // is a VALU). Refer to SPG 4.6.12.1. "Requirements for WMMA data hazards" for
  // numbers, which depends on the category of the first WMMA.
  const int WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
  const int VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
  unsigned Category = 0;
 
  auto IsWMMAHazardFn = [MI, TII, &Category, this](const MachineInstr &I) {
    if (!TII->isXDLWMMA(I))
      return false;
 
    Category = getWMMAHazardInstInCategory(I, TII, TSchedModel, ST);
    return hasWMMAToWMMARegOverlap(I, *MI);
  };
 
  auto IsVALUHazardFn = [MI, TII, &Category, this](const MachineInstr &I) {
    if (!TII->isXDLWMMA(I))
      return false;
 
    Category = getWMMAHazardInstInCategory(I, TII, TSchedModel, ST);
    return hasWMMAToVALURegOverlap(I, *MI);
  };
 
  int WaitStatesNeeded = -1;
  int ExistingVALUs = 0; // Existing number of VALU ops in between.
  bool IsLowestRateWMMA = ST.hasGFX125xLowestRateWMMA();
 
  // getWaitStatesSinceVALU checks for a hazard between instruction 'I' and
  // 'MI':
  // - If a hazard exists: returns the number of VALUs in between and sets
  //   'Category' via IsWMMAHazardFn/IsVALUHazardFn for instruction 'I'.
  // - If no hazard exists: returns INT_MAX, making WaitStatesNeeded negative,
  //   so no V_NOP insertion is needed.
  if (TII->isXDLWMMA(*MI)) {
    // Maximum of MMAWaitStates.
    const int WMMAWaitsLimit = IsLowestRateWMMA ? 17 : 9;
    ExistingVALUs = getWaitStatesSinceVALU(IsWMMAHazardFn, WMMAWaitsLimit);
    WaitStatesNeeded = WMMAWaitStates[Category] - ExistingVALUs;
  } else { // Must be a co-executable VALU.
           // Maximum of VALUWaitStates.
    const int VALUWaitsLimit = IsLowestRateWMMA ? 16 : 8;
    ExistingVALUs = getWaitStatesSinceVALU(IsVALUHazardFn, VALUWaitsLimit);
    WaitStatesNeeded = VALUWaitStates[Category] - ExistingVALUs;
  }
 
  return WaitStatesNeeded;
}
 
bool GCNHazardRecognizer::hasWMMAToWMMARegOverlap(
    const MachineInstr &WMMA, const MachineInstr &MI) const {
  Register D0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::vdst)->getReg();
  Register A1 = TII.getNamedOperand(MI, AMDGPU::OpName::src0)->getReg();
  Register B1 = TII.getNamedOperand(MI, AMDGPU::OpName::src1)->getReg();
 
  // WMMA0 writes (D0), WMMA1 reads (A1/B1/Idx1).
  if (TRI.regsOverlap(D0, A1) || TRI.regsOverlap(D0, B1))
    return true;
 
  if (SIInstrInfo::isSWMMAC(MI)) {
    Register Idx1 = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
    if (TRI.regsOverlap(D0, Idx1))
      return true;
  }
  return false;
}
 
bool GCNHazardRecognizer::hasWMMAToVALURegOverlap(
    const MachineInstr &WMMA, const MachineInstr &MI) const {
  // WMMA writes, VALU reads.
  Register D0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::vdst)->getReg();
  for (const MachineOperand &ValuUse : MI.explicit_uses()) {
    if (ValuUse.isReg() && TRI.regsOverlap(D0, ValuUse.getReg()))
      return true;
  }
 
  // WMMA reads or writes, VALU writes.
  Register A0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::src0)->getReg();
  Register B0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::src1)->getReg();
  SmallVector<Register, 4> WMMARegs({D0, A0, B0});
 
  if (SIInstrInfo::isSWMMAC(WMMA)) {
    Register Idx0 = TII.getNamedOperand(WMMA, AMDGPU::OpName::src2)->getReg();
    WMMARegs.push_back(Idx0);
  }
 
  for (const MachineOperand &ValuDef : MI.defs()) {
    Register VDstReg = ValuDef.getReg();
    for (Register WMMAReg : WMMARegs) {
      if (TRI.regsOverlap(VDstReg, WMMAReg))
        return true;
    }
  }
  return false;
}
 
bool GCNHazardRecognizer::isCoexecutionHazardFor(const MachineInstr &I,
                                                 const MachineInstr &MI) const {
  // I is the potential WMMA hazard source, MI is the instruction being checked
  // for hazard.
  if (!TII.isXDLWMMA(I))
    return false;
 
  // Dispatch based on MI type
  if (TII.isXDLWMMA(MI))
    return hasWMMAToWMMARegOverlap(I, MI);
  if (isCoexecutableVALUInst(MI))
    return hasWMMAToVALURegOverlap(I, MI);
 
  return false;
}
 
bool GCNHazardRecognizer::hasWMMAHazardInLoop(MachineLoop *L, MachineInstr *MI,
                                              bool IncludeSubloops) {
  // Scan loop for any WMMA that hazards MI.
  // TODO: Avoid full loop scan when WMMA is beyond VALU distance.
  for (MachineBasicBlock *MBB : L->getBlocks()) {
    if (!IncludeSubloops && MLI->getLoopFor(MBB) != L)
      continue;
    for (MachineInstr &I : *MBB) {
      if (&I == MI)
        continue;
      if (isCoexecutionHazardFor(I, *MI))
        return true;
    }
  }
  return false;
}
 
bool GCNHazardRecognizer::tryHoistWMMAVnopsFromLoop(MachineInstr *MI,
                                                    int WaitStatesNeeded) {
  if (!MLI)
    return false;
 
  MachineLoop *L = MLI->getLoopFor(MI->getParent());
  if (!L) {
    ++NumWMMAHoistingBailed;
    return false;
  }
 
  // If innermost loop has WMMA hazard, we can't hoist at all
  if (hasWMMAHazardInLoop(L, MI)) {
    ++NumWMMAHoistingBailed;
    return false;
  }
 
  // Find outermost loop with no internal hazard
  MachineLoop *TargetLoop = L;
  while (MachineLoop *Parent = TargetLoop->getParentLoop()) {
    if (hasWMMAHazardInLoop(Parent, MI, false))
      break;             // Parent has hazard in its own blocks, stop here
    TargetLoop = Parent; // Safe to hoist further out
  }
 
  // Need valid preheader to insert V_NOPs
  MachineBasicBlock *Preheader = TargetLoop->getLoopPreheader();
  if (!Preheader) {
    ++NumWMMAHoistingBailed;
    return false;
  }
 
  LLVM_DEBUG(dbgs() << "WMMA V_NOP Hoisting: Moving " << WaitStatesNeeded
                    << " V_NOPs from loop to " << printMBBReference(*Preheader)
                    << "\n");
 
  emitVNops(*Preheader, Preheader->getFirstTerminator(), WaitStatesNeeded,
            /*IsHoisting=*/true);
  NumWMMANopsHoisted += WaitStatesNeeded;
  return true;
}
 
bool GCNHazardRecognizer::fixWMMACoexecutionHazards(MachineInstr *MI) {
  int WaitStatesNeeded = checkWMMACoexecutionHazards(MI);
  if (WaitStatesNeeded <= 0)
    return false;
 
  if (EnableWMMAVnopHoisting && tryHoistWMMAVnopsFromLoop(MI, WaitStatesNeeded))
    return true;
 
  emitVNops(*MI->getParent(), MI->getIterator(), WaitStatesNeeded);
  return true;
}
 
bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) {
  if (!ST.hasShift64HighRegBug())
    return false;
  assert(!ST.hasExtendedWaitCounts());
 
  switch (MI->getOpcode()) {
  default:
    return false;
  case AMDGPU::V_LSHLREV_B64_e64:
  case AMDGPU::V_LSHRREV_B64_e64:
  case AMDGPU::V_ASHRREV_I64_e64:
    break;
  }
 
  MachineOperand *Amt = TII.getNamedOperand(*MI, AMDGPU::OpName::src0);
  if (!Amt->isReg())
    return false;
 
  Register AmtReg = Amt->getReg();
  const MachineRegisterInfo &MRI = MF.getRegInfo();
  // Check if this is a last VGPR in the allocation block.
  if (!TRI.isVGPR(MRI, AmtReg) || ((AmtReg - AMDGPU::VGPR0) & 7) != 7)
    return false;
 
  if (AmtReg != AMDGPU::VGPR255 && MRI.isPhysRegUsed(AmtReg + 1))
    return false;
 
  assert(ST.needsAlignedVGPRs());
  static_assert(AMDGPU::VGPR0 + 1 == AMDGPU::VGPR1);
 
  const DebugLoc &DL = MI->getDebugLoc();
  MachineBasicBlock *MBB = MI->getParent();
  MachineOperand *Src1 = TII.getNamedOperand(*MI, AMDGPU::OpName::src1);
 
  // In:
  //
  //    Dst = shiftrev64 Amt, Src1
  //
  // if  Dst!=Src1 then avoid the bug with:
  //
  //    Dst.sub0 = Amt
  //    Dst = shift64 Dst.sub0, Src1
 
  Register DstReg = MI->getOperand(0).getReg();
  if (!Src1->isReg() || Src1->getReg() != DstReg) {
    Register DstLo = TRI.getSubReg(DstReg, AMDGPU::sub0);
    runOnInstruction(
        BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), DstLo).add(*Amt));
    Amt->setReg(DstLo);
    Amt->setIsKill(true);
    return true;
  }
 
  bool Overlapped = MI->modifiesRegister(AmtReg, &TRI);
  Register NewReg;
  for (MCRegister Reg : Overlapped ? AMDGPU::VReg_64_Align2RegClass
                                   : AMDGPU::VGPR_32RegClass) {
    if (!MI->modifiesRegister(Reg, &TRI) && !MI->readsRegister(Reg, &TRI)) {
      NewReg = Reg;
      break;
    }
  }
 
  Register NewAmt = Overlapped ? (Register)TRI.getSubReg(NewReg, AMDGPU::sub1)
                               : NewReg;
  Register NewAmtLo;
 
  if (Overlapped)
    NewAmtLo = TRI.getSubReg(NewReg, AMDGPU::sub0);
 
  // Insert a full wait count because found register might be pending a wait.
  BuildMI(*MBB, MI, DL, TII.get(AMDGPU::S_WAITCNT))
      .addImm(0);
 
  // Insert V_SWAP_B32 instruction(s) and run hazard recognizer on them.
  if (Overlapped)
    runOnInstruction(
        BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmtLo)
            .addDef(AmtReg - 1)
            .addReg(AmtReg - 1, RegState::Undef)
            .addReg(NewAmtLo, RegState::Undef));
  runOnInstruction(BuildMI(*MBB, MI, DL, TII.get(AMDGPU::V_SWAP_B32), NewAmt)
                       .addDef(AmtReg)
                       .addReg(AmtReg, RegState::Undef)
                       .addReg(NewAmt, RegState::Undef));
 
  // Instructions emitted after the current instruction will be processed by the
  // parent loop of the hazard recognizer in a natural way.
  BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32),
          AmtReg)
      .addDef(NewAmt)
      .addReg(NewAmt)
      .addReg(AmtReg);
  if (Overlapped)
    BuildMI(*MBB, std::next(MI->getIterator()), DL, TII.get(AMDGPU::V_SWAP_B32),
            AmtReg - 1)
        .addDef(NewAmtLo)
        .addReg(NewAmtLo)
        .addReg(AmtReg - 1);
 
  // Re-running hazard recognizer on the modified instruction is not necessary,
  // inserted V_SWAP_B32 has already both read and write new registers so
  // hazards related to these register has already been handled.
  Amt->setReg(NewAmt);
  Amt->setIsKill(false);
  // We do not update liveness, so verifier may see it as undef.
  Amt->setIsUndef();
  if (Overlapped) {
    MI->getOperand(0).setReg(NewReg);
    Src1->setReg(NewReg);
    Src1->setIsKill(false);
    Src1->setIsUndef();
  }
 
  return true;
}
 
int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) const {
  int NSAtoVMEMWaitStates = 1;
 
  if (!ST.hasNSAtoVMEMBug())
    return 0;
 
  if (!SIInstrInfo::isMUBUF(*MI) && !SIInstrInfo::isMTBUF(*MI))
    return 0;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  const auto *Offset = TII->getNamedOperand(*MI, AMDGPU::OpName::offset);
  if (!Offset || (Offset->getImm() & 6) == 0)
    return 0;
 
  auto IsHazardFn = [TII](const MachineInstr &I) {
    if (!SIInstrInfo::isMIMG(I))
      return false;
    const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(I.getOpcode());
    return Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA &&
           TII->getInstSizeInBytes(I) >= 16;
  };
 
  return NSAtoVMEMWaitStates - getWaitStatesSince(IsHazardFn, 1);
}
 
int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(
    MachineInstr *MI) const {
  int FPAtomicToDenormModeWaitStates = 3;
 
  if (!ST.hasFPAtomicToDenormModeHazard())
    return 0;
  assert(!ST.hasExtendedWaitCounts());
 
  if (MI->getOpcode() != AMDGPU::S_DENORM_MODE)
    return 0;
 
  auto IsHazardFn = [](const MachineInstr &I) {
    if (!SIInstrInfo::isVMEM(I))
      return false;
    return SIInstrInfo::isFPAtomic(I);
  };
 
  auto IsExpiredFn = [](const MachineInstr &MI, int WaitStates) {
    if (WaitStates >= 3 || SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
      return true;
 
    return SIInstrInfo::isWaitcnt(MI.getOpcode());
  };
 
  return FPAtomicToDenormModeWaitStates -
         ::getWaitStatesSince(IsHazardFn, MI, IsExpiredFn);
}
 
int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) const {
  assert(SIInstrInfo::isMAI(*MI));
 
  return ST.hasGFX90AInsts() ? checkMAIHazards90A(MI) : checkMAIHazards908(MI);
}
 
int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) const {
  // Early exit if no padding is requested.
  if (MFMAPaddingRatio == 0)
    return 0;
 
  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
  if (!SIInstrInfo::isMFMA(*MI) || MFI->getOccupancy() < 2)
    return 0;
 
  int NeighborMFMALatency = 0;
  auto IsNeighboringMFMA = [&NeighborMFMALatency,
                            this](const MachineInstr &MI) {
    if (!SIInstrInfo::isMFMA(MI))
      return false;
 
    NeighborMFMALatency = this->getMFMAPipelineWaitStates(MI);
    return true;
  };
 
  const int MaxMFMAPipelineWaitStates = 16;
  int WaitStatesSinceNeighborMFMA =
      getWaitStatesSince(IsNeighboringMFMA, MaxMFMAPipelineWaitStates);
 
  int NeighborMFMAPaddingNeeded =
      (NeighborMFMALatency * MFMAPaddingRatio / 100) -
      WaitStatesSinceNeighborMFMA;
 
  return std::max(0, NeighborMFMAPaddingNeeded);
}
 
int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) const {
  int WaitStatesNeeded = 0;
  unsigned Opc = MI->getOpcode();
 
  auto IsVALUFn = [](const MachineInstr &MI) {
    return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) || MI.isInlineAsm();
  };
 
  if (Opc != AMDGPU::V_ACCVGPR_READ_B32_e64) { // MFMA or v_accvgpr_write
    const int LegacyVALUWritesVGPRWaitStates = 2;
    const int VALUWritesExecWaitStates = 4;
    const int MaxWaitStates = 4;
 
    int WaitStatesNeededForUse = VALUWritesExecWaitStates -
      getWaitStatesSinceDef(AMDGPU::EXEC, IsVALUFn, MaxWaitStates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
    if (WaitStatesNeeded < MaxWaitStates) {
      for (const MachineOperand &Use : MI->explicit_uses()) {
        const int MaxWaitStates = 2;
 
        if (!Use.isReg() || !TRI.isVGPR(MF.getRegInfo(), Use.getReg()))
          continue;
 
        int WaitStatesNeededForUse = LegacyVALUWritesVGPRWaitStates -
          getWaitStatesSinceDef(Use.getReg(), IsVALUFn, MaxWaitStates);
        WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
        if (WaitStatesNeeded == MaxWaitStates)
          break;
      }
    }
  }
 
  for (const MachineOperand &Op : MI->explicit_operands()) {
    if (!Op.isReg() || !TRI.isAGPR(MF.getRegInfo(), Op.getReg()))
      continue;
 
    if (Op.isDef() && Opc != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
      continue;
 
    const int MFMAWritesAGPROverlappedSrcABWaitStates = 4;
    const int MFMAWritesAGPROverlappedSrcCWaitStates = 2;
    const int MFMA4x4WritesAGPRAccVgprReadWaitStates = 4;
    const int MFMA16x16WritesAGPRAccVgprReadWaitStates = 10;
    const int MFMA32x32WritesAGPRAccVgprReadWaitStates = 18;
    const int MFMA4x4WritesAGPRAccVgprWriteWaitStates = 1;
    const int MFMA16x16WritesAGPRAccVgprWriteWaitStates = 7;
    const int MFMA32x32WritesAGPRAccVgprWriteWaitStates = 15;
    const int MaxWaitStates = 18;
    Register Reg = Op.getReg();
    unsigned HazardDefLatency = 0;
 
    auto IsOverlappedMFMAFn = [Reg, &HazardDefLatency,
                               this](const MachineInstr &MI) {
      if (!SIInstrInfo::isMFMA(MI))
        return false;
      Register DstReg = MI.getOperand(0).getReg();
      if (DstReg == Reg)
        return false;
      HazardDefLatency =
          std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
      return TRI.regsOverlap(DstReg, Reg);
    };
 
    int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn,
                                                   MaxWaitStates);
    int NeedWaitStates = MFMAWritesAGPROverlappedSrcABWaitStates;
    int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
    int OpNo = Op.getOperandNo();
    if (OpNo == SrcCIdx) {
      NeedWaitStates = MFMAWritesAGPROverlappedSrcCWaitStates;
    } else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64) {
      switch (HazardDefLatency) {
      case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprReadWaitStates;
               break;
      case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprReadWaitStates;
               break;
      case 16: [[fallthrough]];
      default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprReadWaitStates;
               break;
      }
    } else if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
      switch (HazardDefLatency) {
      case 2:  NeedWaitStates = MFMA4x4WritesAGPRAccVgprWriteWaitStates;
               break;
      case 8:  NeedWaitStates = MFMA16x16WritesAGPRAccVgprWriteWaitStates;
               break;
      case 16: [[fallthrough]];
      default: NeedWaitStates = MFMA32x32WritesAGPRAccVgprWriteWaitStates;
               break;
      }
    }
 
    int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
    if (WaitStatesNeeded == MaxWaitStates)
      return WaitStatesNeeded; // Early exit.
 
    auto IsAccVgprWriteFn = [Reg, this](const MachineInstr &MI) {
      if (MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
        return false;
      Register DstReg = MI.getOperand(0).getReg();
      return TRI.regsOverlap(Reg, DstReg);
    };
 
    const int AccVGPRWriteMFMAReadSrcCWaitStates = 1;
    const int AccVGPRWriteMFMAReadSrcABWaitStates = 3;
    const int AccVGPRWriteAccVgprReadWaitStates = 3;
    NeedWaitStates = AccVGPRWriteMFMAReadSrcABWaitStates;
    if (OpNo == SrcCIdx)
      NeedWaitStates = AccVGPRWriteMFMAReadSrcCWaitStates;
    else if (Opc == AMDGPU::V_ACCVGPR_READ_B32_e64)
      NeedWaitStates = AccVGPRWriteAccVgprReadWaitStates;
 
    WaitStatesNeededForUse = NeedWaitStates -
      getWaitStatesSinceDef(Reg, IsAccVgprWriteFn, MaxWaitStates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
    if (WaitStatesNeeded == MaxWaitStates)
      return WaitStatesNeeded; // Early exit.
  }
 
  if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64) {
    const int MFMA4x4ReadSrcCAccVgprWriteWaitStates = 0;
    const int MFMA16x16ReadSrcCAccVgprWriteWaitStates = 5;
    const int MFMA32x32ReadSrcCAccVgprWriteWaitStates = 13;
    const int MaxWaitStates = 13;
    Register DstReg = MI->getOperand(0).getReg();
    unsigned HazardDefLatency = 0;
 
    auto IsSrcCMFMAFn = [DstReg, &HazardDefLatency,
                         this](const MachineInstr &MI) {
      if (!SIInstrInfo::isMFMA(MI))
        return false;
      Register Reg = TII.getNamedOperand(MI, AMDGPU::OpName::src2)->getReg();
      HazardDefLatency =
          std::max(HazardDefLatency, TSchedModel.computeInstrLatency(&MI));
      return TRI.regsOverlap(Reg, DstReg);
    };
 
    int WaitStatesSince = getWaitStatesSince(IsSrcCMFMAFn, MaxWaitStates);
    int NeedWaitStates;
    switch (HazardDefLatency) {
    case 2:  NeedWaitStates = MFMA4x4ReadSrcCAccVgprWriteWaitStates;
             break;
    case 8:  NeedWaitStates = MFMA16x16ReadSrcCAccVgprWriteWaitStates;
             break;
    case 16: [[fallthrough]];
    default: NeedWaitStates = MFMA32x32ReadSrcCAccVgprWriteWaitStates;
             break;
    }
 
    int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSince;
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
  }
 
  // Pad neighboring MFMA with noops for better inter-wave performance.
  WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI));
 
  return WaitStatesNeeded;
}
 
/// One MFMA can be written with up to four opcodes that differ only in how vdst
/// and src2 are encoded: both are either AGPRs or VGPRs, and the mac form ties
/// vdst to src2 instead of taking them as separate operands. \returns the AGPR,
/// non-mac opcode, so that every form of the same MFMA maps to one value.
static unsigned getMFMANonMacAGPRFormOp(unsigned Opc) {
  if (int NonMacOp = AMDGPU::getMFMAEarlyClobberOp(Opc); NonMacOp != -1)
    Opc = NonMacOp;
  if (int AGPROp = AMDGPU::getAGPRFormOp(Opc); AGPROp != -1)
    Opc = AGPROp;
  return Opc;
}
 
/// \returns true if \p Opc0 and \p Opc1 are the same MFMA, ignoring the mac
/// form and whether vdst/src2 are AGPRs or VGPRs.
static bool isSameMFMA(unsigned Opc0, unsigned Opc1) {
  return getMFMANonMacAGPRFormOp(Opc0) == getMFMANonMacAGPRFormOp(Opc1);
}
 
static int
GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(int NumPasses,
                                                              bool IsGFX950) {
  // xdl def cycles | gfx940 | gfx950
  // 2 pass         |  3        4
  // 4 pass         |  5        6
  // 8 pass         |  9        10
  // 16 pass        |  17       18
  return NumPasses + 1 + IsGFX950;
}
 
static int
GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(int NumPasses,
                                                              bool IsGFX950) {
  // xdl def cycles | gfx940 | gfx950
  // 2 pass         |  3        3
  // 4 pass         |  5        6
  // 8 pass         |  9        10
  // 16 pass        |  17       18
  return NumPasses + 1 + (NumPasses != 2 && IsGFX950);
}
 
static int
GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) {
  // 2 pass -> 2
  // 4 pass -> 4
  // 8 pass -> 8
  // 16 pass -> 16
  return NumPasses;
}
 
static int
GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) {
  // 2 pass -> 4
  // 4 pass -> 6
  // 8 pass -> 10
  // 16 pass -> 18
  return NumPasses + 2;
}
 
static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses,
                                                                bool IsGFX950) {
  // xdl def cycles | gfx942 | gfx950
  // 2 pass         |  5        5
  // 4 pass         |  7        8
  // 8 pass         |  11       12
  // 16 pass        |  19       20
  return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
}
 
int GCNHazardRecognizer::getMFMAOverlappedSrcCWaitStates(
    const MachineInstr *Reader, const MachineInstr *Writer) const {
  constexpr int SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates = 2;
  constexpr int SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates = 8;
  constexpr int SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates = 16;
  constexpr int SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates = 3;
  constexpr int SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates = 9;
  constexpr int SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates = 17;
  constexpr int DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 9;
  constexpr int GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates = 17;
  constexpr int DMFMA4x4WritesVGPROverlappedSrcCWaitStates = 4;
 
  // An XDL read of a non-XDL result needs no wait states. DGEMM is never XDL,
  // so this also covers the f64 writers handled below.
  if (TII.isXDL(*Reader) && !TII.isXDL(*Writer))
    return 0;
 
  switch (Writer->getOpcode()) {
  case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
  case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
  case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
  case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
    return ST.hasGFX950Insts()
               ? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
               : DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
  case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
  case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
    return DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
  default:
    break;
  }
 
  int NumPasses = TSchedModel.computeInstrLatency(Writer);
  if (ST.hasGFX940Insts()) {
    if (!TII.isXDL(*Writer))
      return GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(
          NumPasses);
    return TII.isXDL(*Reader)
               ? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(
                     NumPasses, ST.hasGFX950Insts())
               : GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(
                     NumPasses, ST.hasGFX950Insts());
  }
 
  bool IsDGEMM = SIInstrInfo::isDGEMM(Reader->getOpcode());
  switch (NumPasses) {
  case 2:
    return IsDGEMM ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
                   : SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
  case 8:
    return IsDGEMM ? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
                   : SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
  case 16:
    return IsDGEMM ? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
                   : SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
  default:
    llvm_unreachable("unexpected number of passes");
  }
}
 
int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) const {
  int WaitStatesNeeded = 0;
  unsigned Opc = MI->getOpcode();
 
  auto IsLegacyVALUFn = [](const MachineInstr &MI) {
    return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
           !SIInstrInfo::isMFMA(MI);
  };
 
  auto IsLegacyVALUNotDotFn = [](const MachineInstr &MI) {
    return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
           !SIInstrInfo::isMFMA(MI) && !SIInstrInfo::isDOT(MI);
  };
 
  if (!SIInstrInfo::isMFMA(*MI))
    return WaitStatesNeeded;
 
  const int VALUWritesExecWaitStates = 4;
  int WaitStatesNeededForUse = VALUWritesExecWaitStates -
    getWaitStatesSinceDef(AMDGPU::EXEC, IsLegacyVALUFn,
                          VALUWritesExecWaitStates);
  WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
  int SrcCIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
 
  // Loop for both DGEMM and S/HGEMM 2nd instruction.
  for (const MachineOperand &Use : MI->explicit_uses()) {
    const int LegacyVALUNotDotWritesVGPRWaitStates = 2;
    const int SMFMA4x4WritesVGPROverlappedSrcABWaitStates = 5;
    const int SMFMA16x16WritesVGPROverlappedSrcABWaitStates = 11;
    const int SMFMA32x32WritesVGPROverlappedSrcABWaitStates = 19;
    const int DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates = 6;
    const int DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 11;
    const int GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates = 19;
    const int DMFMA4x4WritesVGPRFullSrcCWaitStates = 4;
    const int GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates = 2;
    const int MaxWaitStates =
        GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
            16, ST.hasGFX950Insts());
 
    if (!Use.isReg())
      continue;
    Register Reg = Use.getReg();
    bool FullReg;
    const MachineInstr *MI1;
 
    auto IsOverlappedMFMAFn = [Reg, &FullReg, &MI1,
                               this](const MachineInstr &MI) {
      if (!SIInstrInfo::isMFMA(MI))
        return false;
      Register DstReg = MI.getOperand(0).getReg();
      FullReg = (DstReg == Reg);
      MI1 = &MI;
      return TRI.regsOverlap(DstReg, Reg);
    };
 
    WaitStatesNeededForUse = LegacyVALUNotDotWritesVGPRWaitStates -
      getWaitStatesSinceDef(Reg, IsLegacyVALUNotDotFn, MaxWaitStates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
    int NumWaitStates =
        getWaitStatesSinceDef(Reg, IsOverlappedMFMAFn, MaxWaitStates);
    if (NumWaitStates == std::numeric_limits<int>::max())
      continue;
 
    int OpNo = Use.getOperandNo();
    unsigned Opc1 = MI1->getOpcode();
    int NeedWaitStates = 0;
    if (OpNo == SrcCIdx) {
      if (!SIInstrInfo::isDGEMM(Opc) &&
          (!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opc1))) {
        NeedWaitStates = 0;
      } else if (FullReg) {
        if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
             Opc == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64) &&
            (Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
             Opc1 == AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64))
          NeedWaitStates = DMFMA4x4WritesVGPRFullSrcCWaitStates;
        else if (ST.hasGFX940Insts() &&
                 TSchedModel.computeInstrLatency(MI1) == 2)
          NeedWaitStates = GFX940_SMFMA4x4WritesVGPRFullSrcCWaitStates;
 
        // The accumulator forwarding path that allows zero wait states is only
        // available while the chain stays on a single MFMA. Two different MFMAs
        // sharing an accumulator need the wait states of a partial overlap.
        if (ST.hasGFX940Insts() && !isSameMFMA(Opc, Opc1)) {
          NeedWaitStates = std::max(NeedWaitStates,
                                    getMFMAOverlappedSrcCWaitStates(MI, MI1));
        }
      } else {
        NeedWaitStates = getMFMAOverlappedSrcCWaitStates(MI, MI1);
      }
    } else {
      switch (Opc1) {
      case AMDGPU::V_MFMA_F64_16X16X4F64_e64:
      case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
      case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
      case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
        NeedWaitStates =
            ST.hasGFX950Insts()
                ? GFX950_DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates
                : DMFMA16x16WritesVGPROverlappedMFMASrcABWaitStates;
        break;
      case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
      case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
        NeedWaitStates = DMFMA4x4WritesVGPROverlappedMFMASrcABWaitStates;
        break;
      default:
        int NumPasses = TSchedModel.computeInstrLatency(MI1);
 
        if (ST.hasGFX940Insts()) {
          NeedWaitStates =
              TII.isXDL(*MI1)
                  ? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
                        NumPasses, ST.hasGFX950Insts())
                  : GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
                        NumPasses);
          break;
        }
 
        switch (NumPasses) {
        case 2:
          NeedWaitStates = SMFMA4x4WritesVGPROverlappedSrcABWaitStates;
          break;
        case 4:
          llvm_unreachable("unexpected number of passes for mfma");
        case 8:
          NeedWaitStates = SMFMA16x16WritesVGPROverlappedSrcABWaitStates;
          break;
        case 16:
        default:
          NeedWaitStates = SMFMA32x32WritesVGPROverlappedSrcABWaitStates;
        }
      }
    }
    assert(NeedWaitStates <= MaxWaitStates &&
           "hazard requirement exceeds the scan window");
    if (WaitStatesNeeded >= NeedWaitStates)
      continue;
 
    WaitStatesNeededForUse = NeedWaitStates - NumWaitStates;
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
    if (WaitStatesNeeded == MaxWaitStates)
      break;
  }
 
  // Pad neighboring MFMA with noops for better inter-wave performance.
  WaitStatesNeeded = std::max(WaitStatesNeeded, checkMFMAPadding(MI));
 
  return WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) const {
  // On gfx90a+ relevant hazards are checked in checkMAIVALUHazards()
  if (!ST.hasMAIInsts() || ST.hasGFX90AInsts())
    return 0;
 
  int WaitStatesNeeded = 0;
 
  auto IsAccVgprReadFn = [](const MachineInstr &MI) {
    return MI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64;
  };
 
  for (const MachineOperand &Op : MI->explicit_uses()) {
    if (!Op.isReg() || !TRI.isVGPR(MF.getRegInfo(), Op.getReg()))
      continue;
 
    Register Reg = Op.getReg();
 
    const int AccVgprReadLdStWaitStates = 2;
    const int VALUWriteAccVgprRdWrLdStDepVALUWaitStates = 1;
    const int MaxWaitStates = 2;
 
    int WaitStatesNeededForUse = AccVgprReadLdStWaitStates -
      getWaitStatesSinceDef(Reg, IsAccVgprReadFn, MaxWaitStates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
    if (WaitStatesNeeded == MaxWaitStates)
      return WaitStatesNeeded; // Early exit.
 
    auto IsVALUAccVgprRdWrCheckFn = [Reg, this](const MachineInstr &MI) {
      if (MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64 &&
          MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
        return false;
      auto IsVALUFn = [](const MachineInstr &MI) {
        return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) &&
               !SIInstrInfo::isMAI(MI);
      };
      return getWaitStatesSinceDef(Reg, IsVALUFn, 2 /*MaxWaitStates*/) <
             std::numeric_limits<int>::max();
    };
 
    WaitStatesNeededForUse = VALUWriteAccVgprRdWrLdStDepVALUWaitStates -
      getWaitStatesSince(IsVALUAccVgprRdWrCheckFn, MaxWaitStates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
  }
 
  return WaitStatesNeeded;
}
 
int GCNHazardRecognizer::checkPermlaneHazards(MachineInstr *MI) const {
  assert(!ST.hasVcmpxPermlaneHazard() &&
         "this is a different vcmpx+permlane hazard");
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const SIInstrInfo *TII = ST.getInstrInfo();
 
  auto IsVCmpXWritesExecFn = [TII, TRI](const MachineInstr &MI) {
    return isVCmpXWritesExec(*TII, *TRI, MI);
  };
 
  auto IsVALUFn = [](const MachineInstr &MI) {
    return SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true);
  };
 
  const int VCmpXWritesExecWaitStates = 4;
  const int VALUWritesVDstWaitStates = 2;
  int WaitStatesNeeded = 0;
 
  for (const MachineOperand &Op : MI->explicit_uses()) {
    if (!Op.isReg() || !TRI->isVGPR(MF.getRegInfo(), Op.getReg()))
      continue;
    Register Reg = Op.getReg();
 
    int WaitStatesSinceDef =
        VALUWritesVDstWaitStates -
        getWaitStatesSinceDef(Reg, IsVALUFn,
                              /*MaxWaitStates=*/VALUWritesVDstWaitStates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesSinceDef);
    if (WaitStatesNeeded >= VALUWritesVDstWaitStates)
      break;
  }
 
  int VCmpXHazardWaits =
      VCmpXWritesExecWaitStates -
      getWaitStatesSince(IsVCmpXWritesExecFn, VCmpXWritesExecWaitStates);
 
  WaitStatesNeeded = std::max(WaitStatesNeeded, VCmpXHazardWaits);
  return WaitStatesNeeded;
}
 
static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) {
  // 2 pass -> 4
  // 4 pass -> 6
  // 8 pass -> 10
  // 16 pass -> 18
  return NumPasses + 2;
}
 
static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses,
                                                       bool IsGFX950) {
  // xdl def cycles | gfx942 | gfx950
  // 2 pass         |  5        5
  // 4 pass         |  7        8
  // 8 pass         |  11       12
  // 16 pass        |  19       20
  return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
}
 
static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses,
                                                              bool IsGFX950) {
  // xdl def cycles | gfx942 | gfx950
  // 2 pass         |  5        5
  // 4 pass         |  7        8
  // 8 pass         |  11       12
  // 16 pass        |  19       20
  return NumPasses + 3 + (NumPasses != 2 && IsGFX950);
}
 
static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) {
  // 2 pass -> 4
  // 4 pass -> 6
  // 8 pass -> 10
  // 16 pass -> 18
  return NumPasses + 2;
}
 
int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) const {
  if (!ST.hasGFX90AInsts())
    return 0;
 
  auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
    return SIInstrInfo::isDGEMM(MI.getOpcode());
  };
 
  // This is checked in checkMAIHazards90A()
  if (SIInstrInfo::isMFMA(*MI))
    return 0;
 
  const MachineRegisterInfo &MRI = MF.getRegInfo();
 
  int WaitStatesNeeded = 0;
 
  bool IsMem = SIInstrInfo::isVMEM(*MI) || SIInstrInfo::isDS(*MI);
  bool IsMemOrExport = IsMem || SIInstrInfo::isEXP(*MI);
  bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
 
  const MachineInstr *MFMA = nullptr;
  unsigned Reg;
  auto IsMFMAWriteFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
    if (!SIInstrInfo::isMFMA(MI) ||
        !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
      return false;
    MFMA = &MI;
    return true;
  };
 
  const MachineInstr *DOT = nullptr;
  auto IsDotWriteFn = [&Reg, &DOT, this](const MachineInstr &MI) {
    if (!SIInstrInfo::isDOT(MI) ||
        !TRI.regsOverlap(MI.getOperand(0).getReg(), Reg))
      return false;
    DOT = &MI;
    return true;
  };
 
  bool DGEMMAfterVALUWrite = false;
  auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
    // Found DGEMM on reverse traversal to def.
    if (SIInstrInfo::isDGEMM(MI.getOpcode()))
      DGEMMAfterVALUWrite = true;
 
    // Only hazard if register is defined by a VALU and a DGEMM is found after
    // after the def.
    if (!TII.isVALU(MI, /*AllowLDSDMA=*/true) || !DGEMMAfterVALUWrite)
      return false;
 
    return true;
  };
 
  int SrcCIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(),
                                           AMDGPU::OpName::src2);
 
  if (IsMemOrExport || IsVALU) {
    const int SMFMA4x4WriteVgprVALUMemExpReadWaitStates = 5;
    const int SMFMA16x16WriteVgprVALUMemExpReadWaitStates = 11;
    const int SMFMA32x32WriteVgprVALUMemExpReadWaitStates = 19;
    const int DMFMA4x4WriteVgprMemExpReadWaitStates = 9;
    const int DMFMA16x16WriteVgprMemExpReadWaitStates = 18;
    const int DMFMA4x4WriteVgprVALUReadWaitStates = 6;
    const int DMFMA16x16WriteVgprVALUReadWaitStates = 11;
    const int GFX950_DMFMA16x16WriteVgprVALUReadWaitStates = 19;
    const int DotWriteSameDotReadSrcAB = 3;
    const int DotWriteDifferentVALURead = 3;
    const int DMFMABetweenVALUWriteVMEMRead = 2;
    const int MaxWaitStates =
        GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(16,
                                                           ST.hasGFX950Insts());
 
    for (const MachineOperand &Use : MI->explicit_uses()) {
      if (!Use.isReg())
        continue;
      Reg = Use.getReg();
 
      DOT = nullptr;
      int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
                                                     MaxWaitStates);
      if (DOT) {
        int NeedWaitStates = 0;
        if (DOT->getOpcode() == MI->getOpcode()) {
          if (&Use - &MI->getOperand(0) != SrcCIdx)
            NeedWaitStates = DotWriteSameDotReadSrcAB;
        } else {
          NeedWaitStates = DotWriteDifferentVALURead;
        }
 
        int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
        WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
      }
 
      // Workaround for HW data hazard bug observed only in GFX90A. When there
      // is a DGEMM instruction in-between a VALU and a VMEM instruction it
      // causes the SQ to incorrectly not insert two wait states between the two
      // instructions needed to avoid data hazard.
      if (IsMem && ST.hasGFX90AInsts() && !ST.hasGFX940Insts()) {
        DGEMMAfterVALUWrite = false;
        if (TRI.isVectorRegister(MRI, Reg)) {
          int WaitStatesNeededForUse =
                DMFMABetweenVALUWriteVMEMRead -
                getWaitStatesSinceDef(Reg, IsDGEMMHazard,
                                      DMFMABetweenVALUWriteVMEMRead);
 
          WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
        }
      }
 
      MFMA = nullptr;
      WaitStatesSinceDef =
          getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
      if (!MFMA)
        continue;
 
      unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
      int NumPasses = HazardDefLatency;
      int NeedWaitStates = MaxWaitStates;
 
      if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
        switch (HazardDefLatency) {
        case 4:
          NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
                                         : DMFMA4x4WriteVgprVALUReadWaitStates;
          break;
        case 8:
        case 16:
          NeedWaitStates =
              IsMemOrExport
                  ? DMFMA16x16WriteVgprMemExpReadWaitStates
                  : (ST.hasGFX950Insts()
                         ? GFX950_DMFMA16x16WriteVgprVALUReadWaitStates
                         : DMFMA16x16WriteVgprVALUReadWaitStates);
          break;
        default:
          llvm_unreachable("unexpected dgemm");
        }
      } else if (ST.hasGFX940Insts()) {
        NeedWaitStates =
            TII.isXDL(*MFMA)
                ? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
                      NumPasses, ST.hasGFX950Insts())
                : GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
                      NumPasses);
      } else {
        switch (HazardDefLatency) {
        case 2:
          NeedWaitStates = SMFMA4x4WriteVgprVALUMemExpReadWaitStates;
          break;
        case 8:
          NeedWaitStates = SMFMA16x16WriteVgprVALUMemExpReadWaitStates;
          break;
        case 16:
          NeedWaitStates = SMFMA32x32WriteVgprVALUMemExpReadWaitStates;
          break;
        default:
          llvm_unreachable("unexpected number of passes for mfma");
        }
      }
 
      assert(NeedWaitStates <= MaxWaitStates &&
             "hazard requirement exceeds the scan window");
      int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
      WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
      if (WaitStatesNeeded == MaxWaitStates)
        break;
    }
  }
 
  unsigned Opc = MI->getOpcode();
  const int DMFMAToFMA64WaitStates = 2;
  if ((Opc == AMDGPU::V_FMA_F64_e64 ||
       Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64 ||
       Opc == AMDGPU::V_FMAC_F64_dpp) &&
      WaitStatesNeeded < DMFMAToFMA64WaitStates) {
    int WaitStatesNeededForUse = DMFMAToFMA64WaitStates -
      getWaitStatesSince(IsDGEMMFn, DMFMAToFMA64WaitStates);
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
  }
 
  if (!IsVALU && !IsMemOrExport)
    return WaitStatesNeeded;
 
  for (const MachineOperand &Def : MI->defs()) {
    const int SMFMA4x4WriteVgprVALUWawWaitStates = 5;
    const int SMFMA16x16WriteVgprVALUWawWaitStates = 11;
    const int SMFMA32x32WriteVgprVALUWawWaitStates = 19;
    const int SMFMA4x4ReadVgprVALUWarWaitStates = 1;
    const int GFX940_XDL4PassReadVgprVALUWarWaitStates = 3;
    const int SMFMA16x16ReadVgprVALUWarWaitStates = 7;
    const int SMFMA32x32ReadVgprVALUWarWaitStates = 15;
    const int DMFMA4x4WriteVgprVALUWriteWaitStates = 6;
    const int DMFMA16x16WriteVgprVALUWriteWaitStates = 11;
    const int DotWriteDifferentVALUWrite = 3;
    const int MaxWaitStates =
        GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(16, ST.hasGFX950Insts());
    const int MaxWarWaitStates = 15;
 
    Reg = Def.getReg();
 
    DOT = nullptr;
    int WaitStatesSinceDef = getWaitStatesSinceDef(Reg, IsDotWriteFn,
                                                   MaxWaitStates);
    if (DOT && DOT->getOpcode() != MI->getOpcode())
      WaitStatesNeeded = std::max(WaitStatesNeeded, DotWriteDifferentVALUWrite -
                                                    WaitStatesSinceDef);
 
    MFMA = nullptr;
    WaitStatesSinceDef =
        getWaitStatesSinceDef(Reg, IsMFMAWriteFn, MaxWaitStates);
    if (MFMA) {
      int NeedWaitStates = MaxWaitStates;
      int NumPasses = TSchedModel.computeInstrLatency(MFMA);
 
      if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
        switch (NumPasses) {
        case 4:
          NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
          break;
        case 8:
        case 16:
          NeedWaitStates = DMFMA16x16WriteVgprVALUWriteWaitStates;
          break;
        default:
          llvm_unreachable("unexpected number of cycles for dgemm");
        }
      } else if (ST.hasGFX940Insts()) {
        NeedWaitStates =
            TII.isXDL(*MFMA)
                ? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
                      NumPasses, ST.hasGFX950Insts())
                : GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
      } else {
        switch (NumPasses) {
        case 2:
          NeedWaitStates = SMFMA4x4WriteVgprVALUWawWaitStates;
          break;
        case 8:
          NeedWaitStates = SMFMA16x16WriteVgprVALUWawWaitStates;
          break;
        case 16:
          NeedWaitStates = SMFMA32x32WriteVgprVALUWawWaitStates;
          break;
        default:
          llvm_unreachable("Unexpected number of passes for mfma");
        }
      }
 
      assert(NeedWaitStates <= MaxWaitStates &&
             "hazard requirement exceeds the scan window");
      int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceDef;
      WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
      if (WaitStatesNeeded == MaxWaitStates)
        break;
    }
 
    auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
      if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(MI.getOpcode()) ||
          !MI.readsRegister(Reg, &TRI))
        return false;
 
      if (ST.hasGFX940Insts() && !TII.isXDL(MI))
        return false;
 
      const MachineOperand *SrcC =
          TII.getNamedOperand(MI, AMDGPU::OpName::src2);
      assert(SrcC);
      if (!SrcC->isReg() || !TRI.regsOverlap(SrcC->getReg(), Reg))
        return false;
 
      MFMA = &MI;
      return true;
    };
 
    MFMA = nullptr;
    int WaitStatesSinceUse = getWaitStatesSince(IsSMFMAReadAsCFn,
                                                MaxWarWaitStates);
    if (!MFMA)
      continue;
 
    unsigned HazardDefLatency = TSchedModel.computeInstrLatency(MFMA);
    int NeedWaitStates = MaxWaitStates;
    switch (HazardDefLatency) {
    case 2:  NeedWaitStates = SMFMA4x4ReadVgprVALUWarWaitStates;
             break;
    case 4:  assert(ST.hasGFX940Insts());
             NeedWaitStates = GFX940_XDL4PassReadVgprVALUWarWaitStates;
             break;
    case 8:  NeedWaitStates = SMFMA16x16ReadVgprVALUWarWaitStates;
             break;
    case 16: [[fallthrough]];
    default: NeedWaitStates = SMFMA32x32ReadVgprVALUWarWaitStates;
             break;
    }
 
    int WaitStatesNeededForUse = NeedWaitStates - WaitStatesSinceUse;
    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
  }
 
  return WaitStatesNeeded;
}
 
bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) const {
  if (!SU->isInstr())
    return false;
 
  const MachineInstr *MAI = nullptr;
 
  auto IsMFMAFn = [&MAI](const MachineInstr &MI) {
    MAI = nullptr;
    if (SIInstrInfo::isMFMA(MI))
      MAI = &MI;
    return MAI != nullptr;
  };
 
  MachineInstr *MI = SU->getInstr();
  if (IsMFMAFn(*MI)) {
    int W = getWaitStatesSince(IsMFMAFn, 16);
    if (MAI)
      return W < (int)TSchedModel.computeInstrLatency(MAI);
  }
 
  return false;
}
 
// Adjust global offsets for instructions bundled with S_GETPC_B64 after
// insertion of a new instruction.
static void updateGetPCBundle(MachineInstr *NewMI) {
  if (!NewMI->isBundled())
    return;
 
  // Find start of bundle.
  auto I = NewMI->getIterator();
  while (I->isBundledWithPred())
    I--;
  if (I->isBundle())
    I++;
 
  // Bail if this is not an S_GETPC bundle.
  if (I->getOpcode() != AMDGPU::S_GETPC_B64)
    return;
 
  // Update offsets of any references in the bundle.
  const unsigned NewBytes = 4;
  assert(NewMI->getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
         "Unexpected instruction insertion in bundle");
  auto NextMI = std::next(NewMI->getIterator());
  auto End = NewMI->getParent()->end();
  while (NextMI != End && NextMI->isBundledWithPred()) {
    for (auto &Operand : NextMI->operands()) {
      if (Operand.isGlobal())
        Operand.setOffset(Operand.getOffset() + NewBytes);
    }
    NextMI++;
  }
}
 
bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) {
  if (!ST.hasVALUMaskWriteHazard())
    return false;
  assert(!ST.hasExtendedWaitCounts());
 
  if (!ST.isWave64())
    return false;
 
  const bool IsSALU = SIInstrInfo::isSALU(*MI);
  const bool IsVALU = SIInstrInfo::isVALU(*MI, /*AllowLDSDMA=*/true);
  if (!IsSALU && !IsVALU)
    return false;
 
  // The hazard sequence is three instructions:
  //   1. VALU reads SGPR as mask
  //   2. VALU/SALU writes SGPR
  //   3. VALU/SALU reads SGPR
  // The hazard can expire if the distance between 2 and 3 is sufficient,
  // or (2) is VALU and (3) is SALU.
  // In practice this happens <10% of the time, hence always assume the hazard
  // exists if (1) and (2) are present to avoid searching all SGPR reads.
 
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const MachineRegisterInfo &MRI = MF.getRegInfo();
 
  auto IgnoreableSGPR = [](const Register Reg) {
    switch (Reg) {
    case AMDGPU::EXEC:
    case AMDGPU::EXEC_LO:
    case AMDGPU::EXEC_HI:
    case AMDGPU::M0:
    case AMDGPU::SGPR_NULL:
    case AMDGPU::SGPR_NULL64:
    case AMDGPU::SCC:
      return true;
    default:
      return false;
    }
  };
  auto IsVCC = [](const Register Reg) {
    return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI;
  };
 
  struct StateType {
    SmallSet<Register, 2> HazardSGPRs;
 
    static unsigned getHashValue(const StateType &State) {
      return hash_combine_range(State.HazardSGPRs);
    }
    static bool isEqual(const StateType &LHS, const StateType &RHS) {
      return LHS.HazardSGPRs == RHS.HazardSGPRs;
    }
  };
 
  SmallVector<const MachineInstr *> WaitInstrs;
  StateType InitialState;
 
  // Look for SGPR write.
  MachineOperand *HazardDef = nullptr;
  for (MachineOperand &Op : MI->all_defs()) {
    Register Reg = Op.getReg();
    if (IgnoreableSGPR(Reg))
      continue;
    if (!IsVCC(Reg)) {
      if (Op.isImplicit())
        continue;
      if (!TRI->isSGPRReg(MRI, Reg))
        continue;
    }
 
    HazardDef = &Op;
    break;
  }
 
  if (!HazardDef)
    return false;
 
  // Setup to track writes to individual SGPRs
  const Register HazardReg = HazardDef->getReg();
  if (AMDGPU::SReg_32RegClass.contains(HazardReg)) {
    InitialState.HazardSGPRs.insert(HazardReg);
  } else {
    assert(AMDGPU::SReg_64RegClass.contains(HazardReg));
    InitialState.HazardSGPRs.insert(TRI->getSubReg(HazardReg, AMDGPU::sub0));
    InitialState.HazardSGPRs.insert(TRI->getSubReg(HazardReg, AMDGPU::sub1));
  }
 
  auto IsHazardFn = [&](StateType &State, const MachineInstr &I) {
    if (State.HazardSGPRs.empty())
      return HazardExpired;
 
    switch (I.getOpcode()) {
    case AMDGPU::V_ADDC_U32_e32:
    case AMDGPU::V_ADDC_U32_dpp:
    case AMDGPU::V_CNDMASK_B16_t16_e32:
    case AMDGPU::V_CNDMASK_B16_fake16_e32:
    case AMDGPU::V_CNDMASK_B16_t16_dpp:
    case AMDGPU::V_CNDMASK_B16_fake16_dpp:
    case AMDGPU::V_CNDMASK_B32_e32:
    case AMDGPU::V_CNDMASK_B32_dpp:
    case AMDGPU::V_DIV_FMAS_F32_e64:
    case AMDGPU::V_DIV_FMAS_F64_e64:
    case AMDGPU::V_SUBB_U32_e32:
    case AMDGPU::V_SUBB_U32_dpp:
    case AMDGPU::V_SUBBREV_U32_e32:
    case AMDGPU::V_SUBBREV_U32_dpp: {
      // These implicitly read VCC as mask source.
      return IsVCC(HazardReg) ? HazardFound : NoHazardFound;
    }
    case AMDGPU::V_ADDC_U32_e64:
    case AMDGPU::V_ADDC_U32_e64_dpp:
    case AMDGPU::V_CNDMASK_B16_t16_e64:
    case AMDGPU::V_CNDMASK_B16_fake16_e64:
    case AMDGPU::V_CNDMASK_B16_t16_e64_dpp:
    case AMDGPU::V_CNDMASK_B16_fake16_e64_dpp:
    case AMDGPU::V_CNDMASK_B32_e64:
    case AMDGPU::V_CNDMASK_B32_e64_dpp:
    case AMDGPU::V_SUBB_U32_e64:
    case AMDGPU::V_SUBB_U32_e64_dpp:
    case AMDGPU::V_SUBBREV_U32_e64:
    case AMDGPU::V_SUBBREV_U32_e64_dpp: {
      // Only check mask register overlaps.
      const MachineOperand *SSRCOp = TII.getNamedOperand(I, AMDGPU::OpName::src2);
      assert(SSRCOp);
      bool Result = TRI->regsOverlap(SSRCOp->getReg(), HazardReg);
      return Result ? HazardFound : NoHazardFound;
    }
    default:
      return NoHazardFound;
    }
  };
 
  auto UpdateStateFn = [&](StateType &State, const MachineInstr &I) {
    // Update tracking of SGPR writes.
    for (auto &Op : I.all_defs()) {
      Register Reg = Op.getReg();
      if (IgnoreableSGPR(Reg))
        continue;
      if (!IsVCC(Reg)) {
        if (Op.isImplicit())
          continue;
        if (!TRI->isSGPRReg(MRI, Reg))
          continue;
      }
 
      // Stop tracking any SGPRs with writes on the basis that they will
      // already have an appropriate wait inserted afterwards.
      SmallVector<Register, 2> Found;
      for (Register SGPR : State.HazardSGPRs) {
        if (Reg == SGPR || TRI->regsOverlap(Reg, SGPR))
          Found.push_back(SGPR);
      }
      for (Register SGPR : Found)
        State.HazardSGPRs.erase(SGPR);
    }
  };
 
  // Check for hazard
  if (!hasHazard<StateType>(InitialState, IsHazardFn, UpdateStateFn,
                            MI->getParent(),
                            std::next(MI->getReverseIterator())))
    return false;
 
  // Compute counter mask
  unsigned DepCtr =
      IsVALU ? (IsVCC(HazardReg) ? AMDGPU::DepCtr::encodeFieldVaVcc(0, ST)
                                 : AMDGPU::DepCtr::encodeFieldVaSdst(0, ST))
             : AMDGPU::DepCtr::encodeFieldSaSdst(0, ST);
 
  // Add s_waitcnt_depctr after SGPR write.
  auto NextMI = std::next(MI->getIterator());
  auto NewMI = BuildMI(*MI->getParent(), NextMI, MI->getDebugLoc(),
                       TII.get(AMDGPU::S_WAITCNT_DEPCTR))
                   .addImm(DepCtr);
 
  // SALU write may be s_getpc in a bundle.
  updateGetPCBundle(NewMI);
 
  return true;
}
 
static bool ensureEntrySetPrio(MachineFunction *MF, int Priority,
                               const SIInstrInfo &TII) {
  MachineBasicBlock &EntryMBB = MF->front();
  if (EntryMBB.begin() != EntryMBB.end()) {
    auto &EntryMI = *EntryMBB.begin();
    if (EntryMI.getOpcode() == AMDGPU::S_SETPRIO &&
        EntryMI.getOperand(0).getImm() >= Priority)
      return false;
  }
 
  BuildMI(EntryMBB, EntryMBB.begin(), DebugLoc(), TII.get(AMDGPU::S_SETPRIO))
      .addImm(Priority);
  return true;
}
 
bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) {
  if (!ST.hasRequiredExportPriority())
    return false;
 
  // Assume the following shader types will never have exports,
  // and avoid adding or adjusting S_SETPRIO.
  MachineBasicBlock *MBB = MI->getParent();
  MachineFunction *MF = MBB->getParent();
  auto CC = MF->getFunction().getCallingConv();
  switch (CC) {
  case CallingConv::AMDGPU_CS:
  case CallingConv::AMDGPU_CS_Chain:
  case CallingConv::AMDGPU_CS_ChainPreserve:
  case CallingConv::AMDGPU_KERNEL:
    return false;
  default:
    break;
  }
 
  const int MaxPriority = 3;
  const int NormalPriority = 2;
  const int PostExportPriority = 0;
 
  auto It = MI->getIterator();
  switch (MI->getOpcode()) {
  case AMDGPU::S_ENDPGM:
  case AMDGPU::S_ENDPGM_SAVED:
  case AMDGPU::S_ENDPGM_ORDERED_PS_DONE:
  case AMDGPU::SI_RETURN_TO_EPILOG:
    // Ensure shader with calls raises priority at entry.
    // This ensures correct priority if exports exist in callee.
    if (MF->getFrameInfo().hasCalls())
      return ensureEntrySetPrio(MF, NormalPriority, TII);
    return false;
  case AMDGPU::S_SETPRIO: {
    // Raise minimum priority unless in workaround.
    auto &PrioOp = MI->getOperand(0);
    int Prio = PrioOp.getImm();
    bool InWA = (Prio == PostExportPriority) &&
                (It != MBB->begin() && TII.isEXP(*std::prev(It)));
    if (InWA || Prio >= NormalPriority)
      return false;
    PrioOp.setImm(std::min(Prio + NormalPriority, MaxPriority));
    return true;
  }
  default:
    if (!TII.isEXP(*MI))
      return false;
    break;
  }
 
  // Check entry priority at each export (as there will only be a few).
  // Note: amdgpu_gfx can only be a callee, so defer to caller setprio.
  bool Changed = false;
  if (CC != CallingConv::AMDGPU_Gfx && CC != CallingConv::AMDGPU_Gfx_WholeWave)
    Changed = ensureEntrySetPrio(MF, NormalPriority, TII);
 
  auto NextMI = std::next(It);
  bool EndOfShader = false;
  if (NextMI != MBB->end()) {
    // Only need WA at end of sequence of exports.
    if (TII.isEXP(*NextMI))
      return Changed;
    // Assume appropriate S_SETPRIO after export means WA already applied.
    if (NextMI->getOpcode() == AMDGPU::S_SETPRIO &&
        NextMI->getOperand(0).getImm() == PostExportPriority)
      return Changed;
    EndOfShader = NextMI->getOpcode() == AMDGPU::S_ENDPGM;
  }
 
  const DebugLoc &DL = MI->getDebugLoc();
 
  // Lower priority.
  BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_SETPRIO))
      .addImm(PostExportPriority);
 
  if (!EndOfShader) {
    // Wait for exports to complete.
    BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_WAITCNT_EXPCNT))
        .addReg(AMDGPU::SGPR_NULL)
        .addImm(0);
  }
 
  BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_NOP)).addImm(0);
  BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_NOP)).addImm(0);
 
  if (!EndOfShader) {
    // Return to normal (higher) priority.
    BuildMI(*MBB, NextMI, DL, TII.get(AMDGPU::S_SETPRIO))
        .addImm(NormalPriority);
  }
 
  return true;
}
 
// Advance past meta instructions (debug values, labels, CFI, KILL, etc.) to the
// next instruction that actually issues. Unlike skipDebugInstructionsForward /
// next_nodbg, this skips the full isMetaInstruction() set.
static MachineBasicBlock::iterator
skipMetaInstructionsForward(MachineBasicBlock::iterator I,
                            MachineBasicBlock::iterator End) {
  while (I != End && I->isMetaInstruction())
    ++I;
  return I;
}
 
bool GCNHazardRecognizer::fixVPermPk16Hazard(MachineInstr *MI) {
  // Requirement #1 of 2:
  // The cross-wave entry-block mitigation is delegated to the mandatory
  // unclaused-VMEM entry prologue (GLOBAL_PREFETCH_B8 + V_NOP).
  assert(ST.hasRequiresInitialUnclausedVmem() &&
         "V_PERM_PK16-hazard subtarget must provide the unclaused-VMEM entry "
         "prologue to satisfy the cross-wave entry mitigation");
 
  if (!SIInstrInfo::isVPermPk16(MI->getOpcode()))
    return false;
 
  MachineBasicBlock *MBB = MI->getParent();
 
  // Requirement #2 of 2:
  // V_PERM_PK16 must be immediately followed by a safe instruction.
  MachineBasicBlock::iterator NextI =
      skipMetaInstructionsForward(std::next(MI->getIterator()), MBB->end());
  if (NextI != MBB->end() && TII.isVPermPk16SafeInstr(*NextI))
    return false;
 
  // EXEC is guaranteed non-zero here: V_PERM_PK16 reports unwanted effects
  // when EXEC is empty, so s_cbranch_execz over this region is retained.
  // A plain V_NOP is therefore a real VALU nop and clears the hazard.
  BuildMI(*MBB, NextI, MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
  return true;
}
 
bool GCNHazardRecognizer::fixGetRegWaitIdle(MachineInstr *MI) {
  if (!isSGetReg(MI->getOpcode()))
    return false;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  switch (getHWReg(TII, *MI)) {
  default:
    return false;
  case AMDGPU::Hwreg::ID_STATUS:
  case AMDGPU::Hwreg::ID_STATE_PRIV:
  case AMDGPU::Hwreg::ID_EXCP_FLAG_PRIV:
  case AMDGPU::Hwreg::ID_EXCP_FLAG_USER:
    break;
  }
 
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII->get(AMDGPU::S_WAITCNT_DEPCTR))
      .addImm(0);
  return true;
}
 
bool GCNHazardRecognizer::fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI) {
  if (MI->getOpcode() != AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
    return false;
 
  const SIInstrInfo *TII = ST.getInstrInfo();
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII->get(AMDGPU::S_WAITCNT_DEPCTR))
      .addImm(AMDGPU::DepCtr::encodeFieldVmVsrc(0, ST));
  BuildMI(*MI->getParent(), std::next(MI->getIterator()), MI->getDebugLoc(),
          TII->get(AMDGPU::S_WAITCNT_DEPCTR))
      .addImm(AMDGPU::DepCtr::encodeFieldVmVsrc(0, ST));
 
  return true;
}
 
bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
  // No reason to check this in pre-RA scheduling, SGPRs have to be allocated
  // for hazard to trigger.
  if (!isHazardRecognizerMode())
    return false;
 
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const SIInstrInfo *TII = ST.getInstrInfo();
  // Hazard expires after 10 SGPR writes by SALU or 8 SGPR writes by VALU.
  const int FlatScrBaseWaitStates = 10;
 
  bool ReadsFlatScrLo =
      MI->readsRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, TRI);
  bool ReadsFlatScrHi =
      MI->readsRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI);
  if (isSGetReg(MI->getOpcode())) {
    switch (getHWReg(TII, *MI)) {
    default:
      break;
    case AMDGPU::Hwreg::ID_FLAT_SCR_LO:
      ReadsFlatScrLo = true;
      break;
    case AMDGPU::Hwreg::ID_FLAT_SCR_HI:
      ReadsFlatScrHi = true;
      break;
    }
  }
 
  const MachineRegisterInfo &MRI = MF.getRegInfo();
 
  auto IsRegDefHazard = [&](Register Reg) -> bool {
    DenseSet<const MachineBasicBlock *> Visited;
    auto IsHazardFn = [TRI, Reg](const MachineInstr &MI) {
      return MI.modifiesRegister(Reg, TRI);
    };
 
    // This literally abuses the idea of waitstates. Instead of waitstates it
    // returns 1 for SGPR written and 0 otherwise.
    auto IsSGPRDef = [TII, TRI, &MRI](const MachineInstr &MI) -> unsigned {
      if (!TII->isSALU(MI) && !TII->isVALU(MI, /*AllowLDSDMA=*/true))
        return 0;
      for (const MachineOperand &MO : MI.all_defs()) {
        if (TRI->isSGPRReg(MRI, MO.getReg()))
          return 1;
      }
      return 0;
    };
 
    auto IsExpiredFn = [=](const MachineInstr &MI, int SgprWrites) {
      if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
        unsigned Wait = MI.getOperand(0).getImm();
        if (AMDGPU::DepCtr::decodeFieldSaSdst(Wait) == 0 &&
            AMDGPU::DepCtr::decodeFieldVaSdst(Wait) == 0)
          return true;
      }
      return SgprWrites >= FlatScrBaseWaitStates;
    };
 
    return ::getWaitStatesSince(
               IsHazardFn, MI->getParent(), std::next(MI->getReverseIterator()),
               0, IsExpiredFn, Visited, IsSGPRDef) < FlatScrBaseWaitStates;
  };
 
  if ((!ReadsFlatScrLo || MRI.isConstantPhysReg(AMDGPU::SGPR102) ||
       !IsRegDefHazard(AMDGPU::SGPR102)) &&
      (!ReadsFlatScrHi || MRI.isConstantPhysReg(AMDGPU::SGPR103) ||
       !IsRegDefHazard(AMDGPU::SGPR103)))
    return false;
 
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII->get(AMDGPU::S_WAITCNT_DEPCTR))
      .addImm(AMDGPU::DepCtr::encodeFieldVaSdst(
          AMDGPU::DepCtr::encodeFieldSaSdst(0, ST), 0));
  return true;
}
 
bool GCNHazardRecognizer::fixSetRegMode(MachineInstr *MI) {
  if (!isSSetReg(MI->getOpcode()) ||
      MI->getOperand(1).getImm() != AMDGPU::Hwreg::ID_MODE)
    return false;
 
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
  return true;
}
 
bool GCNHazardRecognizer::fixTDM(MachineInstr *MI) {
  auto IsTDM = [&](const MachineInstr &MI) -> bool {
    return SIInstrInfo::usesTENSOR_CNT(MI) &&
           MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT;
  };
 
  if (!IsTDM(*MI))
    return false;
 
  auto IsExpiredFn = [](const MachineInstr &MI, int) {
    if (MI.getOpcode() != AMDGPU::S_WAIT_TENSORCNT)
      return false;
    return MI.getOperand(0).getImm() <= 10;
  };
 
  if (::getWaitStatesSince(IsTDM, MI, IsExpiredFn) ==
      std::numeric_limits<int>::max())
    return false;
 
  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
          TII.get(AMDGPU::S_WAIT_TENSORCNT))
      .addImm(10);
  return true;
}