LLVM 22.0.0git
Typedefs | Functions | Variables
GCNHazardRecognizer.cpp File Reference
#include "GCNHazardRecognizer.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/TargetParser/TargetParser.h"

Go to the source code of this file.

Typedefs

using HazardFnResult = enum { HazardFound, HazardExpired, NoHazardFound }
using IsExpiredFn = function_ref<bool(const MachineInstr &, int WaitStates)>
using GetNumWaitStatesFn = function_ref<unsigned int(const MachineInstr &)>

Functions

static bool shouldRunLdsBranchVmemWARHazardFixup (const MachineFunction &MF, const GCNSubtarget &ST)
static bool isDivFMas (unsigned Opcode)
static bool isSGetReg (unsigned Opcode)
static bool isSSetReg (unsigned Opcode)
static bool isRWLane (unsigned Opcode)
static bool isRFE (unsigned Opcode)
static bool isSMovRel (unsigned Opcode)
static bool isSendMsgTraceDataOrGDS (const SIInstrInfo &TII, const MachineInstr &MI)
static bool isPermlane (const MachineInstr &MI)
static bool isLdsDma (const MachineInstr &MI)
static unsigned getHWReg (const SIInstrInfo *TII, const MachineInstr &RegInstr)
static void insertNoopsInBundle (MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity)
template<typename StateT>
static bool hasHazard (StateT State, function_ref< HazardFnResult(StateT &, const MachineInstr &)> IsHazard, function_ref< void(StateT &, const MachineInstr &)> UpdateState, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, DenseSet< const MachineBasicBlock * > &Visited)
static int getWaitStatesSince (GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock * > &Visited, GetNumWaitStatesFn GetNumWaitStates=SIInstrInfo::getNumWaitStates)
static int getWaitStatesSince (GCNHazardRecognizer::IsHazardFn IsHazard, const MachineInstr *MI, IsExpiredFn IsExpired)
static void addRegUnits (const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg)
static void addRegsToSet (const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &DefSet, BitVector &UseSet)
static bool breaksSMEMSoftClause (MachineInstr *MI)
static bool breaksVMEMSoftClause (MachineInstr *MI)
static const MachineOperand * getDstSelForwardingOperand (const MachineInstr &MI, const GCNSubtarget &ST)
 Dest sel forwarding issue occurs if additional logic is needed to swizzle / pack the computed value into correct bit position of the dest register.
static bool consumesDstSelForwardingOperand (const MachineInstr *VALU, const MachineOperand *Dst, const SIRegisterInfo *TRI)
 Checks whether the provided MI "consumes" the operand with a Dest sel fowarding issue Dst .
static bool isVCmpXWritesExec (const SIInstrInfo &TII, const SIRegisterInfo &TRI, const MachineInstr &MI)
static bool isStoreCountWaitZero (const MachineInstr &I)
static bool isCoexecutableVALUInst (const MachineInstr &MI)
static bool IsWMMAHazardInstInCategory (const MachineInstr &MI, const SIInstrInfo *TII, unsigned Latency, unsigned Category)
static int GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates (int NumPasses, bool IsGFX950)
static int GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates (int NumPasses, bool IsGFX950)
static int GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates (int NumPasses)
static int GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates (int NumPasses)
static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates (int NumPasses, bool IsGFX950)
static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates (int NumPasses)
static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates (int NumPasses, bool IsGFX950)
static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates (int NumPasses, bool IsGFX950)
static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates (int NumPasses)
static void updateGetPCBundle (MachineInstr *NewMI)
static bool ensureEntrySetPrio (MachineFunction *MF, int Priority, const SIInstrInfo &TII)

Variables

static cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio ("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))
static cl::opt< unsigned > NopPadding ("amdgpu-snop-padding", cl::init(0), cl::Hidden, cl::desc("Insert a s_nop x before every instruction"))

Typedef Documentation

◆ GetNumWaitStatesFn

using GetNumWaitStatesFn = function_ref<unsigned int(const MachineInstr &)>

Definition at line 441 of file GCNHazardRecognizer.cpp.

◆ HazardFnResult

using HazardFnResult = enum { HazardFound, HazardExpired, NoHazardFound }

Definition at line 438 of file GCNHazardRecognizer.cpp.

◆ IsExpiredFn

using IsExpiredFn = function_ref<bool(const MachineInstr &, int WaitStates)>

Definition at line 440 of file GCNHazardRecognizer.cpp.

Function Documentation

◆ addRegsToSet()

void addRegsToSet ( const SIRegisterInfo & TRI,
iterator_range< MachineInstr::const_mop_iterator > Ops,
BitVector & DefSet,
BitVector & UseSet )
static

Definition at line 588 of file GCNHazardRecognizer.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, addRegUnits(), and TRI.

◆ addRegUnits()

void addRegUnits ( const SIRegisterInfo & TRI,
BitVector & BV,
MCRegister Reg )
static

Definition at line 582 of file GCNHazardRecognizer.cpp.

References Reg, llvm::BitVector::set(), and TRI.

Referenced by addRegsToSet().

◆ breaksSMEMSoftClause()

bool breaksSMEMSoftClause ( MachineInstr * MI)
static

Definition at line 601 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isSMRD(), and MI.

◆ breaksVMEMSoftClause()

bool breaksVMEMSoftClause ( MachineInstr * MI)
static

Definition at line 605 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isVMEM(), and MI.

◆ consumesDstSelForwardingOperand()

bool consumesDstSelForwardingOperand ( const MachineInstr * VALU,
const MachineOperand * Dst,
const SIRegisterInfo * TRI )
static

Checks whether the provided MI "consumes" the operand with a Dest sel fowarding issue Dst .

We may "consume" the Dst via a standard explicit RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)

Definition at line 931 of file GCNHazardRecognizer.cpp.

References TRI.

◆ ensureEntrySetPrio()

bool ensureEntrySetPrio ( MachineFunction * MF,
int Priority,
const SIInstrInfo & TII )
static

Definition at line 3321 of file GCNHazardRecognizer.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineBasicBlock::end(), llvm::MachineFunction::front(), and TII.

◆ getDstSelForwardingOperand()

const MachineOperand * getDstSelForwardingOperand ( const MachineInstr & MI,
const GCNSubtarget & ST )
static

Dest sel forwarding issue occurs if additional logic is needed to swizzle / pack the computed value into correct bit position of the dest register.

This occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with dst_sel that is not aligned to the register. This function analayzes the MI and

Returns
an operand with dst forwarding issue, or nullptr if none exists.

Definition at line 885 of file GCNHazardRecognizer.cpp.

References llvm::SISrcMods::DST_OP_SEL, llvm::AMDGPU::SDWA::DWORD, llvm::AMDGPU::FP4, llvm::AMDGPU::FP8, llvm::AMDGPU::getFPDstSelType(), llvm::AMDGPU::hasNamedOperand(), llvm::SIInstrInfo::isSDWA(), llvm::SIInstrInfo::isVALU(), MI, llvm::SISrcMods::OP_SEL_0, and TII.

◆ getHWReg()

unsigned getHWReg ( const SIInstrInfo * TII,
const MachineInstr & RegInstr )
static

Definition at line 168 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::EncodingFields< HwregId, HwregOffset, HwregSize >::decode(), llvm::MachineOperand::getImm(), and TII.

◆ getWaitStatesSince() [1/2]

int getWaitStatesSince ( GCNHazardRecognizer::IsHazardFn IsHazard,
const MachineBasicBlock * MBB,
MachineBasicBlock::const_reverse_instr_iterator I,
int WaitStates,
IsExpiredFn IsExpired,
DenseSet< const MachineBasicBlock * > & Visited,
GetNumWaitStatesFn GetNumWaitStates = SIInstrInfo::getNumWaitStates )
static

Definition at line 488 of file GCNHazardRecognizer.cpp.

References E(), llvm::SIInstrInfo::getNumWaitStates(), getWaitStatesSince(), I, llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), and MBB.

Referenced by getWaitStatesSince(), and getWaitStatesSince().

◆ getWaitStatesSince() [2/2]

int getWaitStatesSince ( GCNHazardRecognizer::IsHazardFn IsHazard,
const MachineInstr * MI,
IsExpiredFn IsExpired )
static

Definition at line 524 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::getNumWaitStates(), getWaitStatesSince(), and MI.

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates()

int GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates ( int NumPasses)
static

Definition at line 2483 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates()

int GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates ( int NumPasses)
static

Definition at line 2492 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates()

int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates ( int NumPasses)
static

Definition at line 2830 of file GCNHazardRecognizer.cpp.

◆ GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates()

int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates ( int NumPasses)
static

Definition at line 2802 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates()

int GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates ( int NumPasses,
bool IsGFX950 )
static

Definition at line 2472 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates()

int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates ( int NumPasses,
bool IsGFX950 )
static

Definition at line 2500 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates()

int GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates ( int NumPasses,
bool IsGFX950 )
static

Definition at line 2461 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates()

int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates ( int NumPasses,
bool IsGFX950 )
static

Definition at line 2820 of file GCNHazardRecognizer.cpp.

◆ GFX940_XDL_N_PassWriteVgprVALUWawWaitStates()

int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates ( int NumPasses,
bool IsGFX950 )
static

Definition at line 2810 of file GCNHazardRecognizer.cpp.

◆ hasHazard()

template<typename StateT>
bool hasHazard ( StateT State,
function_ref< HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
function_ref< void(StateT &, const MachineInstr &)> UpdateState,
const MachineBasicBlock * MBB,
MachineBasicBlock::const_reverse_instr_iterator I,
DenseSet< const MachineBasicBlock * > & Visited )
static

Definition at line 446 of file GCNHazardRecognizer.cpp.

References E(), hasHazard(), I, llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), and MBB.

Referenced by hasHazard().

◆ insertNoopsInBundle()

void insertNoopsInBundle ( MachineInstr * MI,
const SIInstrInfo & TII,
unsigned Quantity )
static

Definition at line 249 of file GCNHazardRecognizer.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), MI, and TII.

◆ isCoexecutableVALUInst()

bool isCoexecutableVALUInst ( const MachineInstr & MI)
static

Definition at line 1929 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isSWMMAC(), llvm::SIInstrInfo::isTRANS(), llvm::SIInstrInfo::isVALU(), llvm::SIInstrInfo::isWMMA(), and MI.

◆ isDivFMas()

bool isDivFMas ( unsigned Opcode)
static

Definition at line 80 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isLdsDma()

bool isLdsDma ( const MachineInstr & MI)
static

Definition at line 163 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isFLAT(), llvm::SIInstrInfo::isMUBUF(), llvm::SIInstrInfo::isVALU(), and MI.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isPermlane()

bool isPermlane ( const MachineInstr & MI)
static

Definition at line 145 of file GCNHazardRecognizer.cpp.

References MI.

Referenced by llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isRFE()

bool isRFE ( unsigned Opcode)
static

Definition at line 103 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isRWLane()

bool isRWLane ( unsigned Opcode)
static

Definition at line 99 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSendMsgTraceDataOrGDS()

bool isSendMsgTraceDataOrGDS ( const SIInstrInfo & TII,
const MachineInstr & MI )
static

Definition at line 119 of file GCNHazardRecognizer.cpp.

References MI, and TII.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSGetReg()

bool isSGetReg ( unsigned Opcode)
static

Definition at line 84 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSMovRel()

bool isSMovRel ( unsigned Opcode)
static

Definition at line 107 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isSSetReg()

bool isSSetReg ( unsigned Opcode)
static

Definition at line 88 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

◆ isStoreCountWaitZero()

bool isStoreCountWaitZero ( const MachineInstr & I)
static

Definition at line 1454 of file GCNHazardRecognizer.cpp.

References I.

◆ isVCmpXWritesExec()

bool isVCmpXWritesExec ( const SIInstrInfo & TII,
const SIRegisterInfo & TRI,
const MachineInstr & MI )
static

Definition at line 1213 of file GCNHazardRecognizer.cpp.

References MI, TII, and TRI.

◆ IsWMMAHazardInstInCategory()

bool IsWMMAHazardInstInCategory ( const MachineInstr & MI,
const SIInstrInfo * TII,
unsigned Latency,
unsigned Category )
static

Definition at line 1934 of file GCNHazardRecognizer.cpp.

References assert(), llvm::SIInstrInfo::isSWMMAC(), llvm::SIInstrInfo::isWMMA(), llvm::Latency, MI, and TII.

◆ shouldRunLdsBranchVmemWARHazardFixup()

bool shouldRunLdsBranchVmemWARHazardFixup ( const MachineFunction & MF,
const GCNSubtarget & ST )
static

Definition at line 1433 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isDS(), llvm::SIInstrInfo::isFLAT(), llvm::SIInstrInfo::isSegmentSpecificFLAT(), llvm::SIInstrInfo::isVMEM(), MBB, and MI.

Referenced by llvm::GCNHazardRecognizer::GCNHazardRecognizer().

◆ updateGetPCBundle()

void updateGetPCBundle ( MachineInstr * NewMI)
static

Definition at line 3160 of file GCNHazardRecognizer.cpp.

References assert(), llvm::MachineBasicBlock::end(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getParent(), I, and llvm::MachineInstr::isBundled().

Variable Documentation

◆ MFMAPaddingRatio

cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops.")) ( "amdgpu-mfma-padding-ratio" ,
cl::init(0) ,
cl::Hidden ,
cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops.")  )
static

◆ NopPadding

cl::opt< unsigned > NopPadding("amdgpu-snop-padding", cl::init(0), cl::Hidden, cl::desc("Insert a s_nop x before every instruction")) ( "amdgpu-snop-padding" ,
cl::init(0) ,
cl::Hidden ,
cl::desc("Insert a s_nop x before every instruction")  )
static

Referenced by llvm::GCNHazardRecognizer::PreEmitNoops().

Generated on for LLVM by doxygen 1.14.0