GCNHazardRecognizer.cpp File Reference

#include "GCNHazardRecognizer.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/TargetParser/TargetParser.h"

Go to the source code of this file.

Typedefs
using	HazardFnResult = enum { HazardFound, HazardExpired, NoHazardFound }
using	IsExpiredFn = function_ref< bool(const MachineInstr &, int WaitStates)>
using	GetNumWaitStatesFn = function_ref< unsigned int(const MachineInstr &)>

Functions
static bool	shouldRunLdsBranchVmemWARHazardFixup (const MachineFunction &MF, const GCNSubtarget &ST)
static bool	isDivFMas (unsigned Opcode)
static bool	isSGetReg (unsigned Opcode)
static bool	isSSetReg (unsigned Opcode)
static bool	isRWLane (unsigned Opcode)
static bool	isRFE (unsigned Opcode)
static bool	isSMovRel (unsigned Opcode)
static bool	isDGEMM (unsigned Opcode)
static bool	isXDL (const GCNSubtarget &ST, const MachineInstr &MI)
static bool	isSendMsgTraceDataOrGDS (const SIInstrInfo &TII, const MachineInstr &MI)
static bool	isPermlane (const MachineInstr &MI)
static bool	isLdsDma (const MachineInstr &MI)
static unsigned	getHWReg (const SIInstrInfo *TII, const MachineInstr &RegInstr)
static void	insertNoopsInBundle (MachineInstr *MI, const SIInstrInfo &TII, unsigned Quantity)
template<typename StateT >
static bool	hasHazard (StateT State, function_ref< HazardFnResult(StateT &, const MachineInstr &)> IsHazard, function_ref< void(StateT &, const MachineInstr &)> UpdateState, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, DenseSet< const MachineBasicBlock * > &Visited)
static int	getWaitStatesSince (GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB, MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates, IsExpiredFn IsExpired, DenseSet< const MachineBasicBlock * > &Visited, GetNumWaitStatesFn GetNumWaitStates=SIInstrInfo::getNumWaitStates)
static int	getWaitStatesSince (GCNHazardRecognizer::IsHazardFn IsHazard, const MachineInstr *MI, IsExpiredFn IsExpired)
static void	addRegUnits (const SIRegisterInfo &TRI, BitVector &BV, MCRegister Reg)
static void	addRegsToSet (const SIRegisterInfo &TRI, iterator_range< MachineInstr::const_mop_iterator > Ops, BitVector &DefSet, BitVector &UseSet)
static bool	breaksSMEMSoftClause (MachineInstr *MI)
static bool	breaksVMEMSoftClause (MachineInstr *MI)
static const MachineOperand *	getDstSelForwardingOperand (const MachineInstr &MI, const GCNSubtarget &ST)
	Dest sel forwarding issue occurs if additional logic is needed to swizzle / pack the computed value into correct bit position of the dest register.
static bool	consumesDstSelForwardingOperand (const MachineInstr *VALU, const MachineOperand *Dst, const SIRegisterInfo *TRI)
	Checks whether the provided MI "consumes" the operand with a Dest sel fowarding issue Dst .
static bool	isVCmpXWritesExec (const SIInstrInfo &TII, const SIRegisterInfo &TRI, const MachineInstr &MI)
static bool	isStoreCountWaitZero (const MachineInstr &I)
static int	GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates (int NumPasses, bool IsGFX950)
static int	GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates (int NumPasses)
static int	GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates (int NumPasses)
static int	GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates (int NumPasses)
static int	GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates (int NumPasses)
static int	GFX940_XDL_N_PassWriteVgprVALUWawWaitStates (int NumPasses)
static int	GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates (int NumPasses)
static int	GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates (int NumPasses)
static void	updateGetPCBundle (MachineInstr *NewMI)
static std::optional< unsigned >	sgprPairNumber (Register Reg, const SIRegisterInfo &TRI)
static bool	ensureEntrySetPrio (MachineFunction *MF, int Priority, const SIInstrInfo &TII)

Variables
static cl::opt< unsigned, false, MFMAPaddingRatioParser >	MFMAPaddingRatio ("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops."))
static cl::opt< unsigned >	MaxExhaustiveHazardSearch ("amdgpu-max-exhaustive-hazard-search", cl::init(128), cl::Hidden, cl::desc("Maximum function size for exhausive hazard search"))

Typedef Documentation

GetNumWaitStatesFn

using GetNumWaitStatesFn = function_ref<unsigned int(const MachineInstr &)>

Definition at line 461 of file GCNHazardRecognizer.cpp.

HazardFnResult

using HazardFnResult = enum { HazardFound, HazardExpired, NoHazardFound }

Definition at line 458 of file GCNHazardRecognizer.cpp.

IsExpiredFn

using IsExpiredFn = function_ref<bool(const MachineInstr &, int WaitStates)>

Definition at line 460 of file GCNHazardRecognizer.cpp.

Function Documentation

addRegsToSet()

static void addRegsToSet ( const SIRegisterInfo &  TRI, iterator_range< MachineInstr::const_mop_iterator >  Ops, BitVector &  DefSet, BitVector &  UseSet  )

static

Definition at line 608 of file GCNHazardRecognizer.cpp.

References addRegUnits(), and TRI.

addRegUnits()

static void addRegUnits ( const SIRegisterInfo &  TRI, BitVector &  BV, MCRegister  Reg  )

static

Definition at line 602 of file GCNHazardRecognizer.cpp.

References llvm::BitVector::set(), and TRI.

Referenced by addRegsToSet().

breaksSMEMSoftClause()

static bool breaksSMEMSoftClause ( MachineInstr *  MI )

static

Definition at line 621 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isSMRD(), and MI.

breaksVMEMSoftClause()

static bool breaksVMEMSoftClause ( MachineInstr *  MI )

static

Definition at line 625 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isFLAT(), llvm::SIInstrInfo::isVMEM(), and MI.

consumesDstSelForwardingOperand()

static bool consumesDstSelForwardingOperand ( const MachineInstr *  VALU, const MachineOperand *  Dst, const SIRegisterInfo *  TRI  )

static

Checks whether the provided MI "consumes" the operand with a Dest sel fowarding issue Dst .

We may "consume" the Dst via a standard explicit RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)

Definition at line 951 of file GCNHazardRecognizer.cpp.

References TRI.

ensureEntrySetPrio()

static bool ensureEntrySetPrio ( MachineFunction *  MF, int  Priority, const SIInstrInfo &  TII  )

static

Definition at line 3377 of file GCNHazardRecognizer.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineBasicBlock::end(), llvm::MachineFunction::front(), and TII.

getDstSelForwardingOperand()

static const MachineOperand * getDstSelForwardingOperand ( const MachineInstr &  MI, const GCNSubtarget &  ST  )

static

Dest sel forwarding issue occurs if additional logic is needed to swizzle / pack the computed value into correct bit position of the dest register.

This occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with dst_sel that is not aligned to the register. This function analayzes the MI and

Returns

an operand with dst forwarding issue, or nullptr if none exists.

Definition at line 905 of file GCNHazardRecognizer.cpp.

References llvm::SISrcMods::DST_OP_SEL, llvm::AMDGPU::SDWA::DWORD, llvm::AMDGPU::FP4, llvm::AMDGPU::FP8, llvm::AMDGPU::getFPDstSelType(), llvm::AMDGPU::hasNamedOperand(), llvm::SIInstrInfo::isSDWA(), llvm::SIInstrInfo::isVALU(), MI, llvm::SISrcMods::OP_SEL_0, and TII.

getHWReg()

static unsigned getHWReg ( const SIInstrInfo *  TII, const MachineInstr &  RegInstr  )

static

Definition at line 183 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::EncodingFields< Fields >::decode(), llvm::MachineOperand::getImm(), and TII.

getWaitStatesSince() [1/2]

static int getWaitStatesSince ( GCNHazardRecognizer::IsHazardFn  IsHazard, const MachineBasicBlock *  MBB, MachineBasicBlock::const_reverse_instr_iterator  I, int  WaitStates, IsExpiredFn  IsExpired, DenseSet< const MachineBasicBlock * > &  Visited, GetNumWaitStatesFn  GetNumWaitStates = SIInstrInfo::getNumWaitStates  )

static

Definition at line 508 of file GCNHazardRecognizer.cpp.

References getWaitStatesSince(), I, llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), llvm::MachineBasicBlock::instr_rend(), MBB, and llvm::MachineBasicBlock::predecessors().

Referenced by getWaitStatesSince().

getWaitStatesSince() [2/2]

static int getWaitStatesSince ( GCNHazardRecognizer::IsHazardFn  IsHazard, const MachineInstr *  MI, IsExpiredFn  IsExpired  )

static

Definition at line 544 of file GCNHazardRecognizer.cpp.

References getWaitStatesSince(), and MI.

GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates()

static int GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates ( int  NumPasses )

static

Definition at line 2283 of file GCNHazardRecognizer.cpp.

GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates()

static int GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates ( int  NumPasses )

static

Definition at line 2292 of file GCNHazardRecognizer.cpp.

GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates()

static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates ( int  NumPasses )

static

Definition at line 2622 of file GCNHazardRecognizer.cpp.

GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates()

static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates ( int  NumPasses )

static

Definition at line 2598 of file GCNHazardRecognizer.cpp.

GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates()

static int GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates ( int  NumPasses, bool  IsGFX950  )

static

Definition at line 2272 of file GCNHazardRecognizer.cpp.

GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates()

static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates ( int  NumPasses )

static

Definition at line 2300 of file GCNHazardRecognizer.cpp.

GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates()

static int GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates ( int  NumPasses, bool  IsGFX950  )

static

Definition at line 2261 of file GCNHazardRecognizer.cpp.

GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates()

static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates ( int  NumPasses )

static

Definition at line 2614 of file GCNHazardRecognizer.cpp.

GFX940_XDL_N_PassWriteVgprVALUWawWaitStates()

static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates ( int  NumPasses )

static

Definition at line 2606 of file GCNHazardRecognizer.cpp.

hasHazard()

template<typename StateT >

static bool hasHazard ( StateT  State, function_ref< HazardFnResult(StateT &, const MachineInstr &)>  IsHazard, function_ref< void(StateT &, const MachineInstr &)>  UpdateState, const MachineBasicBlock *  MBB, MachineBasicBlock::const_reverse_instr_iterator  I, DenseSet< const MachineBasicBlock * > &  Visited  )

static

Definition at line 466 of file GCNHazardRecognizer.cpp.

References hasHazard(), I, llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), llvm::MachineBasicBlock::instr_rend(), MBB, and llvm::MachineBasicBlock::predecessors().

Referenced by hasHazard().

insertNoopsInBundle()

static void insertNoopsInBundle ( MachineInstr *  MI, const SIInstrInfo &  TII, unsigned  Quantity  )

static

Definition at line 268 of file GCNHazardRecognizer.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), MI, and TII.

isDGEMM()

static bool isDGEMM ( unsigned  Opcode )

static

Definition at line 120 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::getMAIIsDGEMM().

Referenced by isXDL().

isDivFMas()

static bool isDivFMas ( unsigned  Opcode )

static

Definition at line 81 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

isLdsDma()

static bool isLdsDma ( const MachineInstr &  MI )

static

Definition at line 178 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isFLAT(), llvm::SIInstrInfo::isMUBUF(), llvm::SIInstrInfo::isVALU(), and MI.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

isPermlane()

static bool isPermlane ( const MachineInstr &  MI )

static

Definition at line 165 of file GCNHazardRecognizer.cpp.

References MI.

Referenced by llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

isRFE()

static bool isRFE ( unsigned  Opcode )

static

Definition at line 104 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

isRWLane()

static bool isRWLane ( unsigned  Opcode )

static

Definition at line 100 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

isSendMsgTraceDataOrGDS()

static bool isSendMsgTraceDataOrGDS ( const SIInstrInfo &  TII, const MachineInstr &  MI  )

static

Definition at line 139 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::getNamedOperandIdx(), MI, and TII.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

isSGetReg()

static bool isSGetReg ( unsigned  Opcode )

static

Definition at line 85 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

isSMovRel()

static bool isSMovRel ( unsigned  Opcode )

static

Definition at line 108 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

isSSetReg()

static bool isSSetReg ( unsigned  Opcode )

static

Definition at line 89 of file GCNHazardRecognizer.cpp.

Referenced by llvm::GCNHazardRecognizer::getHazardType(), and llvm::GCNHazardRecognizer::PreEmitNoopsCommon().

isStoreCountWaitZero()

static bool isStoreCountWaitZero ( const MachineInstr &  I )

static

Definition at line 1463 of file GCNHazardRecognizer.cpp.

References I.

isVCmpXWritesExec()

static bool isVCmpXWritesExec ( const SIInstrInfo &  TII, const SIRegisterInfo &  TRI, const MachineInstr &  MI  )

static

Definition at line 1224 of file GCNHazardRecognizer.cpp.

References MI, TII, and TRI.

isXDL()

static bool isXDL ( const GCNSubtarget &  ST, const MachineInstr &  MI  )

static

Definition at line 124 of file GCNHazardRecognizer.cpp.

References llvm::AMDGPU::getMAIIsGFX940XDL(), isDGEMM(), llvm::SIInstrInfo::isMAI(), and MI.

sgprPairNumber()

static std::optional< unsigned > sgprPairNumber ( Register  Reg, const SIRegisterInfo &  TRI  )

static

Definition at line 3111 of file GCNHazardRecognizer.cpp.

References TRI.

shouldRunLdsBranchVmemWARHazardFixup()

static bool shouldRunLdsBranchVmemWARHazardFixup ( const MachineFunction &  MF, const GCNSubtarget &  ST  )

static

Definition at line 1442 of file GCNHazardRecognizer.cpp.

References llvm::SIInstrInfo::isDS(), llvm::SIInstrInfo::isSegmentSpecificFLAT(), llvm::SIInstrInfo::isVMEM(), MBB, and MI.

Referenced by llvm::GCNHazardRecognizer::GCNHazardRecognizer().

updateGetPCBundle()

static void updateGetPCBundle ( MachineInstr *  NewMI )

static

Definition at line 2952 of file GCNHazardRecognizer.cpp.

References assert(), llvm::MachineBasicBlock::end(), End, llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getParent(), I, and llvm::MachineInstr::isBundled().

Variable Documentation

MaxExhaustiveHazardSearch

cl::opt< unsigned > MaxExhaustiveHazardSearch("amdgpu-max-exhaustive-hazard-search", cl::init(128), cl::Hidden, cl::desc("Maximum function size for exhausive hazard search")) ( "amdgpu-max-exhaustive-hazard-search"  , cl::init(128)  , cl::Hidden  , cl::desc("Maximum function size for exhausive hazard search")    )

static

MFMAPaddingRatio

cl::opt< unsigned, false, MFMAPaddingRatioParser > MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden, cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops.")) ( "amdgpu-mfma-padding-ratio"  , cl::init(0)  , cl::Hidden  , cl::desc("Fill a percentage of the latency between " "neighboring MFMA with s_nops.")    )

static