LLVM 20.0.0git
Classes | Macros | Functions | Variables
SIISelLowering.cpp File Reference

Custom DAG lowering for SI. More...

#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/ByteProvider.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <optional>

Go to the source code of this file.

Classes

struct  DotSrc
 

Macros

#define DEBUG_TYPE   "si-lower"
 

Functions

 STATISTIC (NumTailCalls, "Number of tail calls")
 
static bool denormalModeIsFlushAllF32 (const MachineFunction &MF)
 
static bool denormalModeIsFlushAllF64F16 (const MachineFunction &MF)
 
static unsigned findFirstFreeSGPR (CCState &CCInfo)
 
static EVT memVTFromLoadIntrData (const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
 
static EVT memVTFromLoadIntrReturn (const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
 
static void processPSInputArgs (SmallVectorImpl< ISD::InputArg > &Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector &Skipped, FunctionType *FType, SIMachineFunctionInfo *Info)
 
static ArgDescriptor allocateVGPR32Input (CCState &CCInfo, unsigned Mask=~0u, ArgDescriptor Arg=ArgDescriptor())
 
static ArgDescriptor allocateSGPR32InputImpl (CCState &CCInfo, const TargetRegisterClass *RC, unsigned NumArgRegs)
 
static void allocateFixedSGPRInputImpl (CCState &CCInfo, const TargetRegisterClass *RC, MCRegister Reg)
 
static void allocateSGPR32Input (CCState &CCInfo, ArgDescriptor &Arg)
 
static void allocateSGPR64Input (CCState &CCInfo, ArgDescriptor &Arg)
 
static void reservePrivateMemoryRegs (const TargetMachine &TM, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
 
static bool canGuaranteeTCO (CallingConv::ID CC)
 
static bool mayTailCallThisCC (CallingConv::ID CC)
 Return true if we might ever do TCO for calls with this calling convention.
 
static std::pair< MachineBasicBlock *, MachineBasicBlock * > splitBlockForLoop (MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop)
 
static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, const DebugLoc &DL, const MachineOperand &Idx, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
 
static MachineBasicBlock::iterator loadM0FromVGPR (const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
 
static std::pair< unsigned, int > computeIndirectRegAndOffset (const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, unsigned VecReg, int Offset)
 
static void setM0ToIndexFromSGPR (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
 
static Register getIndirectSGPRIdx (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
 
static MachineBasicBlockemitIndirectSrc (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
 
static MachineBasicBlockemitIndirectDst (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
 
static MachineBasicBlocklowerWaveReduce (MachineInstr &MI, MachineBasicBlock &BB, const GCNSubtarget &ST, unsigned Opc)
 
static SDValue adjustLoadValueTypeImpl (SDValue Result, EVT LoadVT, const SDLoc &DL, SelectionDAG &DAG, bool Unpacked)
 
static SDValue lowerICMPIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
 
static SDValue lowerFCMPIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
 
static SDValue lowerBALLOTIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
 
static SDValue lowerLaneOp (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
 
static SDNodefindUser (SDValue Value, unsigned Opcode)
 Helper function for LowerBRCOND.
 
static bool isKnownNonNull (SDValue Val, SelectionDAG &DAG, const AMDGPUTargetMachine &TM, unsigned AddrSpace)
 Return true if the value is a known valid address, such that a null check is not necessary.
 
static bool elementPairIsContiguous (ArrayRef< int > Mask, int Elt)
 
static SDValue buildPCRelGlobalAddress (SelectionDAG &DAG, const GlobalValue *GV, const SDLoc &DL, int64_t Offset, EVT PtrVT, unsigned GAFlags=SIInstrInfo::MO_NONE)
 
static SDValue emitNonHSAIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
 
static SDValue emitRemovedIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
 
static SDValue getBuildDwordsVector (SelectionDAG &DAG, SDLoc DL, ArrayRef< SDValue > Elts)
 
static SDValue padEltsToUndef (SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, SDValue Src, int ExtraElts)
 
static SDValue constructRetValue (SelectionDAG &DAG, MachineSDNode *Result, ArrayRef< EVT > ResultTypes, bool IsTexFail, bool Unpacked, bool IsD16, int DMaskPop, int NumVDataDwords, bool IsAtomicPacked16Bit, const SDLoc &DL)
 
static bool parseTexFail (SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE, SDValue *LWE, bool &IsTexFail)
 
static void packImage16bitOpsToDwords (SelectionDAG &DAG, SDValue Op, MVT PackVectorVT, SmallVectorImpl< SDValue > &PackedAddrs, unsigned DimIdx, unsigned EndIdx, unsigned NumGradients)
 
static SDValue selectSOffset (SDValue SOffset, SelectionDAG &DAG, const GCNSubtarget *Subtarget)
 
static SDValue getLoadExtOrTrunc (SelectionDAG &DAG, ISD::LoadExtType ExtType, SDValue Op, const SDLoc &SL, EVT VT)
 
static bool addressMayBeAccessedAsPrivate (const MachineMemOperand *MMO, const SIMachineFunctionInfo &Info)
 
static SDValue getFPBinOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain, SDNodeFlags Flags)
 
static SDValue getFPTernOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain, SDNodeFlags Flags)
 
static SDValue getSPDenormModeValue (uint32_t SPDenormMode, SelectionDAG &DAG, const SIMachineFunctionInfo *Info, const GCNSubtarget *ST)
 
static unsigned getBasePtrIndex (const MemSDNode *N)
 MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.
 
static bool bitOpWithConstantIsReducible (unsigned Opc, uint32_t Val)
 
static uint32_t getConstantPermuteMask (uint32_t C)
 
static uint32_t getPermuteMask (SDValue V)
 
static const std::optional< ByteProvider< SDValue > > calculateSrcByte (const SDValue Op, uint64_t DestByte, uint64_t SrcIndex=0, unsigned Depth=0)
 
static const std::optional< ByteProvider< SDValue > > calculateByteProvider (const SDValue &Op, unsigned Index, unsigned Depth, unsigned StartingIndex=0)
 
static bool isExtendedFrom16Bits (SDValue &Operand)
 
static bool addresses16Bits (int Mask)
 
static bool hasNon16BitAccesses (uint64_t PermMask, SDValue &Op, SDValue &OtherOp)
 
static SDValue getDWordFromOffset (SelectionDAG &DAG, SDLoc SL, SDValue Src, unsigned DWordOffset)
 
static SDValue matchPERM (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
 
static bool vectorEltWillFoldAway (SDValue Op)
 
static unsigned minMaxOpcToMin3Max3Opc (unsigned Opc)
 
static ConstantFPSDNodegetSplatConstantFP (SDValue Op)
 
static bool supportsMin3Max3 (const GCNSubtarget &Subtarget, unsigned Opc, EVT VT)
 
static bool isClampZeroToOne (SDValue A, SDValue B)
 
static SDValue strictFPExtFromF16 (SelectionDAG &DAG, SDValue Src)
 Return the source of an fp_extend from f16 to f32, or a converted FP constant.
 
static SDValue getMad64_32 (SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue N0, SDValue N1, SDValue N2, bool Signed)
 
static std::optional< ByteProvider< SDValue > > handleMulOperand (const SDValue &MulOperand)
 
static unsigned addPermMasks (unsigned First, unsigned Second)
 
static void placeSources (ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, SmallVectorImpl< DotSrc > &Src0s, SmallVectorImpl< DotSrc > &Src1s, int Step)
 
static SDValue resolveSources (SelectionDAG &DAG, SDLoc SL, SmallVectorImpl< DotSrc > &Srcs, bool IsSigned, bool IsAny)
 
static void fixMasks (SmallVectorImpl< DotSrc > &Srcs, unsigned ChainLength)
 
static bool isMul (const SDValue Op)
 
static std::optional< boolcheckDot4MulSignedness (const SDValue &N, ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, const SDValue &S0Op, const SDValue &S1Op, const SelectionDAG &DAG)
 
static unsigned SubIdx2Lane (unsigned Idx)
 Helper function for adjustWritemask.
 
static bool isFrameIndexOp (SDValue Op)
 
static SDValue buildSMovImm32 (SelectionDAG &DAG, const SDLoc &DL, uint64_t Val)
 
static bool isImmConstraint (StringRef Constraint)
 
static uint64_t clearUnusedBits (uint64_t Val, unsigned Size)
 
static int getAlignedAGPRClassID (unsigned UnalignedClassID)
 
static void knownBitsForWorkitemID (const GCNSubtarget &ST, GISelKnownBits &KB, KnownBits &Known, unsigned Dim)
 
static LLVM_ATTRIBUTE_UNUSED bool isCopyFromRegOfInlineAsm (const SDNode *N)
 
static bool atomicIgnoresDenormalModeOrFPModeIsFTZ (const AtomicRMWInst *RMW)
 
static OptimizationRemark emitAtomicRMWLegalRemark (const AtomicRMWInst *RMW)
 
static bool isHalf2OrBFloat2 (Type *Ty)
 
static bool isHalf2 (Type *Ty)
 
static bool isBFloat2 (Type *Ty)
 
static bool isAtomicRMWLegalIntTy (Type *Ty)
 
static bool isAtomicRMWLegalXChgTy (const AtomicRMWInst *RMW)
 
static bool globalMemoryFPAtomicIsLegal (const GCNSubtarget &Subtarget, const AtomicRMWInst *RMW, bool HasSystemScope)
 
static TargetLowering::AtomicExpansionKind atomicSupportedIfLegalIntType (const AtomicRMWInst *RMW)
 
static bool hasCFUser (const Value *V, SmallPtrSet< const Value *, 16 > &Visited, unsigned WaveSize)
 

Variables

static cl::opt< boolDisableLoopAlignment ("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false))
 
static cl::opt< boolUseDivergentRegisterIndexing ("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false))
 

Detailed Description

Custom DAG lowering for SI.

Definition in file SIISelLowering.cpp.

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "si-lower"

Definition at line 50 of file SIISelLowering.cpp.

Function Documentation

◆ addPermMasks()

static unsigned addPermMasks ( unsigned  First,
unsigned  Second 
)
static

Definition at line 13800 of file SIISelLowering.cpp.

References assert(), and llvm::First.

Referenced by placeSources(), and resolveSources().

◆ addresses16Bits()

static bool addresses16Bits ( int  Mask)
static

Definition at line 12092 of file SIISelLowering.cpp.

References assert().

Referenced by hasNon16BitAccesses().

◆ addressMayBeAccessedAsPrivate()

static bool addressMayBeAccessedAsPrivate ( const MachineMemOperand MMO,
const SIMachineFunctionInfo Info 
)
static

Definition at line 10246 of file SIISelLowering.cpp.

References Info.

◆ adjustLoadValueTypeImpl()

static SDValue adjustLoadValueTypeImpl ( SDValue  Result,
EVT  LoadVT,
const SDLoc DL,
SelectionDAG DAG,
bool  Unpacked 
)
static

◆ allocateFixedSGPRInputImpl()

static void allocateFixedSGPRInputImpl ( CCState CCInfo,
const TargetRegisterClass RC,
MCRegister  Reg 
)
static

◆ allocateSGPR32Input()

static void allocateSGPR32Input ( CCState CCInfo,
ArgDescriptor Arg 
)
static

◆ allocateSGPR32InputImpl()

static ArgDescriptor allocateSGPR32InputImpl ( CCState CCInfo,
const TargetRegisterClass RC,
unsigned  NumArgRegs 
)
static

◆ allocateSGPR64Input()

static void allocateSGPR64Input ( CCState CCInfo,
ArgDescriptor Arg 
)
static

◆ allocateVGPR32Input()

static ArgDescriptor allocateVGPR32Input ( CCState CCInfo,
unsigned  Mask = ~0u,
ArgDescriptor  Arg = ArgDescriptor() 
)
static

◆ atomicIgnoresDenormalModeOrFPModeIsFTZ()

static bool atomicIgnoresDenormalModeOrFPModeIsFTZ ( const AtomicRMWInst RMW)
static

◆ atomicSupportedIfLegalIntType()

static TargetLowering::AtomicExpansionKind atomicSupportedIfLegalIntType ( const AtomicRMWInst RMW)
static
Returns
Action to perform on AtomicRMWInsts for integer operations.

Definition at line 16172 of file SIISelLowering.cpp.

References llvm::TargetLoweringBase::CmpXChg, llvm::Value::getType(), isAtomicRMWLegalIntTy(), and llvm::TargetLoweringBase::None.

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ bitOpWithConstantIsReducible()

static bool bitOpWithConstantIsReducible ( unsigned  Opc,
uint32_t  Val 
)
static

Definition at line 11396 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::OR, and llvm::ISD::XOR.

◆ buildPCRelGlobalAddress()

static SDValue buildPCRelGlobalAddress ( SelectionDAG DAG,
const GlobalValue GV,
const SDLoc DL,
int64_t  Offset,
EVT  PtrVT,
unsigned  GAFlags = SIInstrInfo::MO_NONE 
)
static

◆ buildSMovImm32()

static SDValue buildSMovImm32 ( SelectionDAG DAG,
const SDLoc DL,
uint64_t  Val 
)
static

◆ calculateByteProvider()

static const std::optional< ByteProvider< SDValue > > calculateByteProvider ( const SDValue Op,
unsigned  Index,
unsigned  Depth,
unsigned  StartingIndex = 0 
)
static

◆ calculateSrcByte()

static const std::optional< ByteProvider< SDValue > > calculateSrcByte ( const SDValue  Op,
uint64_t  DestByte,
uint64_t  SrcIndex = 0,
unsigned  Depth = 0 
)
static

◆ canGuaranteeTCO()

static bool canGuaranteeTCO ( CallingConv::ID  CC)
static

Definition at line 3511 of file SIISelLowering.cpp.

References CC, and llvm::CallingConv::Fast.

◆ checkDot4MulSignedness()

static std::optional< bool > checkDot4MulSignedness ( const SDValue N,
ByteProvider< SDValue > &  Src0,
ByteProvider< SDValue > &  Src1,
const SDValue S0Op,
const SDValue S1Op,
const SelectionDAG DAG 
)
static

◆ clearUnusedBits()

static uint64_t clearUnusedBits ( uint64_t  Val,
unsigned  Size 
)
static

◆ computeIndirectRegAndOffset()

static std::pair< unsigned, int > computeIndirectRegAndOffset ( const SIRegisterInfo TRI,
const TargetRegisterClass SuperRC,
unsigned  VecReg,
int  Offset 
)
static

◆ constructRetValue()

static SDValue constructRetValue ( SelectionDAG DAG,
MachineSDNode Result,
ArrayRef< EVT ResultTypes,
bool  IsTexFail,
bool  Unpacked,
bool  IsD16,
int  DMaskPop,
int  NumVDataDwords,
bool  IsAtomicPacked16Bit,
const SDLoc DL 
)
static

◆ denormalModeIsFlushAllF32()

static bool denormalModeIsFlushAllF32 ( const MachineFunction MF)
static

◆ denormalModeIsFlushAllF64F16()

static bool denormalModeIsFlushAllF64F16 ( const MachineFunction MF)
static

◆ elementPairIsContiguous()

static bool elementPairIsContiguous ( ArrayRef< int >  Mask,
int  Elt 
)
static

Definition at line 7382 of file SIISelLowering.cpp.

References assert().

◆ emitAtomicRMWLegalRemark()

static OptimizationRemark emitAtomicRMWLegalRemark ( const AtomicRMWInst RMW)
static

◆ emitIndirectDst()

static MachineBasicBlock * emitIndirectDst ( MachineInstr MI,
MachineBasicBlock MBB,
const GCNSubtarget ST 
)
static

◆ emitIndirectSrc()

static MachineBasicBlock * emitIndirectSrc ( MachineInstr MI,
MachineBasicBlock MBB,
const GCNSubtarget ST 
)
static

◆ emitLoadM0FromVGPRLoop()

static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop ( const SIInstrInfo TII,
MachineRegisterInfo MRI,
MachineBasicBlock OrigBB,
MachineBasicBlock LoopBB,
const DebugLoc DL,
const MachineOperand Idx,
unsigned  InitReg,
unsigned  ResultReg,
unsigned  PhiReg,
unsigned  InitSaveExecReg,
int  Offset,
bool  UseGPRIdxMode,
Register SGPRIdxReg 
)
static

◆ emitNonHSAIntrinsicError()

static SDValue emitNonHSAIntrinsicError ( SelectionDAG DAG,
const SDLoc DL,
EVT  VT 
)
static

◆ emitRemovedIntrinsicError()

static SDValue emitRemovedIntrinsicError ( SelectionDAG DAG,
const SDLoc DL,
EVT  VT 
)
static

◆ findFirstFreeSGPR()

static unsigned findFirstFreeSGPR ( CCState CCInfo)
static

◆ findUser()

static SDNode * findUser ( SDValue  Value,
unsigned  Opcode 
)
static

Helper function for LowerBRCOND.

Definition at line 6479 of file SIISelLowering.cpp.

References I, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().

◆ fixMasks()

static void fixMasks ( SmallVectorImpl< DotSrc > &  Srcs,
unsigned  ChainLength 
)
static

Definition at line 13966 of file SIISelLowering.cpp.

◆ getAlignedAGPRClassID()

static int getAlignedAGPRClassID ( unsigned  UnalignedClassID)
static

Definition at line 15622 of file SIISelLowering.cpp.

Referenced by llvm::SITargetLowering::finalizeLowering().

◆ getBasePtrIndex()

static unsigned getBasePtrIndex ( const MemSDNode N)
static

MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.

Theoretically we would also need to check the specific intrinsic, but they all place the pointer operand first.

Definition at line 11362 of file SIISelLowering.cpp.

References llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, N, and llvm::ISD::STORE.

Referenced by llvm::SITargetLowering::hasMemSDNodeUser().

◆ getBuildDwordsVector()

static SDValue getBuildDwordsVector ( SelectionDAG DAG,
SDLoc  DL,
ArrayRef< SDValue Elts 
)
static

◆ getConstantPermuteMask()

static uint32_t getConstantPermuteMask ( uint32_t  C)
static

Definition at line 11448 of file SIISelLowering.cpp.

References llvm::CallingConv::C.

Referenced by getPermuteMask().

◆ getDWordFromOffset()

static SDValue getDWordFromOffset ( SelectionDAG DAG,
SDLoc  SL,
SDValue  Src,
unsigned  DWordOffset 
)
static

◆ getFPBinOp()

static SDValue getFPBinOp ( SelectionDAG DAG,
unsigned  Opcode,
const SDLoc SL,
EVT  VT,
SDValue  A,
SDValue  B,
SDValue  GlueChain,
SDNodeFlags  Flags 
)
static

◆ getFPTernOp()

static SDValue getFPTernOp ( SelectionDAG DAG,
unsigned  Opcode,
const SDLoc SL,
EVT  VT,
SDValue  A,
SDValue  B,
SDValue  C,
SDValue  GlueChain,
SDNodeFlags  Flags 
)
static

◆ getIndirectSGPRIdx()

static Register getIndirectSGPRIdx ( const SIInstrInfo TII,
MachineRegisterInfo MRI,
MachineInstr MI,
int  Offset 
)
static

◆ getLoadExtOrTrunc()

static SDValue getLoadExtOrTrunc ( SelectionDAG DAG,
ISD::LoadExtType  ExtType,
SDValue  Op,
const SDLoc SL,
EVT  VT 
)
static

◆ getMad64_32()

static SDValue getMad64_32 ( SelectionDAG DAG,
const SDLoc SL,
EVT  VT,
SDValue  N0,
SDValue  N1,
SDValue  N2,
bool  Signed 
)
static

◆ getPermuteMask()

static uint32_t getPermuteMask ( SDValue  V)
static

◆ getSPDenormModeValue()

static SDValue getSPDenormModeValue ( uint32_t  SPDenormMode,
SelectionDAG DAG,
const SIMachineFunctionInfo Info,
const GCNSubtarget ST 
)
static

Definition at line 10629 of file SIISelLowering.cpp.

References assert(), llvm::SelectionDAG::getTargetConstant(), and Info.

◆ getSplatConstantFP()

static ConstantFPSDNode * getSplatConstantFP ( SDValue  Op)
static

Definition at line 13088 of file SIISelLowering.cpp.

References llvm::CallingConv::C.

◆ globalMemoryFPAtomicIsLegal()

static bool globalMemoryFPAtomicIsLegal ( const GCNSubtarget Subtarget,
const AtomicRMWInst RMW,
bool  HasSystemScope 
)
static
Returns
true if it's valid to emit a native instruction for RMW, based on the properties of the target memory.

Definition at line 16151 of file SIISelLowering.cpp.

References llvm::Instruction::hasMetadata(), and llvm::GCNSubtarget::supportsAgentScopeFineGrainedRemoteMemoryAtomics().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ handleMulOperand()

static std::optional< ByteProvider< SDValue > > handleMulOperand ( const SDValue MulOperand)
static

Definition at line 13788 of file SIISelLowering.cpp.

References calculateByteProvider().

◆ hasCFUser()

static bool hasCFUser ( const Value V,
SmallPtrSet< const Value *, 16 > &  Visited,
unsigned  WaveSize 
)
static

◆ hasNon16BitAccesses()

static bool hasNon16BitAccesses ( uint64_t  PermMask,
SDValue Op,
SDValue OtherOp 
)
static

Definition at line 12111 of file SIISelLowering.cpp.

References addresses16Bits(), isExtendedFrom16Bits(), and llvm::peekThroughBitcasts().

Referenced by matchPERM().

◆ isAtomicRMWLegalIntTy()

static bool isAtomicRMWLegalIntTy ( Type Ty)
static
Returns
true if atomicrmw integer ops work for the type.

Definition at line 16117 of file SIISelLowering.cpp.

References IT.

Referenced by atomicSupportedIfLegalIntType(), and isAtomicRMWLegalXChgTy().

◆ isAtomicRMWLegalXChgTy()

static bool isAtomicRMWLegalXChgTy ( const AtomicRMWInst RMW)
static

◆ isBFloat2()

static bool isBFloat2 ( Type Ty)
static

◆ isClampZeroToOne()

static bool isClampZeroToOne ( SDValue  A,
SDValue  B 
)
static

Definition at line 13257 of file SIISelLowering.cpp.

References A, and B.

◆ isCopyFromRegOfInlineAsm()

static LLVM_ATTRIBUTE_UNUSED bool isCopyFromRegOfInlineAsm ( const SDNode N)
static

◆ isExtendedFrom16Bits()

static bool isExtendedFrom16Bits ( SDValue Operand)
static

◆ isFrameIndexOp()

static bool isFrameIndexOp ( SDValue  Op)
static

◆ isHalf2()

static bool isHalf2 ( Type Ty)
static

◆ isHalf2OrBFloat2()

static bool isHalf2OrBFloat2 ( Type Ty)
static

◆ isImmConstraint()

static bool isImmConstraint ( StringRef  Constraint)
static

◆ isKnownNonNull()

static bool isKnownNonNull ( SDValue  Val,
SelectionDAG DAG,
const AMDGPUTargetMachine TM,
unsigned  AddrSpace 
)
static

Return true if the value is a known valid address, such that a null check is not necessary.

Definition at line 7040 of file SIISelLowering.cpp.

References TM.

◆ isMul()

static bool isMul ( const SDValue  Op)
static

◆ knownBitsForWorkitemID()

static void knownBitsForWorkitemID ( const GCNSubtarget ST,
GISelKnownBits KB,
KnownBits Known,
unsigned  Dim 
)
static

◆ loadM0FromVGPR()

static MachineBasicBlock::iterator loadM0FromVGPR ( const SIInstrInfo TII,
MachineBasicBlock MBB,
MachineInstr MI,
unsigned  InitResultReg,
unsigned  PhiReg,
int  Offset,
bool  UseGPRIdxMode,
Register SGPRIdxReg 
)
static

◆ lowerBALLOTIntrinsic()

static SDValue lowerBALLOTIntrinsic ( const SITargetLowering TLI,
SDNode N,
SelectionDAG DAG 
)
static

◆ lowerFCMPIntrinsic()

static SDValue lowerFCMPIntrinsic ( const SITargetLowering TLI,
SDNode N,
SelectionDAG DAG 
)
static

◆ lowerICMPIntrinsic()

static SDValue lowerICMPIntrinsic ( const SITargetLowering TLI,
SDNode N,
SelectionDAG DAG 
)
static

◆ lowerLaneOp()

static SDValue lowerLaneOp ( const SITargetLowering TLI,
SDNode N,
SelectionDAG DAG 
)
static

◆ lowerWaveReduce()

static MachineBasicBlock * lowerWaveReduce ( MachineInstr MI,
MachineBasicBlock BB,
const GCNSubtarget ST,
unsigned  Opc 
)
static

◆ matchPERM()

static SDValue matchPERM ( SDNode N,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ mayTailCallThisCC()

static bool mayTailCallThisCC ( CallingConv::ID  CC)
static

Return true if we might ever do TCO for calls with this calling convention.

Definition at line 3516 of file SIISelLowering.cpp.

References llvm::CallingConv::AMDGPU_Gfx, llvm::CallingConv::C, canGuaranteeTCO(), and CC.

◆ memVTFromLoadIntrData()

static EVT memVTFromLoadIntrData ( const SITargetLowering TLI,
const DataLayout DL,
Type Ty,
unsigned  MaxNumLanes 
)
static

◆ memVTFromLoadIntrReturn()

static EVT memVTFromLoadIntrReturn ( const SITargetLowering TLI,
const DataLayout DL,
Type Ty,
unsigned  MaxNumLanes 
)
static

Definition at line 1134 of file SIISelLowering.cpp.

References assert(), DL, and memVTFromLoadIntrData().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

◆ minMaxOpcToMin3Max3Opc()

static unsigned minMaxOpcToMin3Max3Opc ( unsigned  Opc)
static

◆ packImage16bitOpsToDwords()

static void packImage16bitOpsToDwords ( SelectionDAG DAG,
SDValue  Op,
MVT  PackVectorVT,
SmallVectorImpl< SDValue > &  PackedAddrs,
unsigned  DimIdx,
unsigned  EndIdx,
unsigned  NumGradients 
)
static

◆ padEltsToUndef()

static SDValue padEltsToUndef ( SelectionDAG DAG,
const SDLoc DL,
EVT  CastVT,
SDValue  Src,
int  ExtraElts 
)
static

◆ parseTexFail()

static bool parseTexFail ( SDValue  TexFailCtrl,
SelectionDAG DAG,
SDValue TFE,
SDValue LWE,
bool IsTexFail 
)
static

◆ placeSources()

static void placeSources ( ByteProvider< SDValue > &  Src0,
ByteProvider< SDValue > &  Src1,
SmallVectorImpl< DotSrc > &  Src0s,
SmallVectorImpl< DotSrc > &  Src1s,
int  Step 
)
static

◆ processPSInputArgs()

static void processPSInputArgs ( SmallVectorImpl< ISD::InputArg > &  Splits,
CallingConv::ID  CallConv,
ArrayRef< ISD::InputArg Ins,
BitVector Skipped,
FunctionType FType,
SIMachineFunctionInfo Info 
)
static

◆ reservePrivateMemoryRegs()

static void reservePrivateMemoryRegs ( const TargetMachine TM,
MachineFunction MF,
const SIRegisterInfo TRI,
SIMachineFunctionInfo Info 
)
static

◆ resolveSources()

static SDValue resolveSources ( SelectionDAG DAG,
SDLoc  SL,
SmallVectorImpl< DotSrc > &  Srcs,
bool  IsSigned,
bool  IsAny 
)
static

◆ selectSOffset()

static SDValue selectSOffset ( SDValue  SOffset,
SelectionDAG DAG,
const GCNSubtarget Subtarget 
)
static

◆ setM0ToIndexFromSGPR()

static void setM0ToIndexFromSGPR ( const SIInstrInfo TII,
MachineRegisterInfo MRI,
MachineInstr MI,
int  Offset 
)
static

◆ splitBlockForLoop()

static std::pair< MachineBasicBlock *, MachineBasicBlock * > splitBlockForLoop ( MachineInstr MI,
MachineBasicBlock MBB,
bool  InstInLoop 
)
static

◆ STATISTIC()

STATISTIC ( NumTailCalls  ,
"Number of tail calls"   
)

◆ strictFPExtFromF16()

static SDValue strictFPExtFromF16 ( SelectionDAG DAG,
SDValue  Src 
)
static

Return the source of an fp_extend from f16 to f32, or a converted FP constant.

Definition at line 13525 of file SIISelLowering.cpp.

References llvm::APFloat::convert(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstantFP(), llvm::APFloatBase::IEEEhalf(), and llvm::APFloatBase::rmNearestTiesToEven.

◆ SubIdx2Lane()

static unsigned SubIdx2Lane ( unsigned  Idx)
static

Helper function for adjustWritemask.

Definition at line 14809 of file SIISelLowering.cpp.

References Idx.

◆ supportsMin3Max3()

static bool supportsMin3Max3 ( const GCNSubtarget Subtarget,
unsigned  Opc,
EVT  VT 
)
static

◆ vectorEltWillFoldAway()

static bool vectorEltWillFoldAway ( SDValue  Op)
static

Definition at line 12953 of file SIISelLowering.cpp.

Variable Documentation

◆ DisableLoopAlignment

cl::opt< bool > DisableLoopAlignment("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false)) ( "amdgpu-disable-loop-alignment"  ,
cl::desc("Do not align and prefetch loops")  ,
cl::init(false)   
)
static

◆ UseDivergentRegisterIndexing

cl::opt< bool > UseDivergentRegisterIndexing("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false)) ( "amdgpu-use-divergent-register-indexing"  ,
cl::Hidden  ,
cl::desc("Use indirect register addressing for divergent indexes")  ,
cl::init(false)   
)
static