SIISelLowering.cpp File Reference

#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetMachine.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/LegacyDivergenceAnalysis.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"

Include dependency graph for SIISelLowering.cpp:

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "si-lower"

Functions
	STATISTIC (NumTailCalls, "Number of tail calls")
static bool	hasFP32Denormals (const MachineFunction &MF)
static bool	hasFP64FP16Denormals (const MachineFunction &MF)
static unsigned	findFirstFreeSGPR (CCState &CCInfo)
static EVT	memVTFromImageData (Type *Ty, unsigned DMaskLanes)
static EVT	memVTFromImageReturn (Type *Ty, unsigned DMaskLanes)
static void	processPSInputArgs (SmallVectorImpl< ISD::InputArg > &Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector &Skipped, FunctionType *FType, SIMachineFunctionInfo *Info)
static ArgDescriptor	allocateVGPR32Input (CCState &CCInfo, unsigned Mask=~0u, ArgDescriptor Arg=ArgDescriptor())
static ArgDescriptor	allocateSGPR32InputImpl (CCState &CCInfo, const TargetRegisterClass *RC, unsigned NumArgRegs)
static void	allocateFixedSGPRInputImpl (CCState &CCInfo, const TargetRegisterClass *RC, MCRegister Reg)
static void	allocateSGPR32Input (CCState &CCInfo, ArgDescriptor &Arg)
static void	allocateSGPR64Input (CCState &CCInfo, ArgDescriptor &Arg)
static void	reservePrivateMemoryRegs (const TargetMachine &TM, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static bool	canGuaranteeTCO (CallingConv::ID CC)
static bool	mayTailCallThisCC (CallingConv::ID CC)
	Return true if we might ever do TCO for calls with this calling convention. More...
static std::pair< MachineBasicBlock *, MachineBasicBlock * >	splitBlockForLoop (MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop)
static MachineBasicBlock::iterator	emitLoadM0FromVGPRLoop (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, const DebugLoc &DL, const MachineOperand &Idx, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static MachineBasicBlock::iterator	loadM0FromVGPR (const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static std::pair< unsigned, int >	computeIndirectRegAndOffset (const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, unsigned VecReg, int Offset)
static void	setM0ToIndexFromSGPR (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static Register	getIndirectSGPRIdx (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static MachineBasicBlock *	emitIndirectSrc (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static MachineBasicBlock *	emitIndirectDst (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static SDValue	adjustLoadValueTypeImpl (SDValue Result, EVT LoadVT, const SDLoc &DL, SelectionDAG &DAG, bool Unpacked)
static SDValue	lowerICMPIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue	lowerFCMPIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue	lowerBALLOTIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDNode *	findUser (SDValue Value, unsigned Opcode)
	Helper function for LowerBRCOND. More...
static bool	elementPairIsContiguous (ArrayRef< int > Mask, int Elt)
static SDValue	buildPCRelGlobalAddress (SelectionDAG &DAG, const GlobalValue *GV, const SDLoc &DL, int64_t Offset, EVT PtrVT, unsigned GAFlags=SIInstrInfo::MO_NONE)
static SDValue	emitNonHSAIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue	emitRemovedIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue	getBuildDwordsVector (SelectionDAG &DAG, SDLoc DL, ArrayRef< SDValue > Elts)
static SDValue	padEltsToUndef (SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, SDValue Src, int ExtraElts)
static SDValue	constructRetValue (SelectionDAG &DAG, MachineSDNode *Result, ArrayRef< EVT > ResultTypes, bool IsTexFail, bool Unpacked, bool IsD16, int DMaskPop, int NumVDataDwords, const SDLoc &DL)
static bool	parseTexFail (SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE, SDValue *LWE, bool &IsTexFail)
static void	packImage16bitOpsToDwords (SelectionDAG &DAG, SDValue Op, MVT PackVectorVT, SmallVectorImpl< SDValue > &PackedAddrs, unsigned DimIdx, unsigned EndIdx, unsigned NumGradients)
static void	updateBufferMMO (MachineMemOperand *MMO, SDValue VOffset, SDValue SOffset, SDValue Offset, SDValue VIndex=SDValue())
	Update MMO based on the offset inputs to an intrinsic. More...
static unsigned	getIdxEn (SDValue VIndex)
static SDValue	getLoadExtOrTrunc (SelectionDAG &DAG, ISD::LoadExtType ExtType, SDValue Op, const SDLoc &SL, EVT VT)
static SDValue	getFPBinOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain, SDNodeFlags Flags)
static SDValue	getFPTernOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain, SDNodeFlags Flags)
static SDValue	getSPDenormModeValue (int SPDenormMode, SelectionDAG &DAG, const SDLoc &SL, const GCNSubtarget *ST)
static unsigned	getBasePtrIndex (const MemSDNode *N)
	MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID. More...
static bool	bitOpWithConstantIsReducible (unsigned Opc, uint32_t Val)
static bool	isBoolSGPR (SDValue V)
static uint32_t	getConstantPermuteMask (uint32_t C)
static uint32_t	getPermuteMask (SelectionDAG &DAG, SDValue V)
static bool	vectorEltWillFoldAway (SDValue Op)
static unsigned	minMaxOpcToMin3Max3Opc (unsigned Opc)
static ConstantFPSDNode *	getSplatConstantFP (SDValue Op)
static bool	isClampZeroToOne (SDValue A, SDValue B)
static bool	shouldExpandVectorDynExt (SDNode *N)
static SDValue	getMad64_32 (SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue N0, SDValue N1, SDValue N2, bool Signed)
static unsigned	SubIdx2Lane (unsigned Idx)
	Helper function for adjustWritemask. More...
static bool	isFrameIndexOp (SDValue Op)
static SDValue	buildSMovImm32 (SelectionDAG &DAG, const SDLoc &DL, uint64_t Val)
static bool	isImmConstraint (StringRef Constraint)
static uint64_t	clearUnusedBits (uint64_t Val, unsigned Size)
static int	getAlignedAGPRClassID (unsigned UnalignedClassID)
static void	knownBitsForWorkitemID (const GCNSubtarget &ST, GISelKnownBits &KB, KnownBits &Known, unsigned Dim)
static LLVM_ATTRIBUTE_UNUSED bool	isCopyFromRegOfInlineAsm (const SDNode *N)
static bool	fpModeMatchesGlobalFPAtomicMode (const AtomicRMWInst *RMW)
static bool	hasCFUser (const Value *V, SmallPtrSet< const Value *, 16 > &Visited, unsigned WaveSize)

Variables
static cl::opt< bool >	DisableLoopAlignment ("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false))
static cl::opt< bool >	VGPRReserveforSGPRSpill ("amdgpu-reserve-vgpr-for-sgpr-spill", cl::desc("Allocates one VGPR for future SGPR Spill"), cl::init(true))
static cl::opt< bool >	UseDivergentRegisterIndexing ("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false))

Detailed Description

Custom DAG lowering for SI

Definition in file SIISelLowering.cpp.

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "si-lower"

Definition at line 38 of file SIISelLowering.cpp.

Function Documentation

adjustLoadValueTypeImpl()

static SDValue adjustLoadValueTypeImpl ( SDValue  Result, EVT  LoadVT, const SDLoc &  DL, SelectionDAG &  DAG, bool  Unpacked  )

static

Definition at line 4684 of file SIISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::i16, llvm::EVT::isVector(), and llvm::ISD::TRUNCATE.

Referenced by constructRetValue().

allocateFixedSGPRInputImpl()

static void allocateFixedSGPRInputImpl ( CCState &  CCInfo, const TargetRegisterClass *  RC, MCRegister  Reg  )

static

Definition at line 1982 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), llvm::CCState::getMachineFunction(), and Reg.

Referenced by allocateSGPR32Input(), and allocateSGPR64Input().

allocateSGPR32Input()

static void allocateSGPR32Input ( CCState &  CCInfo, ArgDescriptor &  Arg  )

static

Definition at line 1991 of file SIISelLowering.cpp.

References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and Arg.

Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().

allocateSGPR32InputImpl()

static ArgDescriptor allocateSGPR32InputImpl ( CCState &  CCInfo, const TargetRegisterClass *  RC, unsigned  NumArgRegs  )

static

Definition at line 1962 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), llvm::TargetRegisterClass::begin(), llvm::ArgDescriptor::createRegister(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::makeArrayRef(), Reg, llvm::report_fatal_error(), and llvm::ArrayRef< T >::size().

Referenced by allocateSGPR32Input(), and allocateSGPR64Input().

allocateSGPR64Input()

static void allocateSGPR64Input ( CCState &  CCInfo, ArgDescriptor &  Arg  )

static

Definition at line 1999 of file SIISelLowering.cpp.

References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and Arg.

Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().

allocateVGPR32Input()

static ArgDescriptor allocateVGPR32Input ( CCState &  CCInfo, unsigned  Mask = ~0u, ArgDescriptor  Arg = ArgDescriptor()  )

static

Definition at line 1937 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), Align, llvm::CCState::AllocateReg(), llvm::CCState::AllocateStack(), Arg, assert(), llvm::sys::path::begin(), llvm::ArgDescriptor::createArg(), llvm::ArgDescriptor::createRegister(), llvm::ArgDescriptor::createStack(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::MachineFunction::getRegInfo(), llvm::makeArrayRef(), llvm::BitmaskEnumDetail::Mask(), Offset, Reg, llvm::LLT::scalar(), llvm::MachineRegisterInfo::setType(), and llvm::ArrayRef< T >::size().

Referenced by llvm::SITargetLowering::allocateSpecialInputVGPRs().

bitOpWithConstantIsReducible()

static bool bitOpWithConstantIsReducible ( unsigned  Opc, uint32_t  Val  )

static

Definition at line 9047 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::OR, and llvm::ISD::XOR.

buildPCRelGlobalAddress()

static SDValue buildPCRelGlobalAddress ( SelectionDAG &  DAG, const GlobalValue *  GV, const SDLoc &  DL, int64_t  Offset, EVT  PtrVT, unsigned  GAFlags = SIInstrInfo::MO_NONE  )

static

Definition at line 5760 of file SIISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::MVT::i32, llvm::isInt< 32 >(), llvm::SIInstrInfo::MO_NONE, Offset, and llvm::AMDGPUISD::PC_ADD_REL_OFFSET.

buildSMovImm32()

static SDValue buildSMovImm32 ( SelectionDAG &  DAG, const SDLoc &  DL, uint64_t  Val  )

static

Definition at line 11509 of file SIISelLowering.cpp.

References DL, llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::MVT::i32.

Referenced by llvm::SITargetLowering::buildRSRC(), and llvm::SITargetLowering::wrapAddr64Rsrc().

canGuaranteeTCO()

static bool canGuaranteeTCO ( CallingConv::ID  CC )

static

Definition at line 2918 of file SIISelLowering.cpp.

References llvm::CallingConv::Fast.

Referenced by llvm::SITargetLowering::isEligibleForTailCallOptimization(), and mayTailCallThisCC().

clearUnusedBits()

static uint64_t clearUnusedBits ( uint64_t  Val, unsigned  Size  )

static

Definition at line 11702 of file SIISelLowering.cpp.

References llvm::AMDGPU::isInlinableIntLiteral(), and llvm::Check::Size.

Referenced by llvm::APInt::APInt(), llvm::APInt::ashrInPlace(), llvm::SITargetLowering::checkAsmConstraintVal(), llvm::APInt::flipAllBits(), llvm::SITargetLowering::LowerAsmOperandForConstraint(), llvm::APInt::operator<<=(), llvm::APInt::operator=(), llvm::APInt::operator^=(), llvm::APInt::operator|=(), and llvm::APInt::setAllBits().

computeIndirectRegAndOffset()

static std::pair<unsigned, int> computeIndirectRegAndOffset ( const SIRegisterInfo &  TRI, const TargetRegisterClass *  SuperRC, unsigned  VecReg, int  Offset  )

static

Definition at line 3724 of file SIISelLowering.cpp.

References llvm::TargetRegisterInfo::getRegSizeInBits(), llvm::SIRegisterInfo::getSubRegFromChannel(), Offset, and TRI.

Referenced by emitIndirectDst(), and emitIndirectSrc().

constructRetValue()

static SDValue constructRetValue ( SelectionDAG &  DAG, MachineSDNode *  Result, ArrayRef< EVT >  ResultTypes, bool  IsTexFail, bool  Unpacked, bool  IsD16, int  DMaskPop, int  NumVDataDwords, const SDLoc &  DL  )

static

Definition at line 5962 of file SIISelLowering.cpp.

References adjustLoadValueTypeImpl(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::MVT::i32, llvm::EVT::isVector(), llvm::MVT::isVector(), padEltsToUndef(), and llvm::ISD::TRUNCATE.

elementPairIsContiguous()

static bool elementPairIsContiguous ( ArrayRef< int >  Mask, int  Elt  )

static

Definition at line 5643 of file SIISelLowering.cpp.

References assert(), and llvm::BitmaskEnumDetail::Mask().

emitIndirectDst()

static MachineBasicBlock* emitIndirectDst ( MachineInstr &  MI, MachineBasicBlock &  MBB, const GCNSubtarget &  ST  )

static

Definition at line 3863 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::MachineRegisterInfo::clearKillFlags(), computeIndirectRegAndOffset(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), llvm::TargetRegisterInfo::getRegSizeInBits(), I, llvm::MachineOperand::isReg(), loadM0FromVGPR(), MBB, MI, MRI, Offset, setM0ToIndexFromSGPR(), llvm::ARM_MB::ST, SubReg, TII, TRI, and val.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

emitIndirectSrc()

static MachineBasicBlock* emitIndirectSrc ( MachineInstr &  MI, MachineBasicBlock &  MBB, const GCNSubtarget &  ST  )

static

Definition at line 3777 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), computeIndirectRegAndOffset(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), llvm::TargetRegisterInfo::getRegSizeInBits(), I, llvm::RegState::Implicit, loadM0FromVGPR(), MBB, MI, MRI, Offset, setM0ToIndexFromSGPR(), llvm::ARM_MB::ST, SubReg, TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

emitLoadM0FromVGPRLoop()

static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop ( const SIInstrInfo *  TII, MachineRegisterInfo &  MRI, MachineBasicBlock &  OrigBB, MachineBasicBlock &  LoopBB, const DebugLoc &  DL, const MachineOperand &  Idx, unsigned  InitReg, unsigned  ResultReg, unsigned  PhiReg, unsigned  InitSaveExecReg, int  Offset, bool  UseGPRIdxMode, Register &  SGPRIdxReg  )

static

Definition at line 3583 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineOperand::getSubReg(), llvm::MachineFunction::getSubtarget(), llvm::getUndefRegState(), I, llvm::MachineOperand::isUndef(), llvm::RegState::Kill, llvm::M0(), MRI, Offset, llvm::MachineRegisterInfo::setSimpleHint(), llvm::ARM_MB::ST, TII, and TRI.

Referenced by loadM0FromVGPR().

emitNonHSAIntrinsicError()

static SDValue emitNonHSAIntrinsicError ( SelectionDAG &  DAG, const SDLoc &  DL, EVT  VT  )

static

Definition at line 5894 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().

emitRemovedIntrinsicError()

static SDValue emitRemovedIntrinsicError ( SelectionDAG &  DAG, const SDLoc &  DL, EVT  VT  )

static

Definition at line 5903 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().

findFirstFreeSGPR()

static unsigned findFirstFreeSGPR ( CCState &  CCInfo )

static

Definition at line 67 of file SIISelLowering.cpp.

References llvm::CCState::isAllocated(), llvm_unreachable, llvm::AMDGPU::HSAMD::Kernel::CodeProps::Key::NumSGPRs, and Reg.

Referenced by llvm::SITargetLowering::allocateSystemSGPRs().

findUser()

static SDNode* findUser ( SDValue  Value, unsigned  Opcode  )

static

Helper function for LowerBRCOND.

Definition at line 5025 of file SIISelLowering.cpp.

References E, I, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().

fpModeMatchesGlobalFPAtomicMode()

static bool fpModeMatchesGlobalFPAtomicMode ( const AtomicRMWInst *  RMW )

static

Definition at line 12187 of file SIISelLowering.cpp.

References llvm::Function::getDenormalMode(), llvm::Type::getFltSemantics(), llvm::DenormalMode::getIEEE(), llvm::Instruction::getParent(), llvm::BasicBlock::getParent(), llvm::DenormalMode::getPreserveSign(), llvm::Type::getScalarType(), llvm::Value::getType(), and llvm::APFloatBase::IEEEsingle().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

getAlignedAGPRClassID()

static int getAlignedAGPRClassID ( unsigned  UnalignedClassID )

static

Definition at line 11805 of file SIISelLowering.cpp.

Referenced by llvm::SITargetLowering::finalizeLowering().

getBasePtrIndex()

static unsigned getBasePtrIndex ( const MemSDNode *  N )

static

MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.

Theoretically we would also need to check the specific intrinsic, but they all place the pointer operand first.

Definition at line 9013 of file SIISelLowering.cpp.

References llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, N, and llvm::ISD::STORE.

Referenced by llvm::GCRelocateInst::getBasePtr().

getBuildDwordsVector()

static SDValue getBuildDwordsVector ( SelectionDAG &  DAG, SDLoc  DL, ArrayRef< SDValue >  Elts  )

static

Definition at line 5912 of file SIISelLowering.cpp.

References assert(), DL, llvm::ArrayRef< T >::empty(), llvm::MVT::f32, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), i, llvm::ArrayRef< T >::size(), and llvm::MVT::v16f32.

getConstantPermuteMask()

static uint32_t getConstantPermuteMask ( uint32_t  C )

static

Definition at line 9101 of file SIISelLowering.cpp.

Referenced by getPermuteMask().

getFPBinOp()

static SDValue getFPBinOp ( SelectionDAG &  DAG, unsigned  Opcode, const SDLoc &  SL, EVT  VT, SDValue  A, SDValue  B, SDValue  GlueChain, SDNodeFlags  Flags  )

static

Definition at line 8450 of file SIISelLowering.cpp.

References assert(), B, llvm::ISD::FMUL, llvm::AMDGPUISD::FMUL_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm_unreachable, and llvm::MVT::Other.

getFPTernOp()

static SDValue getFPTernOp ( SelectionDAG &  DAG, unsigned  Opcode, const SDLoc &  SL, EVT  VT, SDValue  A, SDValue  B, SDValue  C, SDValue  GlueChain, SDNodeFlags  Flags  )

static

Definition at line 8472 of file SIISelLowering.cpp.

References assert(), B, llvm::ISD::FMA, llvm::AMDGPUISD::FMA_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm_unreachable, and llvm::MVT::Other.

getIdxEn()

static unsigned getIdxEn ( SDValue  VIndex )

static

Definition at line 6906 of file SIISelLowering.cpp.

getIndirectSGPRIdx()

static Register getIndirectSGPRIdx ( const SIInstrInfo *  TII, MachineRegisterInfo &  MRI, MachineInstr &  MI, int  Offset  )

static

Definition at line 3758 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, llvm::MachineOperand::getReg(), I, MBB, MI, MRI, Offset, and TII.

Referenced by emitIndirectDst(), and emitIndirectSrc().

getLoadExtOrTrunc()

static SDValue getLoadExtOrTrunc ( SelectionDAG &  DAG, ISD::LoadExtType  ExtType, SDValue  Op, const SDLoc &  SL, EVT  VT  )

static

Definition at line 8072 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::EVT::bitsLT(), llvm::ISD::EXTLOAD, llvm::SelectionDAG::getNode(), llvm_unreachable, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

getMad64_32()

static SDValue getMad64_32 ( SelectionDAG &  DAG, const SDLoc &  SL, EVT  VT, SDValue  N0, SDValue  N1, SDValue  N2, bool  Signed  )

static

Definition at line 10421 of file SIISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::MVT::i1, llvm::MVT::i64, llvm::AMDGPUISD::MAD_I64_I32, llvm::AMDGPUISD::MAD_U64_U32, Signed, and llvm::ISD::TRUNCATE.

getPermuteMask()

static uint32_t getPermuteMask ( SelectionDAG &  DAG, SDValue  V  )

static

Definition at line 9121 of file SIISelLowering.cpp.

References llvm::ISD::AND, assert(), getConstantPermuteMask(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::OR, llvm::ISD::SHL, and llvm::ISD::SRL.

getSPDenormModeValue()

static SDValue getSPDenormModeValue ( int  SPDenormMode, SelectionDAG &  DAG, const SDLoc &  SL, const GCNSubtarget *  ST  )

static

Definition at line 8550 of file SIISelLowering.cpp.

References assert(), FP_DENORM_FLUSH_IN_FLUSH_OUT, FP_DENORM_FLUSH_NONE, llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getTargetConstant(), hasFP64FP16Denormals(), llvm::MVT::i32, Mode, and llvm::ARM_MB::ST.

getSplatConstantFP()

static ConstantFPSDNode* getSplatConstantFP ( SDValue  Op )

static

Definition at line 9970 of file SIISelLowering.cpp.

hasCFUser()

static bool hasCFUser ( const Value *  V, SmallPtrSet< const Value *, 16 > &  Visited, unsigned  WaveSize  )

static

Definition at line 12305 of file SIISelLowering.cpp.

References llvm::Value::getType(), llvm::SmallPtrSetImpl< PtrType >::insert(), IT, and llvm::Value::users().

Referenced by llvm::SITargetLowering::requiresUniformRegister().

hasFP32Denormals()

static bool hasFP32Denormals ( const MachineFunction &  MF )

static

Definition at line 57 of file SIISelLowering.cpp.

References llvm::MachineFunction::getInfo(), and Info.

Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd(), and llvm::SITargetLowering::isFPExtFoldable().

hasFP64FP16Denormals()

static bool hasFP64FP16Denormals ( const MachineFunction &  MF )

static

Definition at line 62 of file SIISelLowering.cpp.

References llvm::MachineFunction::getInfo(), and Info.

Referenced by llvm::SITargetLowering::denormalsEnabledForType(), getSPDenormModeValue(), llvm::SITargetLowering::isFMADLegal(), and llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd().

isBoolSGPR()

static bool isBoolSGPR ( SDValue  V )

static

Definition at line 9082 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::AMDGPUISD::FP_CLASS, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::MVT::i1, llvm::ISD::OR, llvm::ISD::SETCC, and llvm::ISD::XOR.

isClampZeroToOne()

static bool isClampZeroToOne ( SDValue  A, SDValue  B  )

static

Definition at line 10104 of file SIISelLowering.cpp.

References B.

isCopyFromRegOfInlineAsm()

static LLVM_ATTRIBUTE_UNUSED bool isCopyFromRegOfInlineAsm ( const SDNode *  N )

static

Definition at line 12063 of file SIISelLowering.cpp.

References assert(), llvm::ISD::CopyFromReg, llvm::ISD::INLINEASM, llvm::ISD::INLINEASM_BR, and N.

Referenced by llvm::SITargetLowering::isSDNodeSourceOfDivergence().

isFrameIndexOp()

static bool isFrameIndexOp ( SDValue  Op )

static

Definition at line 11203 of file SIISelLowering.cpp.

References llvm::ISD::AssertZext.

isImmConstraint()

static bool isImmConstraint ( StringRef  Constraint )

static

Definition at line 11667 of file SIISelLowering.cpp.

References llvm::StringRef::size().

Referenced by llvm::SITargetLowering::getConstraintType(), and llvm::SITargetLowering::LowerAsmOperandForConstraint().

knownBitsForWorkitemID()

static void knownBitsForWorkitemID ( const GCNSubtarget &  ST, GISelKnownBits &  KB, KnownBits &  Known, unsigned  Dim  )

static

Definition at line 11921 of file SIISelLowering.cpp.

References llvm::countLeadingZeros(), llvm::MachineFunction::getFunction(), llvm::GISelKnownBits::getMachineFunction(), llvm::APInt::setHighBits(), llvm::ARM_MB::ST, and llvm::KnownBits::Zero.

Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().

loadM0FromVGPR()

static MachineBasicBlock::iterator loadM0FromVGPR ( const SIInstrInfo *  TII, MachineBasicBlock &  MBB, MachineInstr &  MI, unsigned  InitResultReg, unsigned  PhiReg, int  Offset, bool  UseGPRIdxMode, Register &  SGPRIdxReg  )

static

Definition at line 3675 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, emitLoadM0FromVGPRLoop(), First, llvm::MachineBasicBlock::getParent(), llvm::TargetRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, MRI, Offset, llvm::MachineBasicBlock::removeSuccessor(), splitBlockForLoop(), llvm::ARM_MB::ST, TII, and TRI.

Referenced by emitIndirectDst(), and emitIndirectSrc().

lowerBALLOTIntrinsic()

static SDValue lowerBALLOTIntrinsic ( const SITargetLowering &  TLI, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 4862 of file SIISelLowering.cpp.

References Arg, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::i32, N, llvm::AMDGPUISD::SETCC, llvm::ISD::SETCC, and llvm::ISD::SETNE.

lowerFCMPIntrinsic()

static SDValue lowerFCMPIntrinsic ( const SITargetLowering &  TLI, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 4832 of file SIISelLowering.cpp.

References llvm::EVT::bitsEq(), llvm::MVT::f16, llvm::MVT::f32, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getFCmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isFPPredicate(), llvm::TargetLoweringBase::isTypeLegal(), N, llvm::AMDGPUISD::SETCC, and llvm::AMDGPU::HSAMD::Kernel::CodeProps::Key::WavefrontSize.

lowerICMPIntrinsic()

static SDValue lowerICMPIntrinsic ( const SITargetLowering &  TLI, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 4797 of file SIISelLowering.cpp.

References llvm::EVT::bitsEq(), DL, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getICmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::i16, llvm::MVT::i32, llvm::CmpInst::isIntPredicate(), llvm::CmpInst::isSigned(), llvm::TargetLoweringBase::isTypeLegal(), N, llvm::AMDGPUISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::AMDGPU::HSAMD::Kernel::CodeProps::Key::WavefrontSize, and llvm::ISD::ZERO_EXTEND.

mayTailCallThisCC()

static bool mayTailCallThisCC ( CallingConv::ID  CC )

static

Return true if we might ever do TCO for calls with this calling convention.

Definition at line 2923 of file SIISelLowering.cpp.

References llvm::CallingConv::AMDGPU_Gfx, llvm::CallingConv::C, and canGuaranteeTCO().

Referenced by llvm::SITargetLowering::isEligibleForTailCallOptimization().

memVTFromImageData()

static EVT memVTFromImageData ( Type *  Ty, unsigned  DMaskLanes  )

static

Definition at line 1033 of file SIISelLowering.cpp.

References assert(), llvm::Type::getContext(), llvm::EVT::getEVT(), llvm::EVT::getVectorVT(), and llvm::min().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic(), and memVTFromImageReturn().

memVTFromImageReturn()

static EVT memVTFromImageReturn ( Type *  Ty, unsigned  DMaskLanes  )

static

Definition at line 1047 of file SIISelLowering.cpp.

References memVTFromImageData(), and llvm::ARM_MB::ST.

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

minMaxOpcToMin3Max3Opc()

static unsigned minMaxOpcToMin3Max3Opc ( unsigned  Opc )

static

Definition at line 9907 of file SIISelLowering.cpp.

References llvm::AMDGPUISD::FMAX3, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::AMDGPUISD::FMIN3, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm_unreachable, llvm::ISD::SMAX, llvm::AMDGPUISD::SMAX3, llvm::ISD::SMIN, llvm::AMDGPUISD::SMIN3, llvm::ISD::UMAX, llvm::AMDGPUISD::UMAX3, llvm::ISD::UMIN, and llvm::AMDGPUISD::UMIN3.

packImage16bitOpsToDwords()

static void packImage16bitOpsToDwords ( SelectionDAG &  DAG, SDValue  Op, MVT  PackVectorVT, SmallVectorImpl< SDValue > &  PackedAddrs, unsigned  DimIdx, unsigned  EndIdx, unsigned  NumGradients  )

static

Definition at line 6053 of file SIISelLowering.cpp.

References Addr, llvm::ISD::ANY_EXTEND, DL, llvm::MVT::f32, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), I, llvm::MVT::i16, and llvm::MVT::i32.

padEltsToUndef()

static SDValue padEltsToUndef ( SelectionDAG &  DAG, const SDLoc &  DL, EVT  CastVT, SDValue  Src, int  ExtraElts  )

static

Definition at line 5941 of file SIISelLowering.cpp.

References DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isVector(), and llvm::RegState::Undef.

Referenced by constructRetValue().

parseTexFail()

static bool parseTexFail ( SDValue  TexFailCtrl, SelectionDAG &  DAG, SDValue *  TFE, SDValue *  LWE, bool &  IsTexFail  )

static

Definition at line 6035 of file SIISelLowering.cpp.

References DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::i32, and x2().

processPSInputArgs()

static void processPSInputArgs ( SmallVectorImpl< ISD::InputArg > &  Splits, CallingConv::ID  CallConv, ArrayRef< ISD::InputArg >  Ins, BitVector &  Skipped, FunctionType *  FType, SIMachineFunctionInfo *  Info  )

static

Definition at line 1838 of file SIISelLowering.cpp.

References llvm::CallingConv::AMDGPU_PS, Arg, assert(), E, I, Info, llvm::MipsISD::Ins, and llvm::BitVector::set().

Referenced by llvm::SITargetLowering::LowerFormalArguments().

reservePrivateMemoryRegs()

static void reservePrivateMemoryRegs ( const TargetMachine &  TM, MachineFunction &  MF, const SIRegisterInfo &  TRI, SIMachineFunctionInfo &  Info  )

static

Definition at line 2187 of file SIISelLowering.cpp.

References assert(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineFrameInfo::hasCalls(), llvm::MachineFrameInfo::hasStackObjects(), Info, llvm::MachineRegisterInfo::isLiveIn(), llvm::AMDGPU::isShader(), MRI, llvm::CodeGenOpt::None, llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER, Reg, llvm::report_fatal_error(), llvm::ARM_MB::ST, TM, and TRI.

Referenced by llvm::SITargetLowering::finalizeLowering().

setM0ToIndexFromSGPR()

static void setM0ToIndexFromSGPR ( const SIInstrInfo *  TII, MachineRegisterInfo &  MRI, MachineInstr &  MI, int  Offset  )

static

Definition at line 3738 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), assert(), llvm::BuildMI(), DL, llvm::MachineOperand::getReg(), I, llvm::M0(), MBB, MI, Offset, and TII.

Referenced by emitIndirectDst(), and emitIndirectSrc().

shouldExpandVectorDynExt()

static bool shouldExpandVectorDynExt ( SDNode *  N )

static

Definition at line 10199 of file SIISelLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::isDivergent(), N, and llvm::SITargetLowering::shouldExpandVectorDynExt().

splitBlockForLoop()

static std::pair<MachineBasicBlock *, MachineBasicBlock *> splitBlockForLoop ( MachineInstr &  MI, MachineBasicBlock &  MBB, bool  InstInLoop  )

static

Returns

{ LoopBody, Remainder }

Definition at line 3481 of file SIISelLowering.cpp.

References llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getParent(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

Referenced by llvm::SITargetLowering::emitGWSMemViolTestLoop(), and loadM0FromVGPR().

STATISTIC()

STATISTIC	(	NumTailCalls	,
		"Number of tail calls"
	)

SubIdx2Lane()

static unsigned SubIdx2Lane ( unsigned  Idx )

static

Helper function for adjustWritemask.

Definition at line 11028 of file SIISelLowering.cpp.

updateBufferMMO()

static void updateBufferMMO ( MachineMemOperand *  MMO, SDValue  VOffset, SDValue  SOffset, SDValue  Offset, SDValue  VIndex = SDValue()  )

static

Update MMO based on the offset inputs to an intrinsic.

Definition at line 6854 of file SIISelLowering.cpp.

References isZero(), Offset, llvm::MachineMemOperand::setOffset(), and llvm::MachineMemOperand::setValue().

vectorEltWillFoldAway()

static bool vectorEltWillFoldAway ( SDValue  Op )

static

Definition at line 9815 of file SIISelLowering.cpp.

Variable Documentation

DisableLoopAlignment

cl::opt<bool> DisableLoopAlignment("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false))

static

Referenced by llvm::SITargetLowering::getPrefLoopAlignment().

UseDivergentRegisterIndexing

cl::opt<bool> UseDivergentRegisterIndexing("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false))

static

Referenced by llvm::SITargetLowering::shouldExpandVectorDynExt().

VGPRReserveforSGPRSpill

cl::opt<bool> VGPRReserveforSGPRSpill("amdgpu-reserve-vgpr-for-sgpr-spill", cl::desc("Allocates one VGPR for future SGPR Spill"), cl::init(true))

static

Referenced by llvm::SITargetLowering::finalizeLowering().