26#include "llvm/IR/IntrinsicsAMDGPU.h"
33#define DEBUG_TYPE "AMDGPUtti"
36 "amdgpu-unroll-threshold-private",
37 cl::desc(
"Unroll threshold for AMDGPU if private memory used in a loop"),
41 "amdgpu-unroll-threshold-local",
42 cl::desc(
"Unroll threshold for AMDGPU if local memory used in a loop"),
46 "amdgpu-unroll-threshold-if",
47 cl::desc(
"Unroll threshold increment for AMDGPU for each if statement inside loop"),
51 "amdgpu-unroll-runtime-local",
52 cl::desc(
"Allow runtime unroll for AMDGPU if local memory used in a loop"),
56 "amdgpu-unroll-max-block-to-analyze",
57 cl::desc(
"Inner loop block size threshold to analyze in unroll for AMDGPU"),
62 cl::desc(
"Cost of alloca argument"));
70 cl::desc(
"Maximum alloca size to use for inline cost"));
75 cl::desc(
"Maximum number of BBs allowed in a function after inlining"
76 " (compile time constraint)"));
84 for (
const Value *V :
I->operand_values()) {
87 if (
const PHINode *
PHI = dyn_cast<PHINode>(V)) {
89 return SubLoop->contains(PHI); }))
99 TargetTriple(
TM->getTargetTriple()),
101 TLI(ST->getTargetLowering()) {}
106 const Function &
F = *L->getHeader()->getParent();
108 F.getFnAttributeAsParsedInteger(
"amdgpu-unroll-threshold", 300);
109 UP.
MaxCount = std::numeric_limits<unsigned>::max();
122 const unsigned MaxAlloca = (256 - 16) * 4;
128 if (
MDNode *LoopUnrollThreshold =
130 if (LoopUnrollThreshold->getNumOperands() == 2) {
131 ConstantInt *MetaThresholdValue = mdconst::extract_or_null<ConstantInt>(
132 LoopUnrollThreshold->getOperand(1));
133 if (MetaThresholdValue) {
139 ThresholdPrivate = std::min(ThresholdPrivate, UP.
Threshold);
140 ThresholdLocal = std::min(ThresholdLocal, UP.
Threshold);
145 unsigned MaxBoost = std::max(ThresholdPrivate, ThresholdLocal);
147 const DataLayout &
DL = BB->getModule()->getDataLayout();
148 unsigned LocalGEPsSeen = 0;
151 return SubLoop->contains(BB); }))
160 if (
const BranchInst *Br = dyn_cast<BranchInst>(&
I)) {
161 if (UP.
Threshold < MaxBoost && Br->isConditional()) {
164 if ((L->contains(Succ0) && L->isLoopExiting(Succ0)) ||
165 (L->contains(Succ1) && L->isLoopExiting(Succ1)))
171 << *L <<
" due to " << *Br <<
'\n');
183 unsigned AS =
GEP->getAddressSpace();
184 unsigned Threshold = 0;
186 Threshold = ThresholdPrivate;
188 Threshold = ThresholdLocal;
203 if (AllocaSize > MaxAlloca)
212 if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 ||
213 (!isa<GlobalVariable>(
GEP->getPointerOperand()) &&
214 !isa<Argument>(
GEP->getPointerOperand())))
217 << *L <<
" due to LDS use.\n");
222 bool HasLoopDef =
false;
225 if (!Inst || L->isLoopInvariant(
Op))
229 return SubLoop->contains(Inst); }))
253 << *L <<
" due to " << *
GEP <<
'\n');
276 AMDGPU::FeatureEnableLoadStoreOpt, AMDGPU::FeatureEnableSIScheduler,
277 AMDGPU::FeatureEnableUnsafeDSOffsetFolding, AMDGPU::FeatureFlatForGlobal,
278 AMDGPU::FeaturePromoteAlloca, AMDGPU::FeatureUnalignedScratchAccess,
279 AMDGPU::FeatureUnalignedAccessMode,
281 AMDGPU::FeatureAutoWaitcntBeforeBarrier,
284 AMDGPU::FeatureSGPRInitBug, AMDGPU::FeatureXNACK,
285 AMDGPU::FeatureTrapHandler,
289 AMDGPU::FeatureSRAMECC,
292 AMDGPU::FeatureFastFMAF32, AMDGPU::HalfRate64Ops};
297 TLI(ST->getTargetLowering()), CommonTTI(
TM,
F),
298 IsGraphics(AMDGPU::isGraphics(
F.getCallingConv())) {
301 HasFP64FP16Denormals =
338 if (Opcode == Instruction::Load || Opcode == Instruction::Store)
339 return 32 * 4 / ElemWidth;
346 unsigned ChainSizeInBytes,
348 unsigned VecRegBitWidth = VF * LoadSize;
351 return 128 / LoadSize;
357 unsigned ChainSizeInBytes,
359 unsigned VecRegBitWidth = VF * StoreSize;
360 if (VecRegBitWidth > 128)
361 return 128 / StoreSize;
384 unsigned AddrSpace)
const {
397 unsigned AddrSpace)
const {
403 unsigned AddrSpace)
const {
420 unsigned DestAddrSpace,
unsigned SrcAlign,
unsigned DestAlign,
421 std::optional<uint32_t> AtomicElementSize)
const {
423 if (AtomicElementSize)
426 unsigned MinAlign = std::min(SrcAlign, DestAlign);
451 unsigned RemainingBytes,
unsigned SrcAddrSpace,
unsigned DestAddrSpace,
452 unsigned SrcAlign,
unsigned DestAlign,
453 std::optional<uint32_t> AtomicCpySize)
const {
454 assert(RemainingBytes < 16);
458 OpsOut,
Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
459 DestAlign, AtomicCpySize);
461 unsigned MinAlign = std::min(SrcAlign, DestAlign);
465 while (RemainingBytes >= 8) {
471 while (RemainingBytes >= 4) {
478 while (RemainingBytes >= 2) {
484 while (RemainingBytes) {
502 case Intrinsic::amdgcn_ds_ordered_add:
503 case Intrinsic::amdgcn_ds_ordered_swap:
504 case Intrinsic::amdgcn_ds_fadd:
505 case Intrinsic::amdgcn_ds_fmin:
506 case Intrinsic::amdgcn_ds_fmax: {
507 auto *Ordering = dyn_cast<ConstantInt>(Inst->
getArgOperand(2));
508 auto *Volatile = dyn_cast<ConstantInt>(Inst->
getArgOperand(4));
509 if (!Ordering || !Volatile)
512 unsigned OrderingVal = Ordering->getZExtValue();
519 Info.WriteMem =
true;
520 Info.IsVolatile = !Volatile->isZero();
535 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
540 unsigned NElts = LT.second.isVector() ?
541 LT.second.getVectorNumElements() : 1;
550 return get64BitInstrCost(
CostKind) * LT.first * NElts;
553 NElts = (NElts + 1) / 2;
556 return getFullRateInstrCost() * LT.first * NElts;
562 if (SLT == MVT::i64) {
564 return 2 * getFullRateInstrCost() * LT.first * NElts;
568 NElts = (NElts + 1) / 2;
570 return LT.first * NElts * getFullRateInstrCost();
572 const int QuarterRateCost = getQuarterRateInstrCost(
CostKind);
573 if (SLT == MVT::i64) {
574 const int FullRateCost = getFullRateInstrCost();
575 return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts;
579 NElts = (NElts + 1) / 2;
582 return QuarterRateCost * NElts * LT.first;
589 if (
const auto *
FAdd = dyn_cast<BinaryOperator>(*CxtI->
user_begin())) {
594 if (ST->
has16BitInsts() && SLT == MVT::f16 && !HasFP64FP16Denormals)
609 NElts = (NElts + 1) / 2;
611 return LT.first * NElts * get64BitInstrCost(
CostKind);
614 NElts = (NElts + 1) / 2;
616 if (SLT == MVT::f32 || SLT == MVT::f16)
617 return LT.first * NElts * getFullRateInstrCost();
623 if (SLT == MVT::f64) {
629 Cost += 3 * getFullRateInstrCost();
631 return LT.first *
Cost * NElts;
636 if ((SLT == MVT::f32 && !HasFP32Denormals) ||
638 return LT.first * getQuarterRateInstrCost(
CostKind) * NElts;
649 4 * getFullRateInstrCost() + 2 * getQuarterRateInstrCost(
CostKind);
650 return LT.first *
Cost * NElts;
653 if (SLT == MVT::f32 || SLT == MVT::f16) {
655 int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() +
656 1 * getQuarterRateInstrCost(
CostKind);
658 if (!HasFP32Denormals) {
660 Cost += 2 * getFullRateInstrCost();
663 return LT.first * NElts *
Cost;
684 case Intrinsic::round:
685 case Intrinsic::uadd_sat:
686 case Intrinsic::usub_sat:
687 case Intrinsic::sadd_sat:
688 case Intrinsic::ssub_sat:
698 if (ICA.
getID() == Intrinsic::fabs)
707 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(
RetTy);
709 unsigned NElts = LT.second.isVector() ?
710 LT.second.getVectorNumElements() : 1;
715 return LT.first * NElts * get64BitInstrCost(
CostKind);
719 NElts = (NElts + 1) / 2;
722 unsigned InstRate = getQuarterRateInstrCost(
CostKind);
724 switch (ICA.
getID()) {
727 : getQuarterRateInstrCost(
CostKind);
729 case Intrinsic::uadd_sat:
730 case Intrinsic::usub_sat:
731 case Intrinsic::sadd_sat:
732 case Intrinsic::ssub_sat:
733 static const auto ValidSatTys = {MVT::v2i16, MVT::v4i16};
734 if (
any_of(ValidSatTys, [<](
MVT M) {
return M == LT.second; }))
739 return LT.first * NElts * InstRate;
745 assert((
I ==
nullptr ||
I->getOpcode() == Opcode) &&
746 "Opcode should reflect passed instruction.");
749 const int CBrCost = SCost ? 5 : 7;
751 case Instruction::Br: {
753 auto BI = dyn_cast_or_null<BranchInst>(
I);
754 if (BI && BI->isUnconditional())
755 return SCost ? 1 : 4;
760 case Instruction::Switch: {
761 auto SI = dyn_cast_or_null<SwitchInst>(
I);
764 return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1);
766 case Instruction::Ret:
767 return SCost ? 1 : 10;
774 std::optional<FastMathFlags> FMF,
786 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
787 return LT.first * getFullRateInstrCost();
801 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
802 return LT.first * getHalfRateInstrCost(
CostKind);
810 case Instruction::ExtractElement:
811 case Instruction::InsertElement: {
826 return Index == ~0u ? 2 : 0;
840 if (Indices.
size() > 1)
848 const int TargetOutputIdx = Indices.
empty() ? -1 : Indices[0];
851 for (
auto &TC : TargetConstraints) {
856 if (TargetOutputIdx != -1 && TargetOutputIdx != OutputIdx++)
862 TRI, TC.ConstraintCode, TC.ConstraintVT).second;
866 if (!RC || !
TRI->isSGPRClass(RC))
876 cast<MetadataAsValue>(ReadReg->
getArgOperand(0))->getMetadata();
878 cast<MDString>(cast<MDNode>(MD)->getOperand(0))->getString();
897 if (
const Argument *
A = dyn_cast<Argument>(V))
906 if (
const LoadInst *Load = dyn_cast<LoadInst>(V))
914 if (isa<AtomicRMWInst>(V) || isa<AtomicCmpXchgInst>(V))
917 if (
const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) {
918 if (Intrinsic->getIntrinsicID() == Intrinsic::read_register)
925 if (
const CallInst *CI = dyn_cast<CallInst>(V)) {
926 if (CI->isInlineAsm())
932 if (isa<InvokeInst>(V))
939 if (
const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V))
942 if (
const CallInst *CI = dyn_cast<CallInst>(V)) {
943 if (CI->isInlineAsm())
961 if (
match(V,
m_LShr(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
963 match(V,
m_AShr(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
965 const Function *
F = cast<Instruction>(V)->getFunction();
971 if (
match(V,
m_c_And(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
973 const Function *
F = cast<Instruction>(V)->getFunction();
988 if (
const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(CI)) {
989 switch (Intrinsic->getIntrinsicID()) {
992 case Intrinsic::amdgcn_if:
993 case Intrinsic::amdgcn_else: {
995 return Indices.
size() == 1 && Indices[0] == 1;
1012 case Intrinsic::amdgcn_ds_fadd:
1013 case Intrinsic::amdgcn_ds_fmin:
1014 case Intrinsic::amdgcn_ds_fmax:
1015 case Intrinsic::amdgcn_is_shared:
1016 case Intrinsic::amdgcn_is_private:
1017 case Intrinsic::amdgcn_flat_atomic_fadd:
1018 case Intrinsic::amdgcn_flat_atomic_fmax:
1019 case Intrinsic::amdgcn_flat_atomic_fmin:
1029 Value *NewV)
const {
1032 case Intrinsic::amdgcn_ds_fadd:
1033 case Intrinsic::amdgcn_ds_fmin:
1034 case Intrinsic::amdgcn_ds_fmax: {
1036 if (!IsVolatile->isZero())
1047 case Intrinsic::amdgcn_is_shared:
1048 case Intrinsic::amdgcn_is_private: {
1049 unsigned TrueAS = IntrID == Intrinsic::amdgcn_is_shared ?
1057 case Intrinsic::ptrmask: {
1063 bool DoTruncate =
false;
1067 if (!
TM.isNoopAddrSpaceCast(OldAS, NewAS)) {
1085 MaskTy =
B.getInt32Ty();
1086 MaskOp =
B.CreateTrunc(MaskOp, MaskTy);
1089 return B.CreateIntrinsic(Intrinsic::ptrmask, {NewV->
getType(), MaskTy},
1092 case Intrinsic::amdgcn_flat_atomic_fadd:
1093 case Intrinsic::amdgcn_flat_atomic_fmax:
1094 case Intrinsic::amdgcn_flat_atomic_fmin: {
1102 {DestTy, SrcTy, DestTy});
1120 if (cast<FixedVectorType>(VT)->getNumElements() == 2 &&
1143 =
static_cast<const GCNSubtarget *
>(
TM.getSubtargetImpl(*Caller));
1145 =
static_cast<const GCNSubtarget *
>(
TM.getSubtargetImpl(*Callee));
1147 const FeatureBitset &CallerBits = CallerST->getFeatureBits();
1148 const FeatureBitset &CalleeBits = CalleeST->getFeatureBits();
1150 FeatureBitset RealCallerBits = CallerBits & ~InlineFeatureIgnoreList;
1151 FeatureBitset RealCalleeBits = CalleeBits & ~InlineFeatureIgnoreList;
1152 if ((RealCallerBits & RealCalleeBits) != RealCalleeBits)
1162 if (Callee->hasFnAttribute(Attribute::AlwaysInline) ||
1163 Callee->hasFnAttribute(Attribute::InlineHint))
1169 if (Callee->size() == 1)
1171 size_t BBSize = Caller->size() + Callee->size() - 1;
1181 const int NrOfSGPRUntilSpill = 26;
1182 const int NrOfVGPRUntilSpill = 32;
1186 unsigned adjustThreshold = 0;
1192 for (
auto ArgVT : ValueVTs) {
1196 SGPRsInUse += CCRegNum;
1198 VGPRsInUse += CCRegNum;
1208 ArgStackCost +=
const_cast<GCNTTIImpl *
>(TTIImpl)->getMemoryOpCost(
1211 ArgStackCost +=
const_cast<GCNTTIImpl *
>(TTIImpl)->getMemoryOpCost(
1217 adjustThreshold += std::max(0, SGPRsInUse - NrOfSGPRUntilSpill) *
1219 adjustThreshold += std::max(0, VGPRsInUse - NrOfVGPRUntilSpill) *
1221 return adjustThreshold;
1230 unsigned AllocaSize = 0;
1233 PointerType *Ty = dyn_cast<PointerType>(PtrArg->getType());
1237 unsigned AddrSpace = Ty->getAddressSpace();
1285 static_assert(InlinerVectorBonusPercent == 0,
"vector bonus assumed to be 0");
1289 return BB.getTerminator()->getNumSuccessors() > 1;
1292 Threshold += Threshold / 2;
1298 unsigned AllocaThresholdBonus = (Threshold * ArgAllocaSize) / AllocaSize;
1300 return AllocaThresholdBonus;
1316 ? getFullRateInstrCost()
1317 : ST->hasHalfRate64Ops() ? getHalfRateInstrCost(
CostKind)
1318 : getQuarterRateInstrCost(
CostKind);
1321std::pair<InstructionCost, MVT>
1322GCNTTIImpl::getTypeLegalizationCost(
Type *Ty)
const {
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Provides AMDGPU specific target descriptions.
The AMDGPU TargetMachine interface definition for hw codegen targets.
static const Function * getParent(const Value *V)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static cl::opt< TargetTransformInfo::TargetCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(TargetTransformInfo::TCK_RecipThroughput), cl::values(clEnumValN(TargetTransformInfo::TCK_RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(TargetTransformInfo::TCK_Latency, "latency", "Instruction latency"), clEnumValN(TargetTransformInfo::TCK_CodeSize, "code-size", "Code size"), clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency")))
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool hasMadMacF32Insts() const
unsigned getMaxWorkitemID(const Function &Kernel, unsigned Dimension) const
Return the maximum workitem ID value in the function, for the given (0, 1, 2) dimension.
unsigned getWavefrontSizeLog2() const
bool has16BitInsts() const
bool hasFastFMAF32() const
bool isSingleLaneExecution(const Function &Kernel) const
Return true if only a single workitem can be active in a wave.
bool hasVOP3PInsts() const
void getPeelingPreferences(Loop *L, ScalarEvolution &SE, TTI::PeelingPreferences &PP)
AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP, OptimizationRemarkEmitter *ORE)
int64_t getMaxMemIntrinsicInlineSizeThreshold() const
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
an instruction to allocate memory on the stack
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind)
Get intrinsic cost based on arguments.
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, Value *Op0, Value *Op1)
InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Opd1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Opd2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args=ArrayRef< const Value * >(), const Instruction *CxtI=nullptr)
TTI::ShuffleKind improveShuffleKindFromMask(TTI::ShuffleKind Kind, ArrayRef< int > Mask, VectorType *Ty, int &Index, VectorType *&SubTy) const
InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, TTI::TargetCostKind CostKind)
Try to calculate op costs for min/max reduction operations.
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef< int > Mask, TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, ArrayRef< const Value * > Args=std::nullopt)
void getPeelingPreferences(Loop *L, ScalarEvolution &SE, TTI::PeelingPreferences &PP)
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I=nullptr)
InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind)
std::pair< InstructionCost, MVT > getTypeLegalizationCost(Type *Ty) const
Estimate the cost of type-legalization and the legalized type.
Conditional or Unconditional Branch instruction.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned getArgOperandNo(const Use *U) const
Given a use for a arg operand, get the arg operand number that corresponds to it.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This class represents a function call, abstracting a target machine's calling convention.
This is the shared class of boolean and integer constants.
static ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getFalse(LLVMContext &Context)
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
Container class for subtarget features.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
bool hasUsableDivScaleConditionOutput() const
Condition output from div_scale is usable.
const SIRegisterInfo * getRegisterInfo() const override
bool hasUnalignedScratchAccess() const
bool hasPackedFP32Ops() const
bool hasFullRate64Ops() const
unsigned getMaxPrivateElementSize(bool ForBufferRSrc=false) const
TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind Vector) const
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind)
InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index, Value *Op0, Value *Op1)
bool isAlwaysUniform(const Value *V) const
void getPeelingPreferences(Loop *L, ScalarEvolution &SE, TTI::PeelingPreferences &PP)
GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
int64_t getMaxMemIntrinsicInlineSizeThreshold() const
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef< int > Mask, TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, ArrayRef< const Value * > Args=std::nullopt)
bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const
InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, TTI::TargetCostKind CostKind)
bool isInlineAsmSourceOfDivergence(const CallInst *CI, ArrayRef< unsigned > Indices={}) const
Analyze if the results of inline asm are divergent.
bool isReadRegisterSourceOfDivergence(const IntrinsicInst *ReadReg) const
unsigned getNumberOfRegisters(unsigned RCID) const
bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const
unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize, unsigned ChainSizeInBytes, VectorType *VecTy) const
bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const
unsigned getMaxInterleaveFactor(ElementCount VF)
unsigned getInliningThresholdMultiplier() const
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP, OptimizationRemarkEmitter *ORE)
unsigned getMinVectorRegisterBitWidth() const
bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const
unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize, unsigned ChainSizeInBytes, VectorType *VecTy) const
Value * rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV, Value *NewV) const
unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const
void getMemcpyLoopResidualLoweringType(SmallVectorImpl< Type * > &OpsOut, LLVMContext &Context, unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign, std::optional< uint32_t > AtomicCpySize) const
InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Op2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args=ArrayRef< const Value * >(), const Instruction *CxtI=nullptr)
bool collectFlatAddressOperands(SmallVectorImpl< int > &OpIndexes, Intrinsic::ID IID) const
unsigned adjustInliningThreshold(const CallBase *CB) const
bool areInlineCompatible(const Function *Caller, const Function *Callee) const
unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I=nullptr)
unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const
bool isSourceOfDivergence(const Value *V) const
bool hasBranchDivergence(const Function *F=nullptr) const
Type * getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign, std::optional< uint32_t > AtomicElementSize) const
InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Module * getParent()
Get the module that this global value is contained inside of...
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::optional< CostType > getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
const BasicBlock * getParent() const
bool hasAllowContract() const LLVM_READONLY
Determine whether the allow-contract flag is set.
Type * getReturnType() const
Intrinsic::ID getID() const
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Represents a single loop in the control flow graph.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
The main scalar evolution driver.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
int InstructionOpcodeToISD(unsigned Opcode) const
Get the ISD node that corresponds to the Instruction class opcode.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
const TargetMachine & getTargetMachine() const
std::vector< AsmOperandInfo > AsmOperandInfoVector
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
Primary interface to the complete machine description for the target machine.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt16Ty(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
Type * getElementType() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ BUFFER_FAT_POINTER
Address space for 160-bit buffer fat pointers.
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
bool isArgPassedInSGPR(const Argument *A)
bool isIntrinsicAlwaysUniform(unsigned IntrID)
bool isIntrinsicSourceOfDivergence(unsigned IntrID)
bool isExtendedGlobalAddrSpace(unsigned AS)
@ C
The default llvm calling convention, compatible with C.
@ ADD
Simple integer binary arithmetic operators.
@ FADD
Simple binary floating point operators.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SHL
Shift and rotation operations.
@ AND
Bitwise operators - logical and, logical or, logical xor.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
bool match(Val *V, const Pattern &P)
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
specific_fpval m_FPOne()
Match a float 1.0 or vector with all elements equal to 1.0.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< TypeSize > *Offsets, TypeSize StartingOffset)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
MDNode * findOptionMDForLoop(const Loop *TheLoop, StringRef Name)
Find string metadata for a loop.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
AtomicOrdering
Atomic ordering for LLVM's memory model.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static constexpr DenormalMode getPreserveSign()
uint64_t getScalarSizeInBits() const
unsigned countMinLeadingOnes() const
Returns the minimum number of leading one bits.
Information about a load/store intrinsic defined by the target.
bool isInlineCompatible(SIModeRegisterDefaults CalleeMode) const