46#include "llvm/IR/IntrinsicsNVPTX.h"
69#define DEBUG_TYPE "nvptx-lower"
81 cl::desc(
"NVPTX Specific: FMA contraction (0: don't do it"
82 " 1: do it 2: do it aggressively"),
87 cl::desc(
"NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
88 " IEEE Compliant F32 div.rnd if available."),
93 cl::desc(
"NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
97 "nvptx-force-min-byval-param-align",
cl::Hidden,
98 cl::desc(
"NVPTX Specific: force 4-byte minimal alignment for byval"
99 " params of device functions."),
181 Offsets->push_back(StartingOffset + 0);
182 Offsets->push_back(StartingOffset + 8);
189 if (
StructType *STy = dyn_cast<StructType>(Ty)) {
190 auto const *SL =
DL.getStructLayout(STy);
192 for(
auto *EI : STy->elements()) {
194 StartingOffset + SL->getElementOffset(ElementNum));
201 for (
unsigned i = 0, e = TempVTs.
size(); i != e; ++i) {
228 (NumElts % 4 == 0 || NumElts == 3)) {
231 NumElts = (NumElts + 3) / 4;
233 for (
unsigned j = 0; j != NumElts; ++j) {
241 Offsets->push_back(Off);
256 "Promotion is not suitable for scalars of size larger than 64-bits");
258 *PromotedVT = MVT::i1;
263 *PromotedVT = MVT::i8;
266 *PromotedVT = MVT::i16;
269 *PromotedVT = MVT::i32;
272 *PromotedVT = MVT::i64;
275 return EVT(*PromotedVT) != VT;
295 if (ParamAlignment < AccessSize)
298 if (Offsets[
Idx] & (AccessSize - 1))
301 EVT EltVT = ValueVTs[
Idx];
305 if (EltSize >= AccessSize)
308 unsigned NumElts = AccessSize / EltSize;
310 if (AccessSize != EltSize * NumElts)
314 if (
Idx + NumElts > ValueVTs.
size())
318 if (NumElts != 4 && NumElts != 2)
321 for (
unsigned j =
Idx + 1; j <
Idx + NumElts; ++j) {
323 if (ValueVTs[j] != EltVT)
327 if (Offsets[j] - Offsets[j - 1] != EltSize)
355 Align ParamAlignment,
bool IsVAArg =
false) {
365 for (
int I = 0, E = ValueVTs.
size();
I != E; ++
I) {
368 for (
unsigned AccessSize : {16, 8, 4, 2}) {
370 I, AccessSize, ValueVTs, Offsets, ParamAlignment);
379 assert(
I + 1 < E &&
"Not enough elements.");
385 assert(
I + 3 < E &&
"Not enough elements.");
456 Op, VT, IsOpSupported ? Action : NoBF16Action);
461 bool IsOpSupported =
false;
543 for (
MVT VT : {MVT::bf16, MVT::f16, MVT::v2bf16, MVT::v2f16, MVT::f32,
544 MVT::f64, MVT::i1, MVT::i8, MVT::i16, MVT::v2i16, MVT::v4i8,
545 MVT::i32, MVT::i64}) {
671 for (
const auto& Ty : {MVT::i16, MVT::i32, MVT::i64}) {
753 const bool IsFP16FP16x2NegAvailable = STI.
getSmVersion() >= 53 &&
756 for (
const auto &VT : {MVT::f16, MVT::v2f16})
781 for (
MVT VT : {MVT::bf16, MVT::f32, MVT::f64}) {
790 for (
MVT VT : {MVT::i1, MVT::i16, MVT::i32, MVT::i64}) {
819 for (
const auto &
Op :
843 return IsAtLeastSm80 ?
Legal : NotSm80Action;
849 setFP16OperationAction(
Op, MVT::v2f16, GetMinMaxAction(
Expand),
Expand);
856 setFP16OperationAction(
Op, MVT::f16, GetMinMaxAction(
Expand),
Expand);
859 setFP16OperationAction(
Op, MVT::v2f16, GetMinMaxAction(
Expand),
Expand);
881#define MAKE_CASE(V) \
1322 bool Reciprocal)
const {
1343 if (Reciprocal || ExtraSteps > 0) {
1345 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f
1346 : Intrinsic::nvvm_rsqrt_approx_f);
1347 else if (VT == MVT::f64)
1348 return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d);
1353 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f
1354 : Intrinsic::nvvm_sqrt_approx_f);
1362 DAG.
getConstant(Intrinsic::nvvm_rcp_approx_ftz_d,
DL, MVT::i32),
1363 MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d));
1385 std::optional<std::pair<unsigned, const APInt &>> VAInfo,
1386 const CallBase &CB,
unsigned UniqueCallSite)
const {
1390 assert(isABI &&
"Non-ABI compilation is not supported");
1394 std::string Prototype;
1396 O <<
"prototype_" << UniqueCallSite <<
" : .callprototype ";
1405 if (
auto *ITy = dyn_cast<IntegerType>(retTy)) {
1406 size = ITy->getBitWidth();
1409 "Floating point type expected here");
1417 O <<
".param .b" <<
size <<
" _";
1418 }
else if (isa<PointerType>(retTy)) {
1419 O <<
".param .b" << PtrVT.getSizeInBits() <<
" _";
1421 O <<
".param .align " << (retAlignment ? retAlignment->value() : 0)
1422 <<
" .b8 _[" <<
DL.getTypeAllocSize(retTy) <<
"]";
1432 unsigned NumArgs = VAInfo ? VAInfo->first : Args.size();
1433 for (
unsigned i = 0, OIdx = 0; i != NumArgs; ++i, ++OIdx) {
1434 Type *Ty = Args[i].Ty;
1440 if (!Outs[OIdx].Flags.isByVal()) {
1444 O <<
".param .align " << ParamAlign.
value() <<
" .b8 ";
1446 O <<
"[" <<
DL.getTypeAllocSize(Ty) <<
"]";
1450 if (
unsigned len = vtparts.
size())
1456 (
getValueType(
DL, Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
1457 "type mismatch between callee prototype and arguments");
1460 if (isa<IntegerType>(Ty)) {
1461 sz = cast<IntegerType>(Ty)->getBitWidth();
1463 }
else if (isa<PointerType>(Ty)) {
1464 sz = PtrVT.getSizeInBits();
1468 O <<
".param .b" << sz <<
" ";
1475 Type *ETy = Args[i].IndirectType;
1476 Align InitialAlign = Outs[OIdx].Flags.getNonZeroByValAlign();
1477 Align ParamByValAlign =
1480 O <<
".param .align " << ParamByValAlign.
value() <<
" .b8 ";
1482 O <<
"[" << Outs[OIdx].Flags.getByValSize() <<
"]";
1486 O << (first ?
"" :
",") <<
" .param .align " << VAInfo->second
1506 return DL.getABITypeAlign(Ty);
1511 if (!DirectCallee) {
1516 if (
const auto *CI = dyn_cast<CallInst>(CB)) {
1519 return StackAlign.value();
1530 return DL.getABITypeAlign(Ty);
1534 switch (ElementType.getSimpleVT().SimpleTy) {
1539 ElementType = MVT::i16;
1544 ElementType = MVT::i32;
1547 ElementType = MVT::i64;
1559 unsigned ArgID,
const SDLoc &dl) {
1566 for (
unsigned i = 0, n = ElementType.getSizeInBits() / 8; i < n; i++) {
1592 EVT MergedType = ElementType;
1599 for (
unsigned i = 0, n = ElementType.getSizeInBits() / 8; i < n; i++) {
1626 if (ElementType != MergedType)
1637 "Support for variadic functions (unsized array parameter) introduced "
1638 "in PTX ISA version 6.0 and requires target sm_30.");
1654 assert(isABI &&
"Non-ABI compilation is not supported");
1676 unsigned VAOffset = 0;
1683 unsigned ParamCount = 0;
1696 for (
unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {
1697 EVT VT = Outs[OIdx].VT;
1698 Type *Ty = Args[i].Ty;
1700 bool IsByVal = Outs[OIdx].Flags.isByVal();
1705 assert((!IsByVal || Args[i].IndirectType) &&
1706 "byval arg must have indirect type");
1707 Type *ETy = (IsByVal ? Args[i].IndirectType : Ty);
1715 Align InitialAlign = Outs[OIdx].Flags.getNonZeroByValAlign();
1719 VAOffset =
alignTo(VAOffset, ArgAlign);
1721 ArgAlign = getArgumentAlignment(CB, Ty, ParamCount + 1,
DL);
1725 (IsByVal ? Outs[OIdx].Flags.getByValSize() :
DL.getTypeAllocSize(Ty));
1731 if (ParamCount == FirstVAArg) {
1737 DeclareParamVTs, DeclareParamOps);
1739 NeedAlign = PassAsArray;
1740 }
else if (PassAsArray) {
1757 SDValue DeclareScalarParamOps[] = {
1762 DeclareScalarParamOps);
1771 bool ExtendIntegerParam =
1776 for (
unsigned j = 0, je = VTs.
size(); j != je; ++j) {
1778 int CurOffset = Offsets[j];
1783 SDValue StVal = OutVals[OIdx];
1787 EltVT =
EVT(PromotedVT);
1792 StVal = DAG.
getNode(Ext, dl, PromotedVT, StVal);
1801 }
else if (ExtendIntegerParam) {
1802 assert(VTs.
size() == 1 &&
"Scalar can't have multiple parts.");
1806 dl, MVT::i32, StVal);
1817 if (VectorInfo[j] ==
PVF_SCALAR && !IsVAArg && PartAlign.has_value() &&
1820 assert(StoreOperands.
empty() &&
"Unfinished preceeding store.");
1822 DAG, Chain, IsByVal ? CurOffset + VAOffset : CurOffset, EltVT,
1823 StVal, InGlue, ParamCount, dl);
1834 assert(StoreOperands.
empty() &&
"Unfinished preceding store.");
1837 DAG.
getConstant(IsVAArg ? FirstVAArg : ParamCount, dl, MVT::i32));
1840 IsByVal ? CurOffset + VAOffset : (IsVAArg ? VAOffset : CurOffset),
1848 unsigned NumElts = StoreOperands.
size() - 3;
1868 EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : EltVT;
1871 Op, dl, DAG.
getVTList(MVT::Other, MVT::Glue), StoreOperands,
1877 StoreOperands.
clear();
1881 if (!IsByVal && IsVAArg) {
1883 "Vectorization is expected to be disabled for variadics.");
1884 VAOffset +=
DL.getTypeAllocSize(
1891 assert(StoreOperands.
empty() &&
"Unfinished parameter store.");
1892 if (!IsByVal && VTs.
size() > 0)
1895 if (IsByVal && IsVAArg)
1903 if (Ins.size() > 0) {
1910 unsigned resultsz =
DL.getTypeAllocSizeInBits(
RetTy);
1921 retAlignment = getArgumentAlignment(CB,
RetTy, 0,
DL);
1922 assert(retAlignment &&
"retAlignment is guaranteed to be set");
1925 Chain, DAG.
getConstant(retAlignment->value(), dl, MVT::i32),
1943 VADeclareParam->
getVTList(), DeclareParamOps);
1951 if (isa<ExternalSymbolSDNode>(Callee)) {
1956 assert(CalleeFunc !=
nullptr &&
"Libcall callee must be set.");
1960 CalleeFunc->
addFnAttr(
"nvptx-libcall-callee",
"true");
1973 DL,
RetTy, Args, Outs, retAlignment,
1975 ? std::optional<std::pair<unsigned, const APInt &>>(std::make_pair(
1978 *CB, UniqueCallSite);
1991 Chain, DAG.
getConstant((Ins.size() == 0) ? 0 : 1, dl, MVT::i32), InGlue
1998 Chain = DAG.
getNode(Opcode, dl, PrintCallVTs, PrintCallOps);
2003 SDValue CallVoidOps[] = { Chain, Callee, InGlue };
2009 SDValue CallArgBeginOps[] = { Chain, InGlue };
2014 for (
unsigned i = 0, e = std::min(CLI.
NumFixedArgs + 1, ParamCount); i != e;
2024 Chain = DAG.
getNode(opcode, dl, CallArgVTs, CallArgOps);
2028 SDValue CallArgEndOps[] = { Chain,
2037 Chain, DAG.
getConstant(UniqueCallSite, dl, MVT::i32), InGlue};
2054 if (Ins.size() > 0) {
2058 assert(VTs.
size() == Ins.size() &&
"Bad value decomposition");
2060 Align RetAlign = getArgumentAlignment(CB,
RetTy, 0,
DL);
2069 bool ExtendIntegerRetVal =
2070 RetTy->isIntegerTy() &&
DL.getTypeAllocSizeInBits(
RetTy) < 32;
2072 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
2073 bool needTruncate =
false;
2074 EVT TheLoadType = VTs[i];
2075 EVT EltType = Ins[i].VT;
2080 TheLoadType =
EVT(PromotedVT);
2081 EltType =
EVT(PromotedVT);
2082 needTruncate =
true;
2085 if (ExtendIntegerRetVal) {
2086 TheLoadType = MVT::i32;
2088 needTruncate =
true;
2090 if (VTs[i].isInteger())
2091 needTruncate =
true;
2098 EltAlign <
DL.getABITypeAlign(
2100 assert(VecIdx == -1 && LoadVTs.
empty() &&
"Orphaned operand list.");
2102 DAG, Chain, Offsets[i], TheLoadType, InGlue, TempProxyRegOps, dl);
2104 ProxyRegTruncates.
push_back(std::optional<MVT>());
2113 assert(VecIdx == -1 && LoadVTs.
empty() &&
"Orphaned operand list.");
2120 unsigned NumElts = LoadVTs.
size();
2140 DAG.
getConstant(Offsets[VecIdx], dl, MVT::i32), InGlue};
2142 Op, dl, DAG.
getVTList(LoadVTs), LoadOperands, TheLoadType,
2146 for (
unsigned j = 0; j < NumElts; ++j) {
2150 ProxyRegTruncates.
push_back(std::optional<MVT>(Ins[VecIdx + j].VT));
2152 ProxyRegTruncates.
push_back(std::optional<MVT>());
2156 InGlue = RetVal.
getValue(NumElts + 1);
2166 DAG.
getCALLSEQ_END(Chain, UniqueCallSite, UniqueCallSite + 1, InGlue, dl);
2172 for (
unsigned i = 0; i < ProxyRegOps.
size(); ++i) {
2173 if (i < RetElts.
size() && RetElts[i]) {
2180 DAG.
getVTList(ProxyRegOps[i].getSimpleValueType(), MVT::Other, MVT::Glue),
2181 { Chain, ProxyRegOps[i], InGlue }
2184 Chain = Ret.getValue(1);
2185 InGlue = Ret.getValue(2);
2187 if (ProxyRegTruncates[i]) {
2194 for (
SDValue &
T : TempProxyRegOps) {
2197 DAG.
getVTList(
T.getSimpleValueType(), MVT::Other, MVT::Glue),
2198 {Chain, T.getOperand(0), InGlue});
2220 "Support for dynamic alloca introduced in PTX ISA version 7.3 and "
2221 "requires target sm_52.",
2231 uint64_t Align = cast<ConstantSDNode>(
Op.getOperand(2))->getZExtValue();
2245 SDValue MergeOps[] = {Alloca, Chain};
2257 unsigned NumOperands = Node->getNumOperands();
2258 for (
unsigned i = 0; i < NumOperands; ++i) {
2259 SDValue SubOp = Node->getOperand(i);
2263 for (
unsigned j = 0; j < NumSubElem; ++j) {
2277 EVT VT =
Op->getValueType(0);
2278 if (!(
Isv2x16VT(VT) || VT == MVT::v4i8))
2284 return Operand->isUndef() || isa<ConstantSDNode>(Operand) ||
2285 isa<ConstantFPSDNode>(Operand);
2289 if (VT == MVT::v4i8) {
2311 EVT VT =
Op->getValueType(0);
2313 return APInt(32, 0);
2315 if (VT == MVT::v2f16 || VT == MVT::v2bf16)
2316 Value = cast<ConstantFPSDNode>(Operand)->getValueAPF().bitcastToAPInt();
2317 else if (VT == MVT::v2i16 || VT == MVT::v4i8)
2323 if (VT == MVT::v4i8)
2325 return Value.zext(32);
2329 Value = GetOperand(
Op, 0) | GetOperand(
Op, 1).shl(16);
2330 }
else if (VT == MVT::v4i8) {
2331 Value = GetOperand(
Op, 0) | GetOperand(
Op, 1).shl(8) |
2332 GetOperand(
Op, 2).shl(16) | GetOperand(
Op, 3).shl(24);
2347 if (VectorVT == MVT::v4i8) {
2359 if (isa<ConstantSDNode>(
Index.getNode()))
2380 if (VectorVT != MVT::v4i8)
2384 if (
Value->isUndef())
2403 if (VectorVT != MVT::v4i8 ||
Op.getValueType() != MVT::v4i8)
2411 if (
I.value() != -1)
2412 Selector |= (
I.value() << (
I.index() * 4));
2430 EVT VT =
Op.getValueType();
2491 EVT VT =
Op.getValueType();
2542 EVT VT =
Op.getValueType();
2545 return LowerFROUND32(
Op, DAG);
2548 return LowerFROUND64(
Op, DAG);
2564 EVT VT =
Op.getValueType();
2570 const int SignBitMask = 0x80000000;
2573 const int PointFiveInBits = 0x3F000000;
2574 SDValue PointFiveWithSignRaw =
2605 EVT VT =
Op.getValueType();
2637 if (
Op.getValueType() == MVT::bf16) {
2641 DAG.
getNode(
Op.getOpcode(), Loc, MVT::f32,
Op.getOperand(0)),
2653 if (
Op.getOperand(0).getValueType() == MVT::bf16) {
2656 Op.getOpcode(), Loc,
Op.getValueType(),
2666 EVT NarrowVT =
Op.getValueType();
2703 EVT WideVT =
Op.getValueType();
2730 if (
Op.getValueType() != MVT::v2i16)
2732 EVT EltVT =
Op.getValueType().getVectorElementType();
2734 for (
int I = 0, E =
Op.getValueType().getVectorNumElements();
I < E;
I++) {
2737 [&](
const SDUse &O) {
2738 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
2739 O.get(), DAG.getIntPtrConstant(I, DL));
2750 switch (
Op.getOpcode()) {
2760 return LowerBUILD_VECTOR(
Op, DAG);
2764 return LowerEXTRACT_VECTOR_ELT(
Op, DAG);
2766 return LowerINSERT_VECTOR_ELT(
Op, DAG);
2768 return LowerVECTOR_SHUFFLE(
Op, DAG);
2770 return LowerCONCAT_VECTORS(
Op, DAG);
2772 return LowerSTORE(
Op, DAG);
2774 return LowerLOAD(
Op, DAG);
2776 return LowerShiftLeftParts(
Op, DAG);
2779 return LowerShiftRightParts(
Op, DAG);
2781 return LowerSelect(
Op, DAG);
2783 return LowerFROUND(
Op, DAG);
2786 return LowerINT_TO_FP(
Op, DAG);
2789 return LowerFP_TO_INT(
Op, DAG);
2791 return LowerFP_ROUND(
Op, DAG);
2793 return LowerFP_EXTEND(
Op, DAG);
2795 return LowerVAARG(
Op, DAG);
2797 return LowerVASTART(
Op, DAG);
2813 return LowerCopyToReg_128(
Op, DAG);
2826 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2827 EVT VT = Node->getValueType(0);
2829 SDValue Tmp1 = Node->getOperand(0);
2830 SDValue Tmp2 = Node->getOperand(1);
2831 const MaybeAlign MA(Node->getConstantOperandVal(3));
2869 SDValue Arg = getParamSymbol(DAG, -1, PtrVT);
2872 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
2873 return DAG.
getStore(
Op.getOperand(0),
DL, VAReg,
Op.getOperand(1),
2883 assert(
Op.getValueType() == MVT::i1 &&
"Custom lowering enabled only for i1");
2894 if (
Op.getValueType() == MVT::i1)
2895 return LowerLOADi1(
Op, DAG);
2899 EVT VT =
Op.getValueType();
2902 EVT MemVT =
Load->getMemoryVT();
2904 MemVT, *
Load->getMemOperand())) {
2924 "Custom lowering for i1 load only");
2926 LD->getBasePtr(),
LD->getPointerInfo(),
2927 MVT::i8,
LD->getAlign(),
2928 LD->getMemOperand()->getFlags());
2933 SDValue Ops[] = { result,
LD->getChain() };
2942 return LowerSTOREi1(
Op, DAG);
2946 if ((
Isv2x16VT(VT) || VT == MVT::v4i8) &&
2948 VT, *
Store->getMemOperand()))
2956 return LowerSTOREVector(
Op, DAG);
3004 if (Alignment < PrefAlign) {
3013 unsigned Opcode = 0;
3020 bool NeedExt =
false;
3024 bool StoreF16x2 =
false;
3052 for (
unsigned i = 0; i < NumElts; ++i) {
3063 for (
unsigned i = 0; i < NumElts; ++i) {
3073 Ops.
append(
N->op_begin() + 2,
N->op_end());
3100 DAG.
getTruncStore(Tmp1, dl, Tmp3, Tmp2,
ST->getPointerInfo(), MVT::i8,
3101 ST->getAlign(),
ST->getMemOperand()->getFlags());
3110 assert(
Op.getOperand(1).getValueType() == MVT::i128 &&
3111 "Custom lowering for 128-bit CopyToReg only");
3125 NewOps[0] =
Op->getOperand(0);
3126 NewOps[1] =
Op->getOperand(1);
3130 NewOps[4] =
Op->getOperand(3);
3135unsigned NVPTXTargetLowering::getNumRegisters(
3137 std::optional<MVT> RegisterVT = std::nullopt)
const {
3138 if (VT == MVT::i128 && RegisterVT == MVT::i128)
3143bool NVPTXTargetLowering::splitValueIntoRegisterParts(
3145 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
3146 if (Val.
getValueType() == MVT::i128 && NumParts == 1) {
3177 std::vector<SDValue> OutChains;
3180 assert(isABI &&
"Non-ABI compilation is not supported");
3184 std::vector<Type *> argTypes;
3185 std::vector<const Argument *> theArgs;
3187 theArgs.push_back(&
I);
3188 argTypes.push_back(
I.getType());
3199 unsigned InsIdx = 0;
3201 for (
unsigned i = 0, e = theArgs.size(); i != e; ++i, ++InsIdx) {
3202 Type *Ty = argTypes[i];
3204 if (theArgs[i]->use_empty()) {
3210 if (vtparts.
empty())
3213 for (
unsigned parti = 0, parte = vtparts.
size(); parti != parte;
3218 if (vtparts.
size() > 0)
3225 for (
unsigned parti = 0; parti < NumRegs; ++parti) {
3242 bool aggregateIsPacked =
false;
3243 if (
StructType *STy = dyn_cast<StructType>(Ty))
3244 aggregateIsPacked = STy->isPacked();
3256 SDValue Arg = getParamSymbol(DAG, i, PtrVT);
3258 for (
unsigned parti = 0, parte = VTs.
size(); parti != parte; ++parti) {
3260 assert(VecIdx == -1 &&
"Orphaned vector.");
3265 if (VectorInfo[parti] &
PVF_LAST) {
3266 unsigned NumElts = parti - VecIdx + 1;
3267 EVT EltVT = VTs[parti];
3270 if (EltVT == MVT::i1)
3272 else if (
Isv2x16VT(EltVT) || EltVT == MVT::v4i8)
3286 if (aggregateIsPacked)
3289 return std::nullopt;
3299 P.getNode()->setIROrder(i + 1);
3300 for (
unsigned j = 0; j < NumElts; ++j) {
3304 if (EltVT == MVT::i1)
3307 else if (EltVT != LoadVT)
3319 Ins[InsIdx].VT.getFixedSizeInBits() >
3323 Elt = DAG.
getNode(Extend, dl, Ins[InsIdx].VT, Elt);
3346 assert(ObjectVT == Ins[InsIdx].VT &&
3347 "Ins type did not match function type");
3348 SDValue Arg = getParamSymbol(DAG, i, PtrVT);
3351 p.getNode()->setIROrder(i + 1);
3355 if (!OutChains.empty())
3371 for (
unsigned i = 0, n = ElementType.getSizeInBits() / 8; i < n; i++) {
3381 DAG.
getVTList(MVT::Other), StoreOperands,
3399 assert(isABI &&
"Non-ABI compilation is not supported");
3408 assert(VTs.
size() == OutVals.
size() &&
"Bad return value decomposition");
3410 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
3411 SDValue PromotedOutVal = OutVals[i];
3414 VTs[i] =
EVT(PromotedVT);
3419 PromotedOutVal = DAG.
getNode(Ext, dl, PromotedVT, PromotedOutVal);
3421 PromotedOutVals.
push_back(PromotedOutVal);
3432 bool ExtendIntegerRetVal =
3433 RetTy->isIntegerTy() &&
DL.getTypeAllocSizeInBits(
RetTy) < 32;
3436 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
3438 SDValue RetVal = PromotedOutVals[i];
3440 if (ExtendIntegerRetVal) {
3443 dl, MVT::i32, RetVal);
3453 EVT ElementType = ExtendIntegerRetVal ? MVT::i32 : VTs[i];
3454 Align ElementTypeAlign =
3455 DL.getABITypeAlign(ElementType.getTypeForEVT(
RetTy->getContext()));
3456 Align ElementAlign =
3458 if (ElementAlign < ElementTypeAlign) {
3459 assert(StoreOperands.
empty() &&
"Orphaned operand list.");
3471 assert(StoreOperands.
empty() &&
"Orphaned operand list.");
3482 unsigned NumElts = StoreOperands.
size() - 2;
3499 EVT TheStoreType = ExtendIntegerRetVal ? MVT::i32 : VTs[i];
3501 Op, dl, DAG.
getVTList(MVT::Other), StoreOperands, TheStoreType,
3504 StoreOperands.
clear();
3514 if (Constraint.
size() > 1)
3520 switch (Intrinsic) {
3524 case Intrinsic::nvvm_tex_1d_v4f32_s32:
3526 case Intrinsic::nvvm_tex_1d_v4f32_f32:
3528 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
3530 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
3532 case Intrinsic::nvvm_tex_1d_v4s32_s32:
3534 case Intrinsic::nvvm_tex_1d_v4s32_f32:
3536 case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
3538 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
3540 case Intrinsic::nvvm_tex_1d_v4u32_s32:
3542 case Intrinsic::nvvm_tex_1d_v4u32_f32:
3544 case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
3546 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
3549 case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
3551 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
3553 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
3555 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
3557 case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
3559 case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
3561 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
3563 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
3565 case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
3567 case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
3569 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
3571 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
3574 case Intrinsic::nvvm_tex_2d_v4f32_s32:
3576 case Intrinsic::nvvm_tex_2d_v4f32_f32:
3578 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
3580 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
3582 case Intrinsic::nvvm_tex_2d_v4s32_s32:
3584 case Intrinsic::nvvm_tex_2d_v4s32_f32:
3586 case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
3588 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
3590 case Intrinsic::nvvm_tex_2d_v4u32_s32:
3592 case Intrinsic::nvvm_tex_2d_v4u32_f32:
3594 case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
3596 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
3599 case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
3601 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
3603 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
3605 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
3607 case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
3609 case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
3611 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
3613 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
3615 case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
3617 case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
3619 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
3621 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
3624 case Intrinsic::nvvm_tex_3d_v4f32_s32:
3626 case Intrinsic::nvvm_tex_3d_v4f32_f32:
3628 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
3630 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
3632 case Intrinsic::nvvm_tex_3d_v4s32_s32:
3634 case Intrinsic::nvvm_tex_3d_v4s32_f32:
3636 case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
3638 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
3640 case Intrinsic::nvvm_tex_3d_v4u32_s32:
3642 case Intrinsic::nvvm_tex_3d_v4u32_f32:
3644 case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
3646 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
3649 case Intrinsic::nvvm_tex_cube_v4f32_f32:
3651 case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
3653 case Intrinsic::nvvm_tex_cube_v4s32_f32:
3655 case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
3657 case Intrinsic::nvvm_tex_cube_v4u32_f32:
3659 case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
3662 case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
3664 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
3666 case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
3668 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
3670 case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
3672 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
3675 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
3677 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
3679 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
3681 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
3683 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
3685 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
3687 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
3689 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
3691 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
3693 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
3695 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
3697 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
3700 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
3702 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
3704 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
3706 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
3708 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
3710 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
3712 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
3714 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
3716 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
3718 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
3720 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
3722 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
3725 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
3727 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
3729 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
3731 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
3733 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
3735 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
3737 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
3739 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
3741 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
3743 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
3745 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
3747 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
3750 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
3752 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
3754 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
3756 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
3758 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
3760 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
3762 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
3764 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
3766 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
3768 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
3770 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
3772 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
3775 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
3777 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
3779 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
3781 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
3783 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
3785 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
3787 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
3789 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
3791 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
3793 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
3795 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
3797 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
3800 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
3802 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
3804 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
3806 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
3808 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
3810 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
3812 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
3814 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
3816 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
3818 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
3820 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
3822 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
3825 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
3827 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
3829 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
3831 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
3833 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
3835 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
3838 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
3840 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
3842 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
3844 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
3846 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
3848 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
3851 case Intrinsic::nvvm_tex_unified_cube_grad_v4f32_f32:
3853 case Intrinsic::nvvm_tex_unified_cube_grad_v4s32_f32:
3855 case Intrinsic::nvvm_tex_unified_cube_grad_v4u32_f32:
3857 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4f32_f32:
3859 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4s32_f32:
3861 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4u32_f32:
3864 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
3866 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
3868 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
3870 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
3872 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
3874 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
3876 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
3878 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
3880 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
3882 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
3884 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
3886 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
3892 switch (Intrinsic) {
3895 case Intrinsic::nvvm_suld_1d_i8_clamp:
3897 case Intrinsic::nvvm_suld_1d_i16_clamp:
3899 case Intrinsic::nvvm_suld_1d_i32_clamp:
3901 case Intrinsic::nvvm_suld_1d_i64_clamp:
3903 case Intrinsic::nvvm_suld_1d_v2i8_clamp:
3905 case Intrinsic::nvvm_suld_1d_v2i16_clamp:
3907 case Intrinsic::nvvm_suld_1d_v2i32_clamp:
3909 case Intrinsic::nvvm_suld_1d_v2i64_clamp:
3911 case Intrinsic::nvvm_suld_1d_v4i8_clamp:
3913 case Intrinsic::nvvm_suld_1d_v4i16_clamp:
3915 case Intrinsic::nvvm_suld_1d_v4i32_clamp:
3917 case Intrinsic::nvvm_suld_1d_array_i8_clamp:
3919 case Intrinsic::nvvm_suld_1d_array_i16_clamp:
3921 case Intrinsic::nvvm_suld_1d_array_i32_clamp:
3923 case Intrinsic::nvvm_suld_1d_array_i64_clamp:
3925 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
3927 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
3929 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
3931 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
3933 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
3935 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
3937 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
3939 case Intrinsic::nvvm_suld_2d_i8_clamp:
3941 case Intrinsic::nvvm_suld_2d_i16_clamp:
3943 case Intrinsic::nvvm_suld_2d_i32_clamp:
3945 case Intrinsic::nvvm_suld_2d_i64_clamp:
3947 case Intrinsic::nvvm_suld_2d_v2i8_clamp:
3949 case Intrinsic::nvvm_suld_2d_v2i16_clamp:
3951 case Intrinsic::nvvm_suld_2d_v2i32_clamp:
3953 case Intrinsic::nvvm_suld_2d_v2i64_clamp:
3955 case Intrinsic::nvvm_suld_2d_v4i8_clamp:
3957 case Intrinsic::nvvm_suld_2d_v4i16_clamp:
3959 case Intrinsic::nvvm_suld_2d_v4i32_clamp:
3961 case Intrinsic::nvvm_suld_2d_array_i8_clamp:
3963 case Intrinsic::nvvm_suld_2d_array_i16_clamp:
3965 case Intrinsic::nvvm_suld_2d_array_i32_clamp:
3967 case Intrinsic::nvvm_suld_2d_array_i64_clamp:
3969 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
3971 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
3973 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
3975 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
3977 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
3979 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
3981 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
3983 case Intrinsic::nvvm_suld_3d_i8_clamp:
3985 case Intrinsic::nvvm_suld_3d_i16_clamp:
3987 case Intrinsic::nvvm_suld_3d_i32_clamp:
3989 case Intrinsic::nvvm_suld_3d_i64_clamp:
3991 case Intrinsic::nvvm_suld_3d_v2i8_clamp:
3993 case Intrinsic::nvvm_suld_3d_v2i16_clamp:
3995 case Intrinsic::nvvm_suld_3d_v2i32_clamp:
3997 case Intrinsic::nvvm_suld_3d_v2i64_clamp:
3999 case Intrinsic::nvvm_suld_3d_v4i8_clamp:
4001 case Intrinsic::nvvm_suld_3d_v4i16_clamp:
4003 case Intrinsic::nvvm_suld_3d_v4i32_clamp:
4005 case Intrinsic::nvvm_suld_1d_i8_trap:
4007 case Intrinsic::nvvm_suld_1d_i16_trap:
4009 case Intrinsic::nvvm_suld_1d_i32_trap:
4011 case Intrinsic::nvvm_suld_1d_i64_trap:
4013 case Intrinsic::nvvm_suld_1d_v2i8_trap:
4015 case Intrinsic::nvvm_suld_1d_v2i16_trap:
4017 case Intrinsic::nvvm_suld_1d_v2i32_trap:
4019 case Intrinsic::nvvm_suld_1d_v2i64_trap:
4021 case Intrinsic::nvvm_suld_1d_v4i8_trap:
4023 case Intrinsic::nvvm_suld_1d_v4i16_trap:
4025 case Intrinsic::nvvm_suld_1d_v4i32_trap:
4027 case Intrinsic::nvvm_suld_1d_array_i8_trap:
4029 case Intrinsic::nvvm_suld_1d_array_i16_trap:
4031 case Intrinsic::nvvm_suld_1d_array_i32_trap:
4033 case Intrinsic::nvvm_suld_1d_array_i64_trap:
4035 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
4037 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
4039 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
4041 case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
4043 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
4045 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
4047 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
4049 case Intrinsic::nvvm_suld_2d_i8_trap:
4051 case Intrinsic::nvvm_suld_2d_i16_trap:
4053 case Intrinsic::nvvm_suld_2d_i32_trap:
4055 case Intrinsic::nvvm_suld_2d_i64_trap:
4057 case Intrinsic::nvvm_suld_2d_v2i8_trap:
4059 case Intrinsic::nvvm_suld_2d_v2i16_trap:
4061 case Intrinsic::nvvm_suld_2d_v2i32_trap:
4063 case Intrinsic::nvvm_suld_2d_v2i64_trap:
4065 case Intrinsic::nvvm_suld_2d_v4i8_trap:
4067 case Intrinsic::nvvm_suld_2d_v4i16_trap:
4069 case Intrinsic::nvvm_suld_2d_v4i32_trap:
4071 case Intrinsic::nvvm_suld_2d_array_i8_trap:
4073 case Intrinsic::nvvm_suld_2d_array_i16_trap:
4075 case Intrinsic::nvvm_suld_2d_array_i32_trap:
4077 case Intrinsic::nvvm_suld_2d_array_i64_trap:
4079 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
4081 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
4083 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
4085 case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
4087 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
4089 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
4091 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
4093 case Intrinsic::nvvm_suld_3d_i8_trap:
4095 case Intrinsic::nvvm_suld_3d_i16_trap:
4097 case Intrinsic::nvvm_suld_3d_i32_trap:
4099 case Intrinsic::nvvm_suld_3d_i64_trap:
4101 case Intrinsic::nvvm_suld_3d_v2i8_trap:
4103 case Intrinsic::nvvm_suld_3d_v2i16_trap:
4105 case Intrinsic::nvvm_suld_3d_v2i32_trap:
4107 case Intrinsic::nvvm_suld_3d_v2i64_trap:
4109 case Intrinsic::nvvm_suld_3d_v4i8_trap:
4111 case Intrinsic::nvvm_suld_3d_v4i16_trap:
4113 case Intrinsic::nvvm_suld_3d_v4i32_trap:
4115 case Intrinsic::nvvm_suld_1d_i8_zero:
4117 case Intrinsic::nvvm_suld_1d_i16_zero:
4119 case Intrinsic::nvvm_suld_1d_i32_zero:
4121 case Intrinsic::nvvm_suld_1d_i64_zero:
4123 case Intrinsic::nvvm_suld_1d_v2i8_zero:
4125 case Intrinsic::nvvm_suld_1d_v2i16_zero:
4127 case Intrinsic::nvvm_suld_1d_v2i32_zero:
4129 case Intrinsic::nvvm_suld_1d_v2i64_zero:
4131 case Intrinsic::nvvm_suld_1d_v4i8_zero:
4133 case Intrinsic::nvvm_suld_1d_v4i16_zero:
4135 case Intrinsic::nvvm_suld_1d_v4i32_zero:
4137 case Intrinsic::nvvm_suld_1d_array_i8_zero:
4139 case Intrinsic::nvvm_suld_1d_array_i16_zero:
4141 case Intrinsic::nvvm_suld_1d_array_i32_zero:
4143 case Intrinsic::nvvm_suld_1d_array_i64_zero:
4145 case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
4147 case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
4149 case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
4151 case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
4153 case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
4155 case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
4157 case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
4159 case Intrinsic::nvvm_suld_2d_i8_zero:
4161 case Intrinsic::nvvm_suld_2d_i16_zero:
4163 case Intrinsic::nvvm_suld_2d_i32_zero:
4165 case Intrinsic::nvvm_suld_2d_i64_zero:
4167 case Intrinsic::nvvm_suld_2d_v2i8_zero:
4169 case Intrinsic::nvvm_suld_2d_v2i16_zero:
4171 case Intrinsic::nvvm_suld_2d_v2i32_zero:
4173 case Intrinsic::nvvm_suld_2d_v2i64_zero:
4175 case Intrinsic::nvvm_suld_2d_v4i8_zero:
4177 case Intrinsic::nvvm_suld_2d_v4i16_zero:
4179 case Intrinsic::nvvm_suld_2d_v4i32_zero:
4181 case Intrinsic::nvvm_suld_2d_array_i8_zero:
4183 case Intrinsic::nvvm_suld_2d_array_i16_zero:
4185 case Intrinsic::nvvm_suld_2d_array_i32_zero:
4187 case Intrinsic::nvvm_suld_2d_array_i64_zero:
4189 case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
4191 case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
4193 case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
4195 case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
4197 case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
4199 case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
4201 case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
4203 case Intrinsic::nvvm_suld_3d_i8_zero:
4205 case Intrinsic::nvvm_suld_3d_i16_zero:
4207 case Intrinsic::nvvm_suld_3d_i32_zero:
4209 case Intrinsic::nvvm_suld_3d_i64_zero:
4211 case Intrinsic::nvvm_suld_3d_v2i8_zero:
4213 case Intrinsic::nvvm_suld_3d_v2i16_zero:
4215 case Intrinsic::nvvm_suld_3d_v2i32_zero:
4217 case Intrinsic::nvvm_suld_3d_v2i64_zero:
4219 case Intrinsic::nvvm_suld_3d_v4i8_zero:
4221 case Intrinsic::nvvm_suld_3d_v4i16_zero:
4223 case Intrinsic::nvvm_suld_3d_v4i32_zero:
4236 switch (Intrinsic) {
4239 case Intrinsic::nvvm_match_all_sync_i32p:
4240 case Intrinsic::nvvm_match_all_sync_i64p:
4245 Info.memVT = MVT::i1;
4250 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col:
4251 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row:
4252 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride:
4253 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride:
4254 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col:
4255 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row:
4256 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride:
4257 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride:
4258 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col:
4259 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row:
4260 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride:
4261 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride:
4262 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col:
4263 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row:
4264 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride:
4265 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride:
4266 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col:
4267 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row:
4268 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride:
4269 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride:
4270 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col:
4271 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row:
4272 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride:
4273 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride: {
4275 Info.memVT = MVT::v8f16;
4276 Info.ptrVal =
I.getArgOperand(0);
4282 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col:
4283 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col_stride:
4284 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col_stride:
4285 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col:
4286 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row:
4287 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row_stride:
4288 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row_stride:
4289 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row:
4290 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col:
4291 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col_stride:
4292 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row:
4293 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row_stride:
4294 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col:
4295 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col_stride:
4296 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col_stride:
4297 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col:
4298 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row:
4299 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row_stride:
4300 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row_stride:
4301 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row:
4302 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col:
4303 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col_stride:
4304 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row:
4305 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row_stride: {
4307 Info.memVT = MVT::v2i32;
4308 Info.ptrVal =
I.getArgOperand(0);
4315 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col:
4316 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col_stride:
4317 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col_stride:
4318 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col:
4319 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row:
4320 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row_stride:
4321 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row_stride:
4322 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row:
4323 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col:
4324 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col_stride:
4325 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row:
4326 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row_stride:
4327 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col:
4328 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col_stride:
4329 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row:
4330 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row_stride:
4332 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col:
4333 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col_stride:
4334 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col_stride:
4335 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col:
4336 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row:
4337 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row_stride:
4338 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row_stride:
4339 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row:
4340 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col:
4341 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col_stride:
4342 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row:
4343 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row_stride:
4344 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col:
4345 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col_stride:
4346 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row:
4347 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row_stride:
4348 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16:
4349 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16: {
4351 Info.memVT = MVT::v4i32;
4352 Info.ptrVal =
I.getArgOperand(0);
4359 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col:
4360 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col_stride:
4361 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col_stride:
4362 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col:
4363 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row:
4364 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row_stride:
4365 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row_stride:
4366 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row:
4368 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col:
4369 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col_stride:
4370 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col_stride:
4371 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col:
4372 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row:
4373 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row_stride:
4374 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row_stride:
4375 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row:
4376 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row:
4377 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row_stride:
4378 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col:
4379 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col_stride:
4380 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row:
4381 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row_stride:
4382 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row_stride:
4383 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row:
4384 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col:
4385 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col_stride:
4386 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col_stride:
4387 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col:
4388 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16:
4389 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16: {
4391 Info.memVT = MVT::i32;
4392 Info.ptrVal =
I.getArgOperand(0);
4399 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col:
4400 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row:
4401 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride:
4402 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride:
4403 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col:
4404 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row:
4405 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride:
4406 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride:
4407 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col:
4408 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row:
4409 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride:
4410 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride: {
4412 Info.memVT = MVT::v4f16;
4413 Info.ptrVal =
I.getArgOperand(0);
4420 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col:
4421 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row:
4422 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride:
4423 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride:
4424 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col:
4425 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row:
4426 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride:
4427 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride:
4428 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col:
4429 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row:
4430 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride:
4431 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride:
4432 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col:
4433 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row:
4434 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col_stride:
4435 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row_stride: {
4437 Info.memVT = MVT::v8f32;
4438 Info.ptrVal =
I.getArgOperand(0);
4445 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col:
4446 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col_stride:
4447 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row:
4448 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row_stride:
4450 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col:
4451 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col_stride:
4452 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row:
4453 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row_stride:
4455 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col:
4456 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col_stride:
4457 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row:
4458 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row_stride:
4459 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col:
4460 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col_stride:
4461 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row:
4462 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row_stride:
4463 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col:
4464 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col_stride:
4465 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row:
4466 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row_stride: {
4468 Info.memVT = MVT::v8i32;
4469 Info.ptrVal =
I.getArgOperand(0);
4476 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col:
4477 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col_stride:
4478 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row:
4479 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row_stride:
4480 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col:
4481 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col_stride:
4482 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row:
4483 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row_stride:
4484 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16:
4485 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16: {
4487 Info.memVT = MVT::v2i32;
4488 Info.ptrVal =
I.getArgOperand(0);
4495 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col:
4496 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col_stride:
4497 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row:
4498 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row_stride:
4500 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col:
4501 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col_stride:
4502 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row:
4503 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row_stride: {
4505 Info.memVT = MVT::f64;
4506 Info.ptrVal =
I.getArgOperand(0);
4513 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col:
4514 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col_stride:
4515 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row:
4516 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row_stride: {
4518 Info.memVT = MVT::v2f64;
4519 Info.ptrVal =
I.getArgOperand(0);
4526 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col:
4527 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row:
4528 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride:
4529 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride:
4530 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col:
4531 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row:
4532 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride:
4533 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride:
4534 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col:
4535 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row:
4536 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride:
4537 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride: {
4539 Info.memVT = MVT::v4f16;
4540 Info.ptrVal =
I.getArgOperand(0);
4547 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col:
4548 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row:
4549 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride:
4550 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride:
4551 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col:
4552 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row:
4553 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride:
4554 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride:
4555 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col:
4556 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row:
4557 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride:
4558 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride:
4559 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col:
4560 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row:
4561 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col_stride:
4562 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row_stride: {
4564 Info.memVT = MVT::v8f32;
4565 Info.ptrVal =
I.getArgOperand(0);
4572 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col:
4573 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col_stride:
4574 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row:
4575 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row_stride:
4576 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col:
4577 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col_stride:
4578 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row:
4579 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row_stride:
4580 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col:
4581 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col_stride:
4582 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row:
4583 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row_stride: {
4585 Info.memVT = MVT::v8i32;
4586 Info.ptrVal =
I.getArgOperand(0);
4593 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col:
4594 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col_stride:
4595 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row:
4596 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row_stride:
4597 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col:
4598 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col_stride:
4599 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row:
4600 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row_stride: {
4602 Info.memVT = MVT::v2i32;
4603 Info.ptrVal =
I.getArgOperand(0);
4610 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col:
4611 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col_stride:
4612 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row:
4613 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row_stride: {
4615 Info.memVT = MVT::v2f64;
4616 Info.ptrVal =
I.getArgOperand(0);
4623 case Intrinsic::nvvm_atomic_load_inc_32:
4624 case Intrinsic::nvvm_atomic_load_dec_32:
4626 case Intrinsic::nvvm_atomic_add_gen_f_cta:
4627 case Intrinsic::nvvm_atomic_add_gen_f_sys:
4628 case Intrinsic::nvvm_atomic_add_gen_i_cta:
4629 case Intrinsic::nvvm_atomic_add_gen_i_sys:
4630 case Intrinsic::nvvm_atomic_and_gen_i_cta:
4631 case Intrinsic::nvvm_atomic_and_gen_i_sys:
4632 case Intrinsic::nvvm_atomic_cas_gen_i_cta:
4633 case Intrinsic::nvvm_atomic_cas_gen_i_sys:
4634 case Intrinsic::nvvm_atomic_dec_gen_i_cta:
4635 case Intrinsic::nvvm_atomic_dec_gen_i_sys:
4636 case Intrinsic::nvvm_atomic_inc_gen_i_cta:
4637 case Intrinsic::nvvm_atomic_inc_gen_i_sys:
4638 case Intrinsic::nvvm_atomic_max_gen_i_cta:
4639 case Intrinsic::nvvm_atomic_max_gen_i_sys:
4640 case Intrinsic::nvvm_atomic_min_gen_i_cta:
4641 case Intrinsic::nvvm_atomic_min_gen_i_sys:
4642 case Intrinsic::nvvm_atomic_or_gen_i_cta:
4643 case Intrinsic::nvvm_atomic_or_gen_i_sys:
4644 case Intrinsic::nvvm_atomic_exch_gen_i_cta:
4645 case Intrinsic::nvvm_atomic_exch_gen_i_sys:
4646 case Intrinsic::nvvm_atomic_xor_gen_i_cta:
4647 case Intrinsic::nvvm_atomic_xor_gen_i_sys: {
4648 auto &
DL =
I.getDataLayout();
4651 Info.ptrVal =
I.getArgOperand(0);
4658 case Intrinsic::nvvm_ldu_global_i:
4659 case Intrinsic::nvvm_ldu_global_f:
4660 case Intrinsic::nvvm_ldu_global_p: {
4661 auto &
DL =
I.getDataLayout();
4663 if (Intrinsic == Intrinsic::nvvm_ldu_global_i)
4665 else if(Intrinsic == Intrinsic::nvvm_ldu_global_p)
4669 Info.ptrVal =
I.getArgOperand(0);
4672 Info.align = cast<ConstantInt>(
I.getArgOperand(1))->getMaybeAlignValue();
4676 case Intrinsic::nvvm_ldg_global_i:
4677 case Intrinsic::nvvm_ldg_global_f:
4678 case Intrinsic::nvvm_ldg_global_p: {
4679 auto &
DL =
I.getDataLayout();
4682 if (Intrinsic == Intrinsic::nvvm_ldg_global_i)
4684 else if(Intrinsic == Intrinsic::nvvm_ldg_global_p)
4688 Info.ptrVal =
I.getArgOperand(0);
4691 Info.align = cast<ConstantInt>(
I.getArgOperand(1))->getMaybeAlignValue();
4696 case Intrinsic::nvvm_tex_1d_v4f32_s32:
4697 case Intrinsic::nvvm_tex_1d_v4f32_f32:
4698 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
4699 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
4700 case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
4701 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
4702 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
4703 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
4704 case Intrinsic::nvvm_tex_2d_v4f32_s32:
4705 case Intrinsic::nvvm_tex_2d_v4f32_f32:
4706 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
4707 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
4708 case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
4709 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
4710 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
4711 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
4712 case Intrinsic::nvvm_tex_3d_v4f32_s32:
4713 case Intrinsic::nvvm_tex_3d_v4f32_f32:
4714 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
4715 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
4716 case Intrinsic::nvvm_tex_cube_v4f32_f32:
4717 case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
4718 case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
4719 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
4720 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
4721 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
4722 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
4723 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
4724 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
4725 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
4726 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
4727 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
4728 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
4729 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
4730 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
4731 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
4732 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
4733 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
4734 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
4735 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
4736 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
4737 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
4738 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
4739 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
4740 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
4741 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
4742 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
4743 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
4744 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
4745 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
4746 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
4747 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
4748 case Intrinsic::nvvm_tex_unified_cube_grad_v4f32_f32:
4749 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4f32_f32:
4750 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
4751 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
4752 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
4753 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
4755 Info.memVT = MVT::v4f32;
4756 Info.ptrVal =
nullptr;
4762 case Intrinsic::nvvm_tex_1d_v4s32_s32:
4763 case Intrinsic::nvvm_tex_1d_v4s32_f32:
4764 case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
4765 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
4766 case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
4767 case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
4768 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
4769 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
4770 case Intrinsic::nvvm_tex_2d_v4s32_s32:
4771 case Intrinsic::nvvm_tex_2d_v4s32_f32:
4772 case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
4773 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
4774 case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
4775 case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
4776 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
4777 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
4778 case Intrinsic::nvvm_tex_3d_v4s32_s32:
4779 case Intrinsic::nvvm_tex_3d_v4s32_f32:
4780 case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
4781 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
4782 case Intrinsic::nvvm_tex_cube_v4s32_f32:
4783 case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
4784 case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
4785 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
4786 case Intrinsic::nvvm_tex_cube_v4u32_f32:
4787 case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
4788 case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
4789 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
4790 case Intrinsic::nvvm_tex_1d_v4u32_s32:
4791 case Intrinsic::nvvm_tex_1d_v4u32_f32:
4792 case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
4793 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
4794 case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
4795 case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
4796 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
4797 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
4798 case Intrinsic::nvvm_tex_2d_v4u32_s32:
4799 case Intrinsic::nvvm_tex_2d_v4u32_f32:
4800 case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
4801 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
4802 case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
4803 case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
4804 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
4805 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
4806 case Intrinsic::nvvm_tex_3d_v4u32_s32:
4807 case Intrinsic::nvvm_tex_3d_v4u32_f32:
4808 case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
4809 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
4810 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
4811 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
4812 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
4813 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
4814 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
4815 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
4816 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
4817 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
4818 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
4819 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
4820 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
4821 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
4822 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
4823 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
4824 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
4825 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
4826 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
4827 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
4828 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
4829 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
4830 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
4831 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
4832 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
4833 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
4834 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
4835 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
4836 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
4837 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
4838 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
4839 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
4840 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
4841 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
4842 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
4843 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
4844 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
4845 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
4846 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
4847 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
4848 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
4849 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
4850 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
4851 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
4852 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
4853 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
4854 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
4855 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
4856 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
4857 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
4858 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
4859 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
4860 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
4861 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
4862 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
4863 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
4864 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
4865 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
4866 case Intrinsic::nvvm_tex_unified_cube_grad_v4s32_f32:
4867 case Intrinsic::nvvm_tex_unified_cube_grad_v4u32_f32:
4868 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4s32_f32:
4869 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4u32_f32:
4870 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
4871 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
4872 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
4873 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
4874 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
4875 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
4876 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
4877 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
4879 Info.memVT = MVT::v4i32;
4880 Info.ptrVal =
nullptr;
4886 case Intrinsic::nvvm_suld_1d_i8_clamp:
4887 case Intrinsic::nvvm_suld_1d_v2i8_clamp:
4888 case Intrinsic::nvvm_suld_1d_v4i8_clamp:
4889 case Intrinsic::nvvm_suld_1d_array_i8_clamp:
4890 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
4891 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
4892 case Intrinsic::nvvm_suld_2d_i8_clamp:
4893 case Intrinsic::nvvm_suld_2d_v2i8_clamp:
4894 case Intrinsic::nvvm_suld_2d_v4i8_clamp:
4895 case Intrinsic::nvvm_suld_2d_array_i8_clamp:
4896 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
4897 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
4898 case Intrinsic::nvvm_suld_3d_i8_clamp:
4899 case Intrinsic::nvvm_suld_3d_v2i8_clamp:
4900 case Intrinsic::nvvm_suld_3d_v4i8_clamp:
4901 case Intrinsic::nvvm_suld_1d_i8_trap:
4902 case Intrinsic::nvvm_suld_1d_v2i8_trap:
4903 case Intrinsic::nvvm_suld_1d_v4i8_trap:
4904 case Intrinsic::nvvm_suld_1d_array_i8_trap:
4905 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
4906 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
4907 case Intrinsic::nvvm_suld_2d_i8_trap:
4908 case Intrinsic::nvvm_suld_2d_v2i8_trap:
4909 case Intrinsic::nvvm_suld_2d_v4i8_trap:
4910 case Intrinsic::nvvm_suld_2d_array_i8_trap:
4911 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
4912 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
4913 case Intrinsic::nvvm_suld_3d_i8_trap:
4914 case Intrinsic::nvvm_suld_3d_v2i8_trap:
4915 case Intrinsic::nvvm_suld_3d_v4i8_trap:
4916 case Intrinsic::nvvm_suld_1d_i8_zero:
4917 case Intrinsic::nvvm_suld_1d_v2i8_zero:
4918 case Intrinsic::nvvm_suld_1d_v4i8_zero:
4919 case Intrinsic::nvvm_suld_1d_array_i8_zero:
4920 case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
4921 case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
4922 case Intrinsic::nvvm_suld_2d_i8_zero:
4923 case Intrinsic::nvvm_suld_2d_v2i8_zero:
4924 case Intrinsic::nvvm_suld_2d_v4i8_zero:
4925 case Intrinsic::nvvm_suld_2d_array_i8_zero:
4926 case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
4927 case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
4928 case Intrinsic::nvvm_suld_3d_i8_zero:
4929 case Intrinsic::nvvm_suld_3d_v2i8_zero:
4930 case Intrinsic::nvvm_suld_3d_v4i8_zero:
4932 Info.memVT = MVT::i8;
4933 Info.ptrVal =
nullptr;
4939 case Intrinsic::nvvm_suld_1d_i16_clamp:
4940 case Intrinsic::nvvm_suld_1d_v2i16_clamp:
4941 case Intrinsic::nvvm_suld_1d_v4i16_clamp:
4942 case Intrinsic::nvvm_suld_1d_array_i16_clamp:
4943 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
4944 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
4945 case Intrinsic::nvvm_suld_2d_i16_clamp:
4946 case Intrinsic::nvvm_suld_2d_v2i16_clamp:
4947 case Intrinsic::nvvm_suld_2d_v4i16_clamp:
4948 case Intrinsic::nvvm_suld_2d_array_i16_clamp:
4949 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
4950 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
4951 case Intrinsic::nvvm_suld_3d_i16_clamp:
4952 case Intrinsic::nvvm_suld_3d_v2i16_clamp:
4953 case Intrinsic::nvvm_suld_3d_v4i16_clamp:
4954 case Intrinsic::nvvm_suld_1d_i16_trap:
4955 case Intrinsic::nvvm_suld_1d_v2i16_trap:
4956 case Intrinsic::nvvm_suld_1d_v4i16_trap:
4957 case Intrinsic::nvvm_suld_1d_array_i16_trap:
4958 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
4959 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
4960 case Intrinsic::nvvm_suld_2d_i16_trap:
4961 case Intrinsic::nvvm_suld_2d_v2i16_trap:
4962 case Intrinsic::nvvm_suld_2d_v4i16_trap:
4963 case Intrinsic::nvvm_suld_2d_array_i16_trap:
4964 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
4965 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
4966 case Intrinsic::nvvm_suld_3d_i16_trap:
4967 case Intrinsic::nvvm_suld_3d_v2i16_trap:
4968 case Intrinsic::nvvm_suld_3d_v4i16_trap:
4969 case Intrinsic::nvvm_suld_1d_i16_zero:
4970 case Intrinsic::nvvm_suld_1d_v2i16_zero:
4971 case Intrinsic::nvvm_suld_1d_v4i16_zero:
4972 case Intrinsic::nvvm_suld_1d_array_i16_zero:
4973 case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
4974 case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
4975 case Intrinsic::nvvm_suld_2d_i16_zero:
4976 case Intrinsic::nvvm_suld_2d_v2i16_zero:
4977 case Intrinsic::nvvm_suld_2d_v4i16_zero:
4978 case Intrinsic::nvvm_suld_2d_array_i16_zero:
4979 case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
4980 case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
4981 case Intrinsic::nvvm_suld_3d_i16_zero:
4982 case Intrinsic::nvvm_suld_3d_v2i16_zero:
4983 case Intrinsic::nvvm_suld_3d_v4i16_zero:
4985 Info.memVT = MVT::i16;
4986 Info.ptrVal =
nullptr;
4992 case Intrinsic::nvvm_suld_1d_i32_clamp:
4993 case Intrinsic::nvvm_suld_1d_v2i32_clamp:
4994 case Intrinsic::nvvm_suld_1d_v4i32_clamp:
4995 case Intrinsic::nvvm_suld_1d_array_i32_clamp:
4996 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
4997 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
4998 case Intrinsic::nvvm_suld_2d_i32_clamp:
4999 case Intrinsic::nvvm_suld_2d_v2i32_clamp:
5000 case Intrinsic::nvvm_suld_2d_v4i32_clamp:
5001 case Intrinsic::nvvm_suld_2d_array_i32_clamp:
5002 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
5003 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
5004 case Intrinsic::nvvm_suld_3d_i32_clamp:
5005 case Intrinsic::nvvm_suld_3d_v2i32_clamp:
5006 case Intrinsic::nvvm_suld_3d_v4i32_clamp:
5007 case Intrinsic::nvvm_suld_1d_i32_trap:
5008 case Intrinsic::nvvm_suld_1d_v2i32_trap:
5009 case Intrinsic::nvvm_suld_1d_v4i32_trap:
5010 case Intrinsic::nvvm_suld_1d_array_i32_trap:
5011 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
5012 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
5013 case Intrinsic::nvvm_suld_2d_i32_trap:
5014 case Intrinsic::nvvm_suld_2d_v2i32_trap:
5015 case Intrinsic::nvvm_suld_2d_v4i32_trap:
5016 case Intrinsic::nvvm_suld_2d_array_i32_trap:
5017 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
5018 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
5019 case Intrinsic::nvvm_suld_3d_i32_trap:
5020 case Intrinsic::nvvm_suld_3d_v2i32_trap:
5021 case Intrinsic::nvvm_suld_3d_v4i32_trap:
5022 case Intrinsic::nvvm_suld_1d_i32_zero:
5023 case Intrinsic::nvvm_suld_1d_v2i32_zero:
5024 case Intrinsic::nvvm_suld_1d_v4i32_zero:
5025 case Intrinsic::nvvm_suld_1d_array_i32_zero:
5026 case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
5027 case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
5028 case Intrinsic::nvvm_suld_2d_i32_zero:
5029 case Intrinsic::nvvm_suld_2d_v2i32_zero:
5030 case Intrinsic::nvvm_suld_2d_v4i32_zero:
5031 case Intrinsic::nvvm_suld_2d_array_i32_zero:
5032 case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
5033 case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
5034 case Intrinsic::nvvm_suld_3d_i32_zero:
5035 case Intrinsic::nvvm_suld_3d_v2i32_zero:
5036 case Intrinsic::nvvm_suld_3d_v4i32_zero:
5038 Info.memVT = MVT::i32;
5039 Info.ptrVal =
nullptr;
5045 case Intrinsic::nvvm_suld_1d_i64_clamp:
5046 case Intrinsic::nvvm_suld_1d_v2i64_clamp:
5047 case Intrinsic::nvvm_suld_1d_array_i64_clamp:
5048 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
5049 case Intrinsic::nvvm_suld_2d_i64_clamp:
5050 case Intrinsic::nvvm_suld_2d_v2i64_clamp:
5051 case Intrinsic::nvvm_suld_2d_array_i64_clamp:
5052 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
5053 case Intrinsic::nvvm_suld_3d_i64_clamp:
5054 case Intrinsic::nvvm_suld_3d_v2i64_clamp:
5055 case Intrinsic::nvvm_suld_1d_i64_trap:
5056 case Intrinsic::nvvm_suld_1d_v2i64_trap:
5057 case Intrinsic::nvvm_suld_1d_array_i64_trap:
5058 case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
5059 case Intrinsic::nvvm_suld_2d_i64_trap:
5060 case Intrinsic::nvvm_suld_2d_v2i64_trap:
5061 case Intrinsic::nvvm_suld_2d_array_i64_trap:
5062 case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
5063 case Intrinsic::nvvm_suld_3d_i64_trap:
5064 case Intrinsic::nvvm_suld_3d_v2i64_trap:
5065 case Intrinsic::nvvm_suld_1d_i64_zero:
5066 case Intrinsic::nvvm_suld_1d_v2i64_zero:
5067 case Intrinsic::nvvm_suld_1d_array_i64_zero:
5068 case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
5069 case Intrinsic::nvvm_suld_2d_i64_zero:
5070 case Intrinsic::nvvm_suld_2d_v2i64_zero:
5071 case Intrinsic::nvvm_suld_2d_array_i64_zero:
5072 case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
5073 case Intrinsic::nvvm_suld_3d_i64_zero:
5074 case Intrinsic::nvvm_suld_3d_v2i64_zero:
5076 Info.memVT = MVT::i64;
5077 Info.ptrVal =
nullptr;
5097 const Align ABITypeAlign = std::min(
Align(128),
DL.getABITypeAlign(ArgTy));
5102 if (!
F || !
F->hasLocalLinkage() ||
5103 F->hasAddressTaken(
nullptr,
5107 return ABITypeAlign;
5110 return std::max(
Align(16), ABITypeAlign);
5117 Align ArgAlign = InitialAlign;
5132 ArgAlign = std::max(ArgAlign,
Align(4));
5142 std::string ParamName;
5147 ParamStr <<
"_vararg";
5149 ParamStr <<
"_param_" <<
Idx;
5201 if (Constraint.
size() == 1) {
5202 switch (Constraint[0]) {
5221std::pair<unsigned, const TargetRegisterClass *>
5225 if (Constraint.
size() == 1) {
5226 switch (Constraint[0]) {
5228 return std::make_pair(0U, &NVPTX::Int1RegsRegClass);
5230 return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
5232 return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
5234 return std::make_pair(0U, &NVPTX::Int32RegsRegClass);
5237 return std::make_pair(0U, &NVPTX::Int64RegsRegClass);
5241 "supported for sm_70 and higher!");
5242 return std::make_pair(0U, &NVPTX::Int128RegsRegClass);
5245 return std::make_pair(0U, &NVPTX::Float32RegsRegClass);
5247 return std::make_pair(0U, &NVPTX::Float64RegsRegClass);
5281 return F.getFnAttribute(
"unsafe-fp-math").getValueAsBool();
5285 const auto *Const = dyn_cast<ConstantSDNode>(Operand);
5286 return Const && Const->getZExtValue() == 0;
5324 if (M->getOpcode() !=
ISD::MUL || !M.getNode()->hasOneUse())
5328 M->getOperand(0), M->getOperand(1), N1);
5330 ((ZeroOpNum == 1) ? N1 : MAD),
5331 ((ZeroOpNum == 1) ? MAD : N1));
5357 int nonAddCount = 0;
5366 int orderNo =
N->getIROrder();
5372 if (orderNo - orderNo2 < 500)
5378 bool opIsLive =
false;
5382 if (isa<ConstantSDNode>(left) || isa<ConstantSDNode>(right))
5387 int orderNo3 =
User->getIROrder();
5388 if (orderNo3 > orderNo) {
5396 int orderNo3 =
User->getIROrder();
5397 if (orderNo3 > orderNo) {
5416 if (
all_of(
N->ops().drop_front(Front).drop_back(Back),
5417 [](
const SDUse &U) { return U.get()->isUndef(); }))
5420 return N->getOperand(0);
5449 if (VT.
isVector() || VT != MVT::i32)
5469 if (VT.
isVector() || !(VT == MVT::f32 || VT == MVT::f64))
5490 if (isa<ConstantSDNode>(Val)) {
5504 ConstantSDNode *BFEBits = dyn_cast<ConstantSDNode>(BFE.getOperand(0));
5516 if (MaskVal != (
uint64_t(1) << BFEBitsVal) - 1)
5540 if (MaskVal != 0xff) {
5545 MemSDNode *Mem = dyn_cast<MemSDNode>(Val);
5552 if (MemVT != MVT::v2i8 && MemVT != MVT::v4i8) {
5565 if (AExt.
getNode() !=
nullptr) {
5590 EVT VT =
N->getValueType(0);
5594 const SDValue &Num =
N->getOperand(0);
5595 const SDValue &Den =
N->getOperand(1);
5598 if (U->getOpcode() == DivOpc && U->getOperand(0) == Num &&
5599 U->getOperand(1) == Den) {
5626 EVT OrigVT =
Op.getOperand(0).getValueType();
5632 EVT OrigVT =
Op.getOperand(0).getValueType();
5659 IsSigned = (LHSSign ==
Signed);
5663 const APInt &Val = CI->getAPIntValue();
5665 return Val.
isIntN(OptSize);
5674 return LHSSign == RHSSign;
5684 EVT MulType =
N->getValueType(0);
5685 if (MulType != MVT::i32 && MulType != MVT::i64) {
5696 if (isa<ConstantSDNode>(
LHS)) {
5725 if (MulType == MVT::i32) {
5726 DemotedVT = MVT::i16;
5728 DemotedVT = MVT::i32;
5745 return DCI.
DAG.
getNode(Opc,
DL, MulType, TruncLHS, TruncRHS);
5749 const auto *Const = dyn_cast<ConstantSDNode>(Operand);
5750 return Const && Const->getZExtValue() == 1;
5758 return Add->getOperand(1);
5761 return Add->getOperand(0);
5800 (ConstOpNo == 1) ?
X : NewMul,
5801 (ConstOpNo == 1) ? NewMul :
X);
5812 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
5863 EVT CCType =
N->getValueType(0);
5867 EVT AType =
A.getValueType();
5868 if (!(CCType == MVT::v2i1 && (AType == MVT::v2f16 || AType == MVT::v2bf16)))
5871 if (
A.getValueType() == MVT::v2bf16 &&
SmVersion < 90)
5882 DL, DCI.
DAG.
getVTList(MVT::i1, MVT::i1), {A, B, N->getOperand(2)});
5899 VectorVT == MVT::v4i8 || VectorVT == MVT::v8i8)
5908 if (!(VectorBits == 16 || VectorBits == 32 || VectorBits == 64))
5928 if (EltVT != EltIVT)
5931 if (EltVT !=
N->getValueType(0))
5941 if (VectorVT != MVT::v4i8)
5952 for (
int I = 0;
I < 4; ++
I) {
5979 EVT VT =
N->getValueType(0);
5980 if (VT != MVT::v16i8)
5987 EVT NewVT = MVT::v4i32;
5990 EVT RetVTs[] = {EltVT, EltVT, EltVT, EltVT, MVT::Other};
5995 LD->getMemOperand());
6000 for (
unsigned i = 0; i < NumElts; i++)
6009 DAGCombinerInfo &DCI)
const {
6011 switch (
N->getOpcode()) {
6049 EVT ResVT =
N->getValueType(0);
6082 Align Alignment = LD->getAlign();
6086 if (Alignment < PrefAlign) {
6101 bool NeedTrunc =
false;
6107 unsigned Opcode = 0;
6109 bool Load16x2 =
false;
6116 LdResVTs = DAG.
getVTList(EltVT, EltVT, MVT::Other);
6120 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
6145 EVT ListVTs[] = {VVT, VVT, VVT, VVT, MVT::Other};
6160 LD->getMemOperand());
6166 for (
unsigned i = 0; i < NumElts; ++i) {
6176 for (
unsigned i = 0; i < NumElts; ++i) {
6203 case Intrinsic::nvvm_ldg_global_i:
6204 case Intrinsic::nvvm_ldg_global_f:
6205 case Intrinsic::nvvm_ldg_global_p:
6206 case Intrinsic::nvvm_ldu_global_i:
6207 case Intrinsic::nvvm_ldu_global_f:
6208 case Intrinsic::nvvm_ldu_global_p: {
6209 EVT ResVT =
N->getValueType(0);
6221 bool NeedTrunc =
false;
6227 unsigned Opcode = 0;
6237 case Intrinsic::nvvm_ldg_global_i:
6238 case Intrinsic::nvvm_ldg_global_f:
6239 case Intrinsic::nvvm_ldg_global_p:
6242 case Intrinsic::nvvm_ldu_global_i:
6243 case Intrinsic::nvvm_ldu_global_f:
6244 case Intrinsic::nvvm_ldu_global_p:
6248 LdResVTs = DAG.
getVTList(EltVT, EltVT, MVT::Other);
6254 case Intrinsic::nvvm_ldg_global_i:
6255 case Intrinsic::nvvm_ldg_global_f:
6256 case Intrinsic::nvvm_ldg_global_p:
6259 case Intrinsic::nvvm_ldu_global_i:
6260 case Intrinsic::nvvm_ldu_global_f:
6261 case Intrinsic::nvvm_ldu_global_p:
6265 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
6278 OtherOps.
append(
N->op_begin() + 2,
N->op_end());
6288 for (
unsigned i = 0; i < NumElts; ++i) {
6306 "Custom handling of non-i8 ldu/ldg?");
6339 assert(Reg.getValueType() == MVT::i128 &&
6340 "Custom lowering for CopyFromReg with 128-bit reg only");
6342 N->getValueType(2)};
6354void NVPTXTargetLowering::ReplaceNodeResults(
6356 switch (
N->getOpcode()) {
6392 auto ITy = cast<llvm::IntegerType>(Ty);
6401 switch (ITy->getBitWidth()) {
6420 switch (ITy->getBitWidth()) {
amdgpu AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file contains the declarations of entities that describe floating point environment and related ...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
static cl::opt< bool > sched4reg("nvptx-sched4reg", cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false))
static SDValue PerformEXTRACTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isConstOne(const SDValue &Operand)
static cl::opt< unsigned > FMAContractLevelOpt("nvptx-fma-level", cl::Hidden, cl::desc("NVPTX Specific: FMA contraction (0: don't do it" " 1: do it 2: do it aggressively"), cl::init(2))
static bool IsPTXVectorType(MVT VT)
static cl::opt< int > UsePrecDivF32("nvptx-prec-divf32", cl::Hidden, cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" " IEEE Compliant F32 div.rnd if available."), cl::init(2))
static SDValue PerformStoreParamCombine(SDNode *N)
static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results)
ReplaceVectorLoad - Convert vector loads into multi-output scalar loads.
static unsigned getOpcForSurfaceInstr(unsigned Intrinsic)
static void ReplaceCopyFromReg_128(SDNode *N, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results)
static bool Is16bitsType(MVT VT)
static SDValue combineMADConstOne(SDValue X, SDValue Add, EVT VT, SDLoc DL, TargetLowering::DAGCombinerInfo &DCI)
static bool IsTypePassedAsArray(const Type *Ty)
static SmallVector< ParamVectorizationFlags, 16 > VectorizePTXValueVTs(const SmallVectorImpl< EVT > &ValueVTs, const SmallVectorImpl< uint64_t > &Offsets, Align ParamAlignment, bool IsVAArg=false)
static unsigned CanMergeParamLoadStoresStartingAt(unsigned Idx, uint32_t AccessSize, const SmallVectorImpl< EVT > &ValueVTs, const SmallVectorImpl< uint64_t > &Offsets, Align ParamAlignment)
static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results)
static SDValue PerformFADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, CodeGenOptLevel OptLevel)
static unsigned getOpcForTextureInstr(unsigned Intrinsic)
static bool isConstZero(const SDValue &Operand)
static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG)
static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputePTXValueVTs - For the given Type Ty, returns the set of primitive EVTs that compose it.
static bool IsMulWideOperandDemotable(SDValue Op, unsigned OptSize, OperandSignedness &S)
IsMulWideOperandDemotable - Checks if the provided DAG node is an operand that can be demoted to OptS...
static SDValue LowerUnalignedStoreParam(SelectionDAG &DAG, SDValue Chain, uint64_t Offset, EVT ElementType, SDValue StVal, SDValue &InGlue, unsigned ArgID, const SDLoc &dl)
static SDValue PerformREMCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, CodeGenOptLevel OptLevel)
static SDValue PerformMULCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformStoreRetvalCombine(SDNode *N)
static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS, unsigned OptSize, bool &IsSigned)
AreMulWideOperandsDemotable - Checks if the given LHS and RHS operands can be demoted to OptSize bits...
static SDValue PerformStoreCombineHelper(SDNode *N, std::size_t Front, std::size_t Back)
static bool adjustElementType(EVT &ElementType)
static SDValue TryMULWIDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
TryMULWIDECombine - Attempt to replace a multiply of M bits with a multiply of M/2 bits that produces...
static SDValue combineMulSelectConstOne(SDValue X, SDValue Select, EVT VT, SDLoc DL, TargetLowering::DAGCombinerInfo &DCI)
static SDValue matchMADConstOnePattern(SDValue Add)
static cl::opt< bool > UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden, cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), cl::init(true))
static SDValue LowerUnalignedStoreRet(SelectionDAG &DAG, SDValue Chain, uint64_t Offset, EVT ElementType, SDValue RetVal, const SDLoc &dl)
static bool PromoteScalarIntegerPTX(const EVT &VT, MVT *PromotedVT)
PromoteScalarIntegerPTX Used to make sure the arguments/returns are suitable for passing and promote ...
static SDValue PerformSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, unsigned int SmVersion)
static SDValue LowerUnalignedLoadRetParam(SelectionDAG &DAG, SDValue &Chain, uint64_t Offset, EVT ElementType, SDValue &InGlue, SmallVectorImpl< SDValue > &TempProxyRegOps, const SDLoc &dl)
static std::atomic< unsigned > GlobalUniqueCallSite
static cl::opt< bool > ForceMinByValParamAlign("nvptx-force-min-byval-param-align", cl::Hidden, cl::desc("NVPTX Specific: force 4-byte minimal alignment for byval" " params of device functions."), cl::init(false))
static SDValue PerformSHLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, CodeGenOptLevel OptLevel)
PerformSHLCombine - Runs PTX-specific DAG combine patterns on SHL nodes.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Class for arbitrary precision integers.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
bool slt(const APInt &RHS) const
Signed less than comparison.
bool isIntN(unsigned N) const
Check if this APInt has an N-bits unsigned integer value.
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
This class represents an incoming formal argument to a Function.
an instruction that atomically reads a memory location, combines it with another value,...
@ Min
*p = old <signed v ? old : v
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
bool isFloatingPointOperation() const
BinOp getOperation() const
bool hasParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Return true if the attribute exists for the given argument.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
This class represents a function call, abstracting a target machine's calling convention.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Diagnostic information for unsupported feature in backend.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
Type * getReturnType() const
Returns the type of the ret val.
unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
MCSection * getDataSection() const
Instances of this class represent a uniqued identifier for a section in the current translation unit.
StringRef getName() const
getName - Get the symbol name.
unsigned getVectorNumElements() const
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static auto integer_valuetypes()
static auto fixedlen_vector_valuetypes()
static MVT getIntegerVT(unsigned BitWidth)
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
unsigned getMaxRequiredAlignment() const
bool hasAtomMinMax64() const
bool hasAtomAddF64() const
const NVPTXTargetLowering * getTargetLowering() const override
unsigned getPTXVersion() const
const NVPTXRegisterInfo * getRegisterInfo() const override
unsigned int getSmVersion() const
bool hasAtomBitwise64() const
bool allowFP16Math() const
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
const NVPTXTargetMachine * nvTM
SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const
NVPTXTargetLowering(const NVPTXTargetMachine &TM, const NVPTXSubtarget &STI)
bool useF32FTZ(const MachineFunction &MF) const
Align getFunctionArgumentAlignment(const Function *F, Type *Ty, unsigned Idx, const DataLayout &DL) const
SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &ExtraSteps, bool &UseOneConst, bool Reciprocal) const override
Hooks for building estimates in place of slower divisions and square roots.
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &dl, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
std::string getParamName(const Function *F, int Idx) const
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
std::string getPrototype(const DataLayout &DL, Type *, const ArgListTy &, const SmallVectorImpl< ISD::OutputArg > &, MaybeAlign retAlignment, std::optional< std::pair< unsigned, const APInt & > > VAInfo, const CallBase &CB, unsigned UniqueCallSite) const
Align getFunctionParamOptimizedAlign(const Function *F, Type *ArgTy, const DataLayout &DL) const
getFunctionParamOptimizedAlign - since function arguments are passed via .param space,...
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx, EVT VT) const override
Return the ValueType of the result of SETCC operations.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
Align getFunctionByValParamAlign(const Function *F, Type *ArgTy, Align InitialAlign, const DataLayout &DL) const
Helper for computing alignment of a device function byval parameter.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool allowFMA(MachineFunction &MF, CodeGenOptLevel OptLevel) const
bool usePrecSqrtF32() const
bool allowUnsafeFPMath(MachineFunction &MF) const
int getDivF32Level() const
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
UniqueStringSaver & getStrPool() const
MCSection * SelectSectionForGlobal(const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const override
~NVPTXTargetObjectFile() override
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
unsigned getIROrder() const
Return the node ordering.
iterator_range< use_iterator > uses()
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SectionKind - This is a simple POD value that classifies the properties of a section.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getSymbolFunctionGlobalAddress(SDValue Op, Function **TargetFunction=nullptr)
Return a GlobalAddress of the function from the current module with name matching the given ExternalS...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
const TargetLowering & getTargetLoweringInfo() const
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
MachineFunction & getMachineFunction() const
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Class to represent struct types.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
const TargetMachine & getTargetMachine() const
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth)
Tells the code generator which bitwidths to bypass.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrNegativeOneBooleanContent
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
void setJumpIsExpensive(bool isExpensive=true)
Tells the code generator not to expand logic operations on comparison predicates into separate sequen...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG) const
Truncate Op to ResultVT.
SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const
Expand round(fp) to fp conversion.
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
MCSymbol * getSymbol(const GlobalValue *GV) const
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
@ VoidTyID
type with no size
bool isAggregateType() const
Return true if the type is an aggregate type.
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeID getTypeID() const
Return the type id for the type.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
StringRef save(const char *S)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ SIGN_EXTEND
Conversion operators.
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
@ Bitcast
Perform the operation on a different, but equivalently sized type.
@ TexUnified1DS32FloatLevel
@ Tex1DArrayFloatFloatLevel
@ TexUnified2DU32FloatGrad
@ Tld4UnifiedG2DFloatFloat
@ TexUnifiedCubeArrayFloatFloatLevel
@ Tld4UnifiedR2DFloatFloat
@ Tex2DArrayS32FloatLevel
@ TexUnified1DArrayFloatFloatLevel
@ TexUnified2DFloatFloatLevel
@ TexUnified3DFloatFloatLevel
@ TexUnified1DFloatFloatLevel
@ TexUnified2DArrayU32Float
@ TexUnified1DArrayFloatFloat
@ Tex1DArrayFloatFloatGrad
@ TexUnifiedCubeArrayU32FloatGrad
@ TexUnified1DFloatFloatGrad
@ TexUnifiedCubeFloatFloatGrad
@ TexUnified2DArrayFloatFloat
@ TexUnified3DU32FloatLevel
@ TexUnified1DArrayU32Float
@ TexUnified2DArrayFloatFloatLevel
@ TexUnified2DFloatFloatGrad
@ TexUnified2DArrayU32S32
@ TexUnifiedCubeArrayS32FloatLevel
@ TexUnified1DArrayS32Float
@ TexUnified1DArrayS32FloatLevel
@ TexUnified2DS32FloatLevel
@ TexUnified3DU32FloatGrad
@ TexUnifiedCubeU32FloatLevel
@ TexUnified2DArrayU32FloatGrad
@ TexUnifiedCubeFloatFloatLevel
@ TexUnified1DArrayFloatS32
@ TexUnifiedCubeS32FloatLevel
@ TexUnified1DS32FloatGrad
@ Tex2DArrayFloatFloatLevel
@ TexUnifiedCubeArrayFloatFloat
@ TexUnifiedCubeArrayFloatFloatGrad
@ TexUnifiedCubeFloatFloat
@ TexUnified1DArrayU32S32
@ TexUnified3DFloatFloatGrad
@ Tld4UnifiedA2DFloatFloat
@ TexUnified3DS32FloatGrad
@ TexUnified2DU32FloatLevel
@ TexUnified1DArrayS32S32
@ TexCubeArrayFloatFloatLevel
@ TexUnified1DU32FloatGrad
@ TexCubeArrayS32FloatLevel
@ Tex2DArrayU32FloatLevel
@ Tex1DArrayU32FloatLevel
@ TexUnified2DArrayU32FloatLevel
@ TexUnified1DArrayFloatFloatGrad
@ TexUnifiedCubeS32FloatGrad
@ TexCubeArrayU32FloatLevel
@ TexUnified3DS32FloatLevel
@ TexUnifiedCubeArrayS32FloatGrad
@ TexUnified2DArrayS32Float
@ Tex2DArrayFloatFloatGrad
@ TexUnifiedCubeArrayS32Float
@ TexUnified2DArrayS32FloatLevel
@ Tex1DArrayS32FloatLevel
@ TexUnifiedCubeArrayU32FloatLevel
@ TexUnified2DArrayS32S32
@ TexUnified2DArrayFloatFloatGrad
@ TexUnifiedCubeU32FloatGrad
@ Tld4UnifiedB2DFloatFloat
@ TexUnified1DArrayU32FloatLevel
@ TexUnified1DArrayS32FloatGrad
@ TexUnified2DS32FloatGrad
@ TexUnified2DArrayS32FloatGrad
@ TexUnified1DU32FloatLevel
@ TexUnifiedCubeArrayU32Float
@ TexUnified2DArrayFloatS32
@ TexUnified1DArrayU32FloatGrad
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
static bool isIndirectCall(const MachineInstr &MI)
bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
MaybeAlign getAlign(const Function &F, unsigned Index)
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
unsigned promoteScalarArgumentSize(unsigned size)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CodeGenOptLevel
Code generation optimization level.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
DWARFExpression::Operation Op
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
constexpr unsigned BitWidth
bool isKernelFunction(const Function &F)
Function * getMaybeBitcastedCallee(const CallBase *CB)
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static const fltSemantics & IEEEsingle() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
DenormalModeKind Output
Denormal flushing mode for floating point instruction results in the default floating point environme...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)