42 #include "llvm/IR/IntrinsicsNVPTX.h"
64 #define DEBUG_TYPE "nvptx-lower"
76 cl::desc(
"NVPTX Specific: FMA contraction (0: don't do it"
77 " 1: do it 2: do it aggressively"),
82 cl::desc(
"NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
83 " IEEE Compliant F32 div.rnd if available."),
88 cl::desc(
"NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
151 uint64_t StartingOffset = 0) {
161 Offsets->push_back(StartingOffset + 0);
162 Offsets->push_back(StartingOffset + 8);
169 if (
StructType *STy = dyn_cast<StructType>(Ty)) {
170 auto const *SL =
DL.getStructLayout(STy);
172 for(
auto *EI : STy->elements()) {
174 StartingOffset + SL->getElementOffset(ElementNum));
181 for (
unsigned i = 0,
e = TempVTs.size();
i !=
e; ++
i) {
183 uint64_t Off = TempOffsets[
i];
192 if (EltVT ==
MVT::f16 && NumElts % 2 == 0) {
196 for (
unsigned j = 0;
j != NumElts; ++
j) {
197 ValueVTs.push_back(EltVT);
202 ValueVTs.push_back(VT);
224 if (ParamAlignment < AccessSize)
227 if (
Offsets[Idx] & (AccessSize - 1))
230 EVT EltVT = ValueVTs[Idx];
234 if (EltSize >= AccessSize)
237 unsigned NumElts = AccessSize / EltSize;
239 if (AccessSize != EltSize * NumElts)
243 if (Idx + NumElts > ValueVTs.size())
247 if (NumElts != 4 && NumElts != 2)
250 for (
unsigned j = Idx + 1;
j < Idx + NumElts; ++
j) {
252 if (ValueVTs[
j] != EltVT)
284 Align ParamAlignment) {
291 for (
int I = 0,
E = ValueVTs.size();
I !=
E; ++
I) {
294 for (
unsigned AccessSize : {16, 8, 4, 2}) {
296 I, AccessSize, ValueVTs,
Offsets, ParamAlignment);
305 assert(
I + 1 <
E &&
"Not enough elements.");
311 assert(
I + 3 <
E &&
"Not enough elements.");
581 return "NVPTXISD::CALL";
583 return "NVPTXISD::RET_FLAG";
585 return "NVPTXISD::LOAD_PARAM";
587 return "NVPTXISD::Wrapper";
589 return "NVPTXISD::DeclareParam";
591 return "NVPTXISD::DeclareScalarParam";
593 return "NVPTXISD::DeclareRet";
595 return "NVPTXISD::DeclareScalarRet";
597 return "NVPTXISD::DeclareRetParam";
599 return "NVPTXISD::PrintCall";
601 return "NVPTXISD::PrintConvergentCall";
603 return "NVPTXISD::PrintCallUni";
605 return "NVPTXISD::PrintConvergentCallUni";
607 return "NVPTXISD::LoadParam";
609 return "NVPTXISD::LoadParamV2";
611 return "NVPTXISD::LoadParamV4";
613 return "NVPTXISD::StoreParam";
615 return "NVPTXISD::StoreParamV2";
617 return "NVPTXISD::StoreParamV4";
619 return "NVPTXISD::StoreParamS32";
621 return "NVPTXISD::StoreParamU32";
623 return "NVPTXISD::CallArgBegin";
625 return "NVPTXISD::CallArg";
627 return "NVPTXISD::LastCallArg";
629 return "NVPTXISD::CallArgEnd";
631 return "NVPTXISD::CallVoid";
633 return "NVPTXISD::CallVal";
635 return "NVPTXISD::CallSymbol";
637 return "NVPTXISD::Prototype";
639 return "NVPTXISD::MoveParam";
641 return "NVPTXISD::StoreRetval";
643 return "NVPTXISD::StoreRetvalV2";
645 return "NVPTXISD::StoreRetvalV4";
647 return "NVPTXISD::PseudoUseParam";
649 return "NVPTXISD::RETURN";
651 return "NVPTXISD::CallSeqBegin";
653 return "NVPTXISD::CallSeqEnd";
655 return "NVPTXISD::CallPrototype";
657 return "NVPTXISD::ProxyReg";
659 return "NVPTXISD::LoadV2";
661 return "NVPTXISD::LoadV4";
663 return "NVPTXISD::LDGV2";
665 return "NVPTXISD::LDGV4";
667 return "NVPTXISD::LDUV2";
669 return "NVPTXISD::LDUV4";
671 return "NVPTXISD::StoreV2";
673 return "NVPTXISD::StoreV4";
675 return "NVPTXISD::FUN_SHFL_CLAMP";
677 return "NVPTXISD::FUN_SHFR_CLAMP";
679 return "NVPTXISD::IMAD";
681 return "NVPTXISD::SETP_F16X2";
683 return "NVPTXISD::Dummy";
685 return "NVPTXISD::MUL_WIDE_SIGNED";
687 return "NVPTXISD::MUL_WIDE_UNSIGNED";
691 return "NVPTXISD::Tex1DFloatFloatLevel";
693 return "NVPTXISD::Tex1DFloatFloatGrad";
697 return "NVPTXISD::Tex1DS32FloatLevel";
699 return "NVPTXISD::Tex1DS32FloatGrad";
703 return "NVPTXISD::Tex1DU32FloatLevel";
705 return "NVPTXISD::Tex1DU32FloatGrad";
709 return "NVPTXISD::Tex1DArrayFloatFloatLevel";
711 return "NVPTXISD::Tex1DArrayFloatFloatGrad";
715 return "NVPTXISD::Tex1DArrayS32FloatLevel";
717 return "NVPTXISD::Tex1DArrayS32FloatGrad";
721 return "NVPTXISD::Tex1DArrayU32FloatLevel";
723 return "NVPTXISD::Tex1DArrayU32FloatGrad";
727 return "NVPTXISD::Tex2DFloatFloatLevel";
729 return "NVPTXISD::Tex2DFloatFloatGrad";
733 return "NVPTXISD::Tex2DS32FloatLevel";
735 return "NVPTXISD::Tex2DS32FloatGrad";
739 return "NVPTXISD::Tex2DU32FloatLevel";
741 return "NVPTXISD::Tex2DU32FloatGrad";
745 return "NVPTXISD::Tex2DArrayFloatFloatLevel";
747 return "NVPTXISD::Tex2DArrayFloatFloatGrad";
751 return "NVPTXISD::Tex2DArrayS32FloatLevel";
753 return "NVPTXISD::Tex2DArrayS32FloatGrad";
757 return "NVPTXISD::Tex2DArrayU32FloatLevel";
759 return "NVPTXISD::Tex2DArrayU32FloatGrad";
763 return "NVPTXISD::Tex3DFloatFloatLevel";
765 return "NVPTXISD::Tex3DFloatFloatGrad";
769 return "NVPTXISD::Tex3DS32FloatLevel";
771 return "NVPTXISD::Tex3DS32FloatGrad";
775 return "NVPTXISD::Tex3DU32FloatLevel";
777 return "NVPTXISD::Tex3DU32FloatGrad";
780 return "NVPTXISD::TexCubeFloatFloatLevel";
783 return "NVPTXISD::TexCubeS32FloatLevel";
786 return "NVPTXISD::TexCubeU32FloatLevel";
788 return "NVPTXISD::TexCubeArrayFloatFloat";
790 return "NVPTXISD::TexCubeArrayFloatFloatLevel";
792 return "NVPTXISD::TexCubeArrayS32Float";
794 return "NVPTXISD::TexCubeArrayS32FloatLevel";
796 return "NVPTXISD::TexCubeArrayU32Float";
798 return "NVPTXISD::TexCubeArrayU32FloatLevel";
800 return "NVPTXISD::Tld4R2DFloatFloat";
802 return "NVPTXISD::Tld4G2DFloatFloat";
804 return "NVPTXISD::Tld4B2DFloatFloat";
806 return "NVPTXISD::Tld4A2DFloatFloat";
808 return "NVPTXISD::Tld4R2DS64Float";
810 return "NVPTXISD::Tld4G2DS64Float";
812 return "NVPTXISD::Tld4B2DS64Float";
814 return "NVPTXISD::Tld4A2DS64Float";
816 return "NVPTXISD::Tld4R2DU64Float";
818 return "NVPTXISD::Tld4G2DU64Float";
820 return "NVPTXISD::Tld4B2DU64Float";
822 return "NVPTXISD::Tld4A2DU64Float";
825 return "NVPTXISD::TexUnified1DFloatS32";
827 return "NVPTXISD::TexUnified1DFloatFloat";
829 return "NVPTXISD::TexUnified1DFloatFloatLevel";
831 return "NVPTXISD::TexUnified1DFloatFloatGrad";
833 return "NVPTXISD::TexUnified1DS32S32";
835 return "NVPTXISD::TexUnified1DS32Float";
837 return "NVPTXISD::TexUnified1DS32FloatLevel";
839 return "NVPTXISD::TexUnified1DS32FloatGrad";
841 return "NVPTXISD::TexUnified1DU32S32";
843 return "NVPTXISD::TexUnified1DU32Float";
845 return "NVPTXISD::TexUnified1DU32FloatLevel";
847 return "NVPTXISD::TexUnified1DU32FloatGrad";
849 return "NVPTXISD::TexUnified1DArrayFloatS32";
851 return "NVPTXISD::TexUnified1DArrayFloatFloat";
853 return "NVPTXISD::TexUnified1DArrayFloatFloatLevel";
855 return "NVPTXISD::TexUnified1DArrayFloatFloatGrad";
857 return "NVPTXISD::TexUnified1DArrayS32S32";
859 return "NVPTXISD::TexUnified1DArrayS32Float";
861 return "NVPTXISD::TexUnified1DArrayS32FloatLevel";
863 return "NVPTXISD::TexUnified1DArrayS32FloatGrad";
865 return "NVPTXISD::TexUnified1DArrayU32S32";
867 return "NVPTXISD::TexUnified1DArrayU32Float";
869 return "NVPTXISD::TexUnified1DArrayU32FloatLevel";
871 return "NVPTXISD::TexUnified1DArrayU32FloatGrad";
873 return "NVPTXISD::TexUnified2DFloatS32";
875 return "NVPTXISD::TexUnified2DFloatFloat";
877 return "NVPTXISD::TexUnified2DFloatFloatLevel";
879 return "NVPTXISD::TexUnified2DFloatFloatGrad";
881 return "NVPTXISD::TexUnified2DS32S32";
883 return "NVPTXISD::TexUnified2DS32Float";
885 return "NVPTXISD::TexUnified2DS32FloatLevel";
887 return "NVPTXISD::TexUnified2DS32FloatGrad";
889 return "NVPTXISD::TexUnified2DU32S32";
891 return "NVPTXISD::TexUnified2DU32Float";
893 return "NVPTXISD::TexUnified2DU32FloatLevel";
895 return "NVPTXISD::TexUnified2DU32FloatGrad";
897 return "NVPTXISD::TexUnified2DArrayFloatS32";
899 return "NVPTXISD::TexUnified2DArrayFloatFloat";
901 return "NVPTXISD::TexUnified2DArrayFloatFloatLevel";
903 return "NVPTXISD::TexUnified2DArrayFloatFloatGrad";
905 return "NVPTXISD::TexUnified2DArrayS32S32";
907 return "NVPTXISD::TexUnified2DArrayS32Float";
909 return "NVPTXISD::TexUnified2DArrayS32FloatLevel";
911 return "NVPTXISD::TexUnified2DArrayS32FloatGrad";
913 return "NVPTXISD::TexUnified2DArrayU32S32";
915 return "NVPTXISD::TexUnified2DArrayU32Float";
917 return "NVPTXISD::TexUnified2DArrayU32FloatLevel";
919 return "NVPTXISD::TexUnified2DArrayU32FloatGrad";
921 return "NVPTXISD::TexUnified3DFloatS32";
923 return "NVPTXISD::TexUnified3DFloatFloat";
925 return "NVPTXISD::TexUnified3DFloatFloatLevel";
927 return "NVPTXISD::TexUnified3DFloatFloatGrad";
929 return "NVPTXISD::TexUnified3DS32S32";
931 return "NVPTXISD::TexUnified3DS32Float";
933 return "NVPTXISD::TexUnified3DS32FloatLevel";
935 return "NVPTXISD::TexUnified3DS32FloatGrad";
937 return "NVPTXISD::TexUnified3DU32S32";
939 return "NVPTXISD::TexUnified3DU32Float";
941 return "NVPTXISD::TexUnified3DU32FloatLevel";
943 return "NVPTXISD::TexUnified3DU32FloatGrad";
945 return "NVPTXISD::TexUnifiedCubeFloatFloat";
947 return "NVPTXISD::TexUnifiedCubeFloatFloatLevel";
949 return "NVPTXISD::TexUnifiedCubeS32Float";
951 return "NVPTXISD::TexUnifiedCubeS32FloatLevel";
953 return "NVPTXISD::TexUnifiedCubeU32Float";
955 return "NVPTXISD::TexUnifiedCubeU32FloatLevel";
957 return "NVPTXISD::TexUnifiedCubeArrayFloatFloat";
959 return "NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel";
961 return "NVPTXISD::TexUnifiedCubeArrayS32Float";
963 return "NVPTXISD::TexUnifiedCubeArrayS32FloatLevel";
965 return "NVPTXISD::TexUnifiedCubeArrayU32Float";
967 return "NVPTXISD::TexUnifiedCubeArrayU32FloatLevel";
969 return "NVPTXISD::Tld4UnifiedR2DFloatFloat";
971 return "NVPTXISD::Tld4UnifiedG2DFloatFloat";
973 return "NVPTXISD::Tld4UnifiedB2DFloatFloat";
975 return "NVPTXISD::Tld4UnifiedA2DFloatFloat";
977 return "NVPTXISD::Tld4UnifiedR2DS64Float";
979 return "NVPTXISD::Tld4UnifiedG2DS64Float";
981 return "NVPTXISD::Tld4UnifiedB2DS64Float";
983 return "NVPTXISD::Tld4UnifiedA2DS64Float";
985 return "NVPTXISD::Tld4UnifiedR2DU64Float";
987 return "NVPTXISD::Tld4UnifiedG2DU64Float";
989 return "NVPTXISD::Tld4UnifiedB2DU64Float";
991 return "NVPTXISD::Tld4UnifiedA2DU64Float";
1188 bool Reciprocal)
const {
1193 if (ExtraSteps == ReciprocalEstimate::Unspecified)
1209 if (Reciprocal || ExtraSteps > 0) {
1211 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f
1212 : Intrinsic::nvvm_rsqrt_approx_f);
1214 return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d);
1219 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f
1220 : Intrinsic::nvvm_sqrt_approx_f);
1229 MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d));
1246 const CallBase &CB,
unsigned UniqueCallSite)
const {
1250 assert(isABI &&
"Non-ABI compilation is not supported");
1254 std::stringstream
O;
1255 O <<
"prototype_" << UniqueCallSite <<
" : .callprototype ";
1263 if (
auto *ITy = dyn_cast<IntegerType>(retTy)) {
1264 size = ITy->getBitWidth();
1267 "Floating point type expected here");
1276 O <<
".param .b" <<
size <<
" _";
1277 }
else if (isa<PointerType>(retTy)) {
1278 O <<
".param .b" << PtrVT.getSizeInBits() <<
" _";
1281 O <<
".param .align " << (retAlignment ? retAlignment->value() : 0)
1282 <<
" .b8 _[" <<
DL.getTypeAllocSize(retTy) <<
"]";
1293 for (
unsigned i = 0,
e =
Args.size();
i !=
e; ++
i, ++OIdx) {
1300 if (!Outs[OIdx].Flags.isByVal()) {
1303 const CallInst *CallI = cast<CallInst>(&CB);
1306 align =
DL.getABITypeAlignment(Ty);
1307 unsigned sz =
DL.getTypeAllocSize(Ty);
1308 O <<
".param .align " <<
align <<
" .b8 ";
1310 O <<
"[" << sz <<
"]";
1314 if (
unsigned len = vtparts.size())
1321 "type mismatch between callee prototype and arguments");
1324 if (isa<IntegerType>(Ty)) {
1325 sz = cast<IntegerType>(Ty)->getBitWidth();
1328 }
else if (isa<PointerType>(Ty)) {
1329 sz = PtrVT.getSizeInBits();
1337 O <<
".param .b" << sz <<
" ";
1341 auto *PTy = dyn_cast<PointerType>(Ty);
1342 assert(PTy &&
"Param with byval attribute should be a pointer type");
1343 Type *ETy = PTy->getElementType();
1345 Align align = Outs[OIdx].Flags.getNonZeroByValAlign();
1346 unsigned sz =
DL.getTypeAllocSize(ETy);
1347 O <<
".param .align " <<
align.value() <<
" .b8 ";
1349 O <<
"[" << sz <<
"]";
1355 Align NVPTXTargetLowering::getArgumentAlignment(
SDValue Callee,
1361 return DL.getABITypeAlign(Ty);
1364 unsigned Alignment = 0;
1367 if (!DirectCallee) {
1372 if (
const auto *CI = dyn_cast<CallInst>(CB)) {
1375 return Align(Alignment);
1377 const Value *CalleeV = CI->getCalledOperand();
1379 while (isa<ConstantExpr>(CalleeV)) {
1384 CalleeV = cast<ConstantExpr>(CalleeV)->getOperand(0);
1389 if (
const auto *CalleeF = dyn_cast<Function>(CalleeV))
1390 DirectCallee = CalleeF;
1397 if (
getAlign(*DirectCallee, Idx, Alignment))
1398 return Align(Alignment);
1402 return DL.getABITypeAlign(Ty);
1421 assert(isABI &&
"Non-ABI compilation is not supported");
1430 unsigned paramCount = 0;
1443 for (
unsigned i = 0,
e =
Args.size();
i !=
e; ++
i, ++OIdx) {
1444 EVT VT = Outs[OIdx].VT;
1447 if (!Outs[OIdx].Flags.isByVal()) {
1451 Align ArgAlign = getArgumentAlignment(
Callee, CB, Ty, paramCount + 1,
DL);
1452 unsigned AllocSize =
DL.getTypeAllocSize(Ty);
1472 SDValue DeclareScalarParamOps[] = {
1477 DeclareScalarParamOps);
1486 bool ExtendIntegerParam =
1491 for (
unsigned j = 0, je = VTs.size();
j != je; ++
j) {
1494 assert(StoreOperands.empty() &&
"Unfinished preceding store.");
1495 StoreOperands.push_back(Chain);
1501 SDValue StVal = OutVals[OIdx];
1502 if (ExtendIntegerParam) {
1503 assert(VTs.size() == 1 &&
"Scalar can't have multiple parts.");
1515 StoreOperands.push_back(StVal);
1518 unsigned NumElts = StoreOperands.size() - 3;
1534 StoreOperands.push_back(InFlag);
1538 EVT TheStoreType = ExtendIntegerParam ?
MVT::i32 : VTs[
j];
1550 StoreOperands.
clear();
1554 assert(StoreOperands.empty() &&
"Unfinished parameter store.");
1564 auto *PTy = dyn_cast<PointerType>(
Args[
i].Ty);
1565 assert(PTy &&
"Type of a byval parameter should be pointer");
1569 unsigned sz = Outs[OIdx].Flags.getByValSize();
1571 Align ArgAlign = Outs[OIdx].Flags.getNonZeroByValAlign();
1579 if (ArgAlign <
Align(4))
1580 ArgAlign =
Align(4);
1588 for (
unsigned j = 0, je = VTs.size();
j != je; ++
j) {
1589 EVT elemtype = VTs[
j];
1601 SDValue CopyParamOps[] = { Chain,
1618 if (
Ins.size() > 0) {
1625 unsigned resultsz =
DL.getTypeAllocSizeInBits(RetTy);
1643 retAlignment = getArgumentAlignment(
Callee, CB, RetTy, 0,
DL);
1644 assert(retAlignment &&
"retAlignment is guaranteed to be set");
1661 if (isa<ExternalSymbolSDNode>(
Callee)) {
1666 assert(CalleeFunc !=
nullptr &&
"Libcall callee must be set.");
1670 CalleeFunc->
addFnAttr(
"nvptx-libcall-callee",
"true");
1684 const char *ProtoStr =
1702 Chain = DAG.
getNode(Opcode, dl, PrintCallVTs, PrintCallOps);
1713 SDValue CallArgBeginOps[] = { Chain, InFlag };
1718 for (
unsigned i = 0,
e = paramCount;
i !=
e; ++
i) {
1727 Chain = DAG.
getNode(opcode, dl, CallArgVTs, CallArgOps);
1731 SDValue CallArgEndOps[] = { Chain,
1749 if (
Ins.size() > 0) {
1753 assert(VTs.size() ==
Ins.size() &&
"Bad value decomposition");
1755 Align RetAlign = getArgumentAlignment(
Callee, CB, RetTy, 0,
DL);
1764 bool ExtendIntegerRetVal =
1765 RetTy->
isIntegerTy() &&
DL.getTypeAllocSizeInBits(RetTy) < 32;
1767 for (
unsigned i = 0,
e = VTs.size();
i !=
e; ++
i) {
1768 bool needTruncate =
false;
1769 EVT TheLoadType = VTs[
i];
1772 if (ExtendIntegerRetVal) {
1775 needTruncate =
true;
1778 needTruncate =
true;
1784 assert(VecIdx == -1 && LoadVTs.empty() &&
"Orphaned operand list.");
1788 LoadVTs.push_back(EltType);
1791 unsigned NumElts = LoadVTs.size();
1813 Op, dl, DAG.
getVTList(LoadVTs), LoadOperands, TheLoadType,
1817 for (
unsigned j = 0;
j < NumElts; ++
j) {
1818 ProxyRegOps.push_back(RetVal.
getValue(
j));
1827 InFlag = RetVal.
getValue(NumElts + 1);
1844 for (
unsigned i = 0;
i < ProxyRegOps.size(); ++
i) {
1848 { Chain, ProxyRegOps[i], InFlag }
1851 Chain =
Ret.getValue(1);
1852 InFlag =
Ret.getValue(2);
1854 if (ProxyRegTruncates[
i].hasValue()) {
1858 InVals.push_back(
Ret);
1875 unsigned NumOperands = Node->getNumOperands();
1876 for (
unsigned i = 0;
i < NumOperands; ++
i) {
1877 SDValue SubOp = Node->getOperand(
i);
1881 for (
unsigned j = 0;
j < NumSubElem; ++
j) {
1903 isa<ConstantFPSDNode>(
Op->getOperand(0)) &&
1904 isa<ConstantFPSDNode>(
Op->getOperand(1))))
1908 cast<ConstantFPSDNode>(
Op->getOperand(0))->getValueAPF().bitcastToAPInt();
1910 cast<ConstantFPSDNode>(
Op->getOperand(1))->getValueAPF().bitcastToAPInt();
1920 if (isa<ConstantSDNode>(
Index.getNode()))
1945 assert(
Op.getNumOperands() == 3 &&
"Not a double-shift!");
1948 EVT VT =
Op.getValueType();
2006 assert(
Op.getNumOperands() == 3 &&
"Not a double-shift!");
2009 EVT VT =
Op.getValueType();
2060 EVT VT =
Op.getValueType();
2063 return LowerFROUND32(
Op, DAG);
2066 return LowerFROUND64(
Op, DAG);
2082 EVT VT =
Op.getValueType();
2088 const int SignBitMask = 0x80000000;
2091 const int PointFiveInBits = 0x3F000000;
2092 SDValue PointFiveWithSignRaw =
2123 EVT VT =
Op.getValueType();
2155 switch (
Op.getOpcode()) {
2165 return LowerBUILD_VECTOR(
Op, DAG);
2169 return LowerEXTRACT_VECTOR_ELT(
Op, DAG);
2171 return LowerCONCAT_VECTORS(
Op, DAG);
2173 return LowerSTORE(
Op, DAG);
2175 return LowerLOAD(
Op, DAG);
2177 return LowerShiftLeftParts(
Op, DAG);
2180 return LowerShiftRightParts(
Op, DAG);
2182 return LowerSelect(
Op, DAG);
2184 return LowerFROUND(
Op, DAG);
2196 assert(
Op.getValueType() ==
MVT::i1 &&
"Custom lowering enabled only for i1");
2208 return LowerLOADi1(
Op, DAG);
2214 EVT MemVT =
Load->getMemoryVT();
2216 MemVT, *
Load->getMemOperand())) {
2236 "Custom lowering for i1 load only");
2238 LD->getPointerInfo(),
LD->getAlignment(),
2239 LD->getMemOperand()->getFlags());
2253 return LowerSTOREi1(
Op, DAG);
2259 VT, *
Store->getMemOperand()))
2263 return LowerSTOREVector(
Op, DAG);
2307 if (Alignment < PrefAlign) {
2316 unsigned Opcode = 0;
2323 bool NeedExt =
false;
2327 bool StoreF16x2 =
false;
2350 Ops.push_back(
N->getOperand(0));
2355 for (
unsigned i = 0;
i < NumElts; ++
i) {
2365 for (
unsigned i = 0;
i < NumElts; ++
i) {
2370 Ops.push_back(ExtVal);
2375 Ops.
append(
N->op_begin() + 2,
N->op_end());
2403 ST->getAlignment(),
ST->getMemOperand()->getFlags());
2408 NVPTXTargetLowering::getParamSymbol(
SelectionDAG &DAG,
int idx,
EVT v)
const {
2409 std::string ParamSym;
2415 std::string *SavedStr =
2423 static const char *
const specialTypes[] = {
"struct._image2d_t",
2424 "struct._image3d_t",
2425 "struct._sampler_t" };
2428 auto *PTy = dyn_cast<PointerType>(Ty);
2436 auto *STy = dyn_cast<StructType>(PTy->getElementType());
2437 if (!STy || STy->isLiteral())
2456 std::vector<SDValue> OutChains;
2459 assert(isABI &&
"Non-ABI compilation is not supported");
2463 std::vector<Type *> argTypes;
2464 std::vector<const Argument *> theArgs;
2466 theArgs.push_back(&
I);
2467 argTypes.push_back(
I.getType());
2478 unsigned InsIdx = 0;
2481 for (
unsigned i = 0,
e = theArgs.size();
i !=
e; ++
i, ++idx, ++InsIdx) {
2482 Type *Ty = argTypes[
i];
2492 "Only kernels can have image/sampler params");
2497 if (theArgs[
i]->use_empty()) {
2503 assert(vtparts.size() > 0 &&
"empty aggregate type not expected");
2504 for (
unsigned parti = 0, parte = vtparts.size(); parti != parte;
2509 if (vtparts.size() > 0)
2516 for (
unsigned parti = 0; parti < NumRegs; ++parti) {
2533 bool aggregateIsPacked =
false;
2534 if (
StructType *STy = dyn_cast<StructType>(Ty))
2535 aggregateIsPacked = STy->isPacked();
2540 assert(VTs.size() > 0 &&
"Unexpected empty type.");
2544 SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
2546 for (
unsigned parti = 0, parte = VTs.size(); parti != parte; ++parti) {
2548 assert(VecIdx == -1 &&
"Orphaned vector.");
2553 if (VectorInfo[parti] &
PVF_LAST) {
2554 unsigned NumElts = parti - VecIdx + 1;
2555 EVT EltVT = VTs[parti];
2573 DAG.
getLoad(VecVT, dl, Root, VecAddr,
2578 P.getNode()->setIROrder(idx + 1);
2579 for (
unsigned j = 0;
j < NumElts; ++
j) {
2591 Ins[InsIdx].VT.getFixedSizeInBits() >
2595 Elt = DAG.
getNode(Extend, dl,
Ins[InsIdx].VT, Elt);
2597 InVals.push_back(Elt);
2619 "Ins type did not match function type");
2620 SDValue Arg = getParamSymbol(DAG, idx, PtrVT);
2623 p.getNode()->setIROrder(idx + 1);
2624 InVals.push_back(
p);
2635 if (!OutChains.empty())
2651 assert(isABI &&
"Non-ABI compilation is not supported");
2659 assert(VTs.size() == OutVals.size() &&
"Bad return value decomposition");
2667 bool ExtendIntegerRetVal =
2668 RetTy->
isIntegerTy() &&
DL.getTypeAllocSizeInBits(RetTy) < 32;
2671 for (
unsigned i = 0,
e = VTs.size();
i !=
e; ++
i) {
2674 assert(StoreOperands.empty() &&
"Orphaned operand list.");
2675 StoreOperands.push_back(Chain);
2680 if (ExtendIntegerRetVal) {
2691 StoreOperands.push_back(RetVal);
2696 unsigned NumElts = StoreOperands.size() - 2;
2713 EVT TheStoreType = ExtendIntegerRetVal ?
MVT::i32 : VTs[
i];
2718 StoreOperands.
clear();
2726 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
2728 if (Constraint.length() > 1)
2735 switch (Intrinsic) {
2739 case Intrinsic::nvvm_tex_1d_v4f32_s32:
2741 case Intrinsic::nvvm_tex_1d_v4f32_f32:
2743 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
2745 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
2747 case Intrinsic::nvvm_tex_1d_v4s32_s32:
2749 case Intrinsic::nvvm_tex_1d_v4s32_f32:
2751 case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
2753 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
2755 case Intrinsic::nvvm_tex_1d_v4u32_s32:
2757 case Intrinsic::nvvm_tex_1d_v4u32_f32:
2759 case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
2761 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
2764 case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
2766 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
2768 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
2770 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
2772 case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
2774 case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
2776 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
2778 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
2780 case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
2782 case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
2784 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
2786 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
2789 case Intrinsic::nvvm_tex_2d_v4f32_s32:
2791 case Intrinsic::nvvm_tex_2d_v4f32_f32:
2793 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
2795 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
2797 case Intrinsic::nvvm_tex_2d_v4s32_s32:
2799 case Intrinsic::nvvm_tex_2d_v4s32_f32:
2801 case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
2803 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
2805 case Intrinsic::nvvm_tex_2d_v4u32_s32:
2807 case Intrinsic::nvvm_tex_2d_v4u32_f32:
2809 case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
2811 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
2814 case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
2816 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
2818 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
2820 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
2822 case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
2824 case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
2826 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
2828 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
2830 case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
2832 case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
2834 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
2836 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
2839 case Intrinsic::nvvm_tex_3d_v4f32_s32:
2841 case Intrinsic::nvvm_tex_3d_v4f32_f32:
2843 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
2845 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
2847 case Intrinsic::nvvm_tex_3d_v4s32_s32:
2849 case Intrinsic::nvvm_tex_3d_v4s32_f32:
2851 case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
2853 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
2855 case Intrinsic::nvvm_tex_3d_v4u32_s32:
2857 case Intrinsic::nvvm_tex_3d_v4u32_f32:
2859 case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
2861 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
2864 case Intrinsic::nvvm_tex_cube_v4f32_f32:
2866 case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
2868 case Intrinsic::nvvm_tex_cube_v4s32_f32:
2870 case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
2872 case Intrinsic::nvvm_tex_cube_v4u32_f32:
2874 case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
2877 case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
2879 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
2881 case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
2883 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
2885 case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
2887 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
2890 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
2892 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
2894 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
2896 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
2898 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
2900 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
2902 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
2904 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
2906 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
2908 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
2910 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
2912 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
2915 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
2917 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
2919 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
2921 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
2923 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
2925 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
2927 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
2929 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
2931 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
2933 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
2935 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
2937 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
2940 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
2942 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
2944 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
2946 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
2948 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
2950 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
2952 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
2954 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
2956 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
2958 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
2960 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
2962 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
2965 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
2967 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
2969 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
2971 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
2973 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
2975 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
2977 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
2979 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
2981 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
2983 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
2985 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
2987 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
2990 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
2992 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
2994 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
2996 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
2998 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
3000 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
3002 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
3004 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
3006 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
3008 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
3010 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
3012 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
3015 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
3017 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
3019 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
3021 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
3023 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
3025 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
3027 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
3029 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
3031 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
3033 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
3035 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
3037 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
3040 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
3042 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
3044 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
3046 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
3048 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
3050 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
3053 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
3055 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
3057 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
3059 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
3061 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
3063 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
3066 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
3068 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
3070 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
3072 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
3074 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
3076 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
3078 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
3080 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
3082 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
3084 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
3086 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
3088 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
3094 switch (Intrinsic) {
3097 case Intrinsic::nvvm_suld_1d_i8_clamp:
3099 case Intrinsic::nvvm_suld_1d_i16_clamp:
3101 case Intrinsic::nvvm_suld_1d_i32_clamp:
3103 case Intrinsic::nvvm_suld_1d_i64_clamp:
3105 case Intrinsic::nvvm_suld_1d_v2i8_clamp:
3107 case Intrinsic::nvvm_suld_1d_v2i16_clamp:
3109 case Intrinsic::nvvm_suld_1d_v2i32_clamp:
3111 case Intrinsic::nvvm_suld_1d_v2i64_clamp:
3113 case Intrinsic::nvvm_suld_1d_v4i8_clamp:
3115 case Intrinsic::nvvm_suld_1d_v4i16_clamp:
3117 case Intrinsic::nvvm_suld_1d_v4i32_clamp:
3119 case Intrinsic::nvvm_suld_1d_array_i8_clamp:
3121 case Intrinsic::nvvm_suld_1d_array_i16_clamp:
3123 case Intrinsic::nvvm_suld_1d_array_i32_clamp:
3125 case Intrinsic::nvvm_suld_1d_array_i64_clamp:
3127 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
3129 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
3131 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
3133 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
3135 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
3137 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
3139 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
3141 case Intrinsic::nvvm_suld_2d_i8_clamp:
3143 case Intrinsic::nvvm_suld_2d_i16_clamp:
3145 case Intrinsic::nvvm_suld_2d_i32_clamp:
3147 case Intrinsic::nvvm_suld_2d_i64_clamp:
3149 case Intrinsic::nvvm_suld_2d_v2i8_clamp:
3151 case Intrinsic::nvvm_suld_2d_v2i16_clamp:
3153 case Intrinsic::nvvm_suld_2d_v2i32_clamp:
3155 case Intrinsic::nvvm_suld_2d_v2i64_clamp:
3157 case Intrinsic::nvvm_suld_2d_v4i8_clamp:
3159 case Intrinsic::nvvm_suld_2d_v4i16_clamp:
3161 case Intrinsic::nvvm_suld_2d_v4i32_clamp:
3163 case Intrinsic::nvvm_suld_2d_array_i8_clamp:
3165 case Intrinsic::nvvm_suld_2d_array_i16_clamp:
3167 case Intrinsic::nvvm_suld_2d_array_i32_clamp:
3169 case Intrinsic::nvvm_suld_2d_array_i64_clamp:
3171 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
3173 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
3175 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
3177 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
3179 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
3181 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
3183 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
3185 case Intrinsic::nvvm_suld_3d_i8_clamp:
3187 case Intrinsic::nvvm_suld_3d_i16_clamp:
3189 case Intrinsic::nvvm_suld_3d_i32_clamp:
3191 case Intrinsic::nvvm_suld_3d_i64_clamp:
3193 case Intrinsic::nvvm_suld_3d_v2i8_clamp:
3195 case Intrinsic::nvvm_suld_3d_v2i16_clamp:
3197 case Intrinsic::nvvm_suld_3d_v2i32_clamp:
3199 case Intrinsic::nvvm_suld_3d_v2i64_clamp:
3201 case Intrinsic::nvvm_suld_3d_v4i8_clamp:
3203 case Intrinsic::nvvm_suld_3d_v4i16_clamp:
3205 case Intrinsic::nvvm_suld_3d_v4i32_clamp:
3207 case Intrinsic::nvvm_suld_1d_i8_trap:
3209 case Intrinsic::nvvm_suld_1d_i16_trap:
3211 case Intrinsic::nvvm_suld_1d_i32_trap:
3213 case Intrinsic::nvvm_suld_1d_i64_trap:
3215 case Intrinsic::nvvm_suld_1d_v2i8_trap:
3217 case Intrinsic::nvvm_suld_1d_v2i16_trap:
3219 case Intrinsic::nvvm_suld_1d_v2i32_trap:
3221 case Intrinsic::nvvm_suld_1d_v2i64_trap:
3223 case Intrinsic::nvvm_suld_1d_v4i8_trap:
3225 case Intrinsic::nvvm_suld_1d_v4i16_trap:
3227 case Intrinsic::nvvm_suld_1d_v4i32_trap:
3229 case Intrinsic::nvvm_suld_1d_array_i8_trap:
3231 case Intrinsic::nvvm_suld_1d_array_i16_trap:
3233 case Intrinsic::nvvm_suld_1d_array_i32_trap:
3235 case Intrinsic::nvvm_suld_1d_array_i64_trap:
3237 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
3239 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
3241 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
3243 case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
3245 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
3247 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
3249 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
3251 case Intrinsic::nvvm_suld_2d_i8_trap:
3253 case Intrinsic::nvvm_suld_2d_i16_trap:
3255 case Intrinsic::nvvm_suld_2d_i32_trap:
3257 case Intrinsic::nvvm_suld_2d_i64_trap:
3259 case Intrinsic::nvvm_suld_2d_v2i8_trap:
3261 case Intrinsic::nvvm_suld_2d_v2i16_trap:
3263 case Intrinsic::nvvm_suld_2d_v2i32_trap:
3265 case Intrinsic::nvvm_suld_2d_v2i64_trap:
3267 case Intrinsic::nvvm_suld_2d_v4i8_trap:
3269 case Intrinsic::nvvm_suld_2d_v4i16_trap:
3271 case Intrinsic::nvvm_suld_2d_v4i32_trap:
3273 case Intrinsic::nvvm_suld_2d_array_i8_trap:
3275 case Intrinsic::nvvm_suld_2d_array_i16_trap:
3277 case Intrinsic::nvvm_suld_2d_array_i32_trap:
3279 case Intrinsic::nvvm_suld_2d_array_i64_trap:
3281 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
3283 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
3285 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
3287 case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
3289 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
3291 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
3293 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
3295 case Intrinsic::nvvm_suld_3d_i8_trap:
3297 case Intrinsic::nvvm_suld_3d_i16_trap:
3299 case Intrinsic::nvvm_suld_3d_i32_trap:
3301 case Intrinsic::nvvm_suld_3d_i64_trap:
3303 case Intrinsic::nvvm_suld_3d_v2i8_trap:
3305 case Intrinsic::nvvm_suld_3d_v2i16_trap:
3307 case Intrinsic::nvvm_suld_3d_v2i32_trap:
3309 case Intrinsic::nvvm_suld_3d_v2i64_trap:
3311 case Intrinsic::nvvm_suld_3d_v4i8_trap:
3313 case Intrinsic::nvvm_suld_3d_v4i16_trap:
3315 case Intrinsic::nvvm_suld_3d_v4i32_trap:
3317 case Intrinsic::nvvm_suld_1d_i8_zero:
3319 case Intrinsic::nvvm_suld_1d_i16_zero:
3321 case Intrinsic::nvvm_suld_1d_i32_zero:
3323 case Intrinsic::nvvm_suld_1d_i64_zero:
3325 case Intrinsic::nvvm_suld_1d_v2i8_zero:
3327 case Intrinsic::nvvm_suld_1d_v2i16_zero:
3329 case Intrinsic::nvvm_suld_1d_v2i32_zero:
3331 case Intrinsic::nvvm_suld_1d_v2i64_zero:
3333 case Intrinsic::nvvm_suld_1d_v4i8_zero:
3335 case Intrinsic::nvvm_suld_1d_v4i16_zero:
3337 case Intrinsic::nvvm_suld_1d_v4i32_zero:
3339 case Intrinsic::nvvm_suld_1d_array_i8_zero:
3341 case Intrinsic::nvvm_suld_1d_array_i16_zero:
3343 case Intrinsic::nvvm_suld_1d_array_i32_zero:
3345 case Intrinsic::nvvm_suld_1d_array_i64_zero:
3347 case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
3349 case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
3351 case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
3353 case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
3355 case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
3357 case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
3359 case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
3361 case Intrinsic::nvvm_suld_2d_i8_zero:
3363 case Intrinsic::nvvm_suld_2d_i16_zero:
3365 case Intrinsic::nvvm_suld_2d_i32_zero:
3367 case Intrinsic::nvvm_suld_2d_i64_zero:
3369 case Intrinsic::nvvm_suld_2d_v2i8_zero:
3371 case Intrinsic::nvvm_suld_2d_v2i16_zero:
3373 case Intrinsic::nvvm_suld_2d_v2i32_zero:
3375 case Intrinsic::nvvm_suld_2d_v2i64_zero:
3377 case Intrinsic::nvvm_suld_2d_v4i8_zero:
3379 case Intrinsic::nvvm_suld_2d_v4i16_zero:
3381 case Intrinsic::nvvm_suld_2d_v4i32_zero:
3383 case Intrinsic::nvvm_suld_2d_array_i8_zero:
3385 case Intrinsic::nvvm_suld_2d_array_i16_zero:
3387 case Intrinsic::nvvm_suld_2d_array_i32_zero:
3389 case Intrinsic::nvvm_suld_2d_array_i64_zero:
3391 case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
3393 case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
3395 case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
3397 case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
3399 case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
3401 case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
3403 case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
3405 case Intrinsic::nvvm_suld_3d_i8_zero:
3407 case Intrinsic::nvvm_suld_3d_i16_zero:
3409 case Intrinsic::nvvm_suld_3d_i32_zero:
3411 case Intrinsic::nvvm_suld_3d_i64_zero:
3413 case Intrinsic::nvvm_suld_3d_v2i8_zero:
3415 case Intrinsic::nvvm_suld_3d_v2i16_zero:
3417 case Intrinsic::nvvm_suld_3d_v2i32_zero:
3419 case Intrinsic::nvvm_suld_3d_v2i64_zero:
3421 case Intrinsic::nvvm_suld_3d_v4i8_zero:
3423 case Intrinsic::nvvm_suld_3d_v4i16_zero:
3425 case Intrinsic::nvvm_suld_3d_v4i32_zero:
3438 switch (Intrinsic) {
3441 case Intrinsic::nvvm_match_all_sync_i32p:
3442 case Intrinsic::nvvm_match_all_sync_i64p:
3452 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col:
3453 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row:
3454 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride:
3455 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride:
3456 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col:
3457 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row:
3458 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride:
3459 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride:
3460 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col:
3461 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row:
3462 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride:
3463 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride:
3464 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col:
3465 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row:
3466 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride:
3467 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride:
3468 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col:
3469 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row:
3470 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride:
3471 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride:
3472 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col:
3473 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row:
3474 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride:
3475 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride: {
3478 Info.ptrVal =
I.getArgOperand(0);
3484 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col:
3485 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col_stride:
3486 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col_stride:
3487 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col:
3488 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row:
3489 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row_stride:
3490 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row_stride:
3491 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row:
3492 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col:
3493 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col_stride:
3494 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col_stride:
3495 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col:
3496 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row:
3497 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row_stride:
3498 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row_stride:
3499 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row: {
3502 Info.ptrVal =
I.getArgOperand(0);
3509 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col:
3510 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col_stride:
3511 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col_stride:
3512 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col:
3513 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row:
3514 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row_stride:
3515 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row_stride:
3516 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row:
3518 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col:
3519 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col_stride:
3520 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col_stride:
3521 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col:
3522 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row:
3523 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row_stride:
3524 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row_stride:
3525 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row: {
3528 Info.ptrVal =
I.getArgOperand(0);
3535 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col:
3536 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col_stride:
3537 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col_stride:
3538 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col:
3539 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row:
3540 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row_stride:
3541 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row_stride:
3542 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row:
3544 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col:
3545 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col_stride:
3546 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col_stride:
3547 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col:
3548 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row:
3549 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row_stride:
3550 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row_stride:
3551 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row:
3552 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row:
3553 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row_stride:
3554 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col:
3555 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col_stride:
3556 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row:
3557 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row_stride:
3558 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row_stride:
3559 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row:
3560 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col:
3561 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col_stride:
3562 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col_stride:
3563 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col: {
3566 Info.ptrVal =
I.getArgOperand(0);
3573 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col:
3574 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row:
3575 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride:
3576 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride:
3577 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col:
3578 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row:
3579 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride:
3580 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride:
3581 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col:
3582 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row:
3583 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride:
3584 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride: {
3587 Info.ptrVal =
I.getArgOperand(0);
3594 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col:
3595 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row:
3596 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride:
3597 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride:
3598 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col:
3599 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row:
3600 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride:
3601 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride:
3602 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col:
3603 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row:
3604 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride:
3605 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride: {
3608 Info.ptrVal =
I.getArgOperand(0);
3615 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col:
3616 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col_stride:
3617 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row:
3618 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row_stride:
3619 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col:
3620 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col_stride:
3621 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row:
3622 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row_stride:
3623 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col:
3624 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col_stride:
3625 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row:
3626 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row_stride: {
3629 Info.ptrVal =
I.getArgOperand(0);
3636 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col:
3637 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col_stride:
3638 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row:
3639 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row_stride:
3640 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col:
3641 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col_stride:
3642 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row:
3643 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row_stride: {
3646 Info.ptrVal =
I.getArgOperand(0);
3653 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col:
3654 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row:
3655 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride:
3656 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride:
3657 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col:
3658 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row:
3659 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride:
3660 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride:
3661 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col:
3662 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row:
3663 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride:
3664 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride: {
3667 Info.ptrVal =
I.getArgOperand(0);
3674 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col:
3675 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row:
3676 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride:
3677 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride:
3678 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col:
3679 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row:
3680 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride:
3681 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride:
3682 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col:
3683 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row:
3684 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride:
3685 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride: {
3688 Info.ptrVal =
I.getArgOperand(0);
3695 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col:
3696 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col_stride:
3697 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row:
3698 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row_stride:
3699 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col:
3700 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col_stride:
3701 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row:
3702 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row_stride:
3703 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col:
3704 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col_stride:
3705 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row:
3706 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row_stride: {
3709 Info.ptrVal =
I.getArgOperand(0);
3716 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col:
3717 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col_stride:
3718 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row:
3719 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row_stride:
3720 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col:
3721 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col_stride:
3722 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row:
3723 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row_stride: {
3726 Info.ptrVal =
I.getArgOperand(0);
3733 case Intrinsic::nvvm_atomic_load_inc_32:
3734 case Intrinsic::nvvm_atomic_load_dec_32:
3736 case Intrinsic::nvvm_atomic_add_gen_f_cta:
3737 case Intrinsic::nvvm_atomic_add_gen_f_sys:
3738 case Intrinsic::nvvm_atomic_add_gen_i_cta:
3739 case Intrinsic::nvvm_atomic_add_gen_i_sys:
3740 case Intrinsic::nvvm_atomic_and_gen_i_cta:
3741 case Intrinsic::nvvm_atomic_and_gen_i_sys:
3742 case Intrinsic::nvvm_atomic_cas_gen_i_cta:
3743 case Intrinsic::nvvm_atomic_cas_gen_i_sys:
3744 case Intrinsic::nvvm_atomic_dec_gen_i_cta:
3745 case Intrinsic::nvvm_atomic_dec_gen_i_sys:
3746 case Intrinsic::nvvm_atomic_inc_gen_i_cta:
3747 case Intrinsic::nvvm_atomic_inc_gen_i_sys:
3748 case Intrinsic::nvvm_atomic_max_gen_i_cta:
3749 case Intrinsic::nvvm_atomic_max_gen_i_sys:
3750 case Intrinsic::nvvm_atomic_min_gen_i_cta:
3751 case Intrinsic::nvvm_atomic_min_gen_i_sys:
3752 case Intrinsic::nvvm_atomic_or_gen_i_cta:
3753 case Intrinsic::nvvm_atomic_or_gen_i_sys:
3754 case Intrinsic::nvvm_atomic_exch_gen_i_cta:
3755 case Intrinsic::nvvm_atomic_exch_gen_i_sys:
3756 case Intrinsic::nvvm_atomic_xor_gen_i_cta:
3757 case Intrinsic::nvvm_atomic_xor_gen_i_sys: {
3758 auto &
DL =
I.getModule()->getDataLayout();
3761 Info.ptrVal =
I.getArgOperand(0);
3768 case Intrinsic::nvvm_ldu_global_i:
3769 case Intrinsic::nvvm_ldu_global_f:
3770 case Intrinsic::nvvm_ldu_global_p: {
3771 auto &
DL =
I.getModule()->getDataLayout();
3773 if (Intrinsic == Intrinsic::nvvm_ldu_global_i)
3775 else if(Intrinsic == Intrinsic::nvvm_ldu_global_p)
3779 Info.ptrVal =
I.getArgOperand(0);
3782 Info.align = cast<ConstantInt>(
I.getArgOperand(1))->getMaybeAlignValue();
3786 case Intrinsic::nvvm_ldg_global_i:
3787 case Intrinsic::nvvm_ldg_global_f:
3788 case Intrinsic::nvvm_ldg_global_p: {
3789 auto &
DL =
I.getModule()->getDataLayout();
3792 if (Intrinsic == Intrinsic::nvvm_ldg_global_i)
3794 else if(Intrinsic == Intrinsic::nvvm_ldg_global_p)
3798 Info.ptrVal =
I.getArgOperand(0);
3801 Info.align = cast<ConstantInt>(
I.getArgOperand(1))->getMaybeAlignValue();
3806 case Intrinsic::nvvm_tex_1d_v4f32_s32:
3807 case Intrinsic::nvvm_tex_1d_v4f32_f32:
3808 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
3809 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
3810 case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
3811 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
3812 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
3813 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
3814 case Intrinsic::nvvm_tex_2d_v4f32_s32:
3815 case Intrinsic::nvvm_tex_2d_v4f32_f32:
3816 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
3817 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
3818 case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
3819 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
3820 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
3821 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
3822 case Intrinsic::nvvm_tex_3d_v4f32_s32:
3823 case Intrinsic::nvvm_tex_3d_v4f32_f32:
3824 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
3825 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
3826 case Intrinsic::nvvm_tex_cube_v4f32_f32:
3827 case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
3828 case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
3829 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
3830 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
3831 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
3832 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
3833 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
3834 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
3835 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
3836 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
3837 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
3838 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
3839 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
3840 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
3841 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
3842 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
3843 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
3844 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
3845 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
3846 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
3847 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
3848 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
3849 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
3850 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
3851 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
3852 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
3853 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
3854 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
3855 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
3856 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
3857 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
3858 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
3859 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
3860 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
3861 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
3864 Info.ptrVal =
nullptr;
3870 case Intrinsic::nvvm_tex_1d_v4s32_s32:
3871 case Intrinsic::nvvm_tex_1d_v4s32_f32:
3872 case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
3873 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
3874 case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
3875 case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
3876 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
3877 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
3878 case Intrinsic::nvvm_tex_2d_v4s32_s32:
3879 case Intrinsic::nvvm_tex_2d_v4s32_f32:
3880 case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
3881 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
3882 case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
3883 case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
3884 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
3885 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
3886 case Intrinsic::nvvm_tex_3d_v4s32_s32:
3887 case Intrinsic::nvvm_tex_3d_v4s32_f32:
3888 case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
3889 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
3890 case Intrinsic::nvvm_tex_cube_v4s32_f32:
3891 case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
3892 case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
3893 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
3894 case Intrinsic::nvvm_tex_cube_v4u32_f32:
3895 case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
3896 case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
3897 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
3898 case Intrinsic::nvvm_tex_1d_v4u32_s32:
3899 case Intrinsic::nvvm_tex_1d_v4u32_f32:
3900 case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
3901 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
3902 case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
3903 case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
3904 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
3905 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
3906 case Intrinsic::nvvm_tex_2d_v4u32_s32:
3907 case Intrinsic::nvvm_tex_2d_v4u32_f32:
3908 case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
3909 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
3910 case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
3911 case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
3912 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
3913 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
3914 case Intrinsic::nvvm_tex_3d_v4u32_s32:
3915 case Intrinsic::nvvm_tex_3d_v4u32_f32:
3916 case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
3917 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
3918 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
3919 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
3920 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
3921 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
3922 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
3923 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
3924 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
3925 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
3926 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
3927 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
3928 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
3929 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
3930 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
3931 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
3932 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
3933 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
3934 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
3935 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
3936 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
3937 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
3938 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
3939 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
3940 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
3941 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
3942 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
3943 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
3944 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
3945 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
3946 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
3947 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
3948 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
3949 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
3950 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
3951 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
3952 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
3953 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
3954 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
3955 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
3956 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
3957 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
3958 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
3959 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
3960 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
3961 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
3962 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
3963 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
3964 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
3965 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
3966 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
3967 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
3968 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
3969 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
3970 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
3971 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
3972 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
3973 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
3974 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
3975 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
3976 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
3977 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
3978 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
3979 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
3980 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
3981 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
3984 Info.ptrVal =
nullptr;
3990 case Intrinsic::nvvm_suld_1d_i8_clamp:
3991 case Intrinsic::nvvm_suld_1d_v2i8_clamp:
3992 case Intrinsic::nvvm_suld_1d_v4i8_clamp:
3993 case Intrinsic::nvvm_suld_1d_array_i8_clamp:
3994 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
3995 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
3996 case Intrinsic::nvvm_suld_2d_i8_clamp:
3997 case Intrinsic::nvvm_suld_2d_v2i8_clamp:
3998 case Intrinsic::nvvm_suld_2d_v4i8_clamp:
3999 case Intrinsic::nvvm_suld_2d_array_i8_clamp:
4000 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
4001 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
4002 case Intrinsic::nvvm_suld_3d_i8_clamp:
4003 case Intrinsic::nvvm_suld_3d_v2i8_clamp:
4004 case Intrinsic::nvvm_suld_3d_v4i8_clamp:
4005 case Intrinsic::nvvm_suld_1d_i8_trap:
4006 case Intrinsic::nvvm_suld_1d_v2i8_trap:
4007 case Intrinsic::nvvm_suld_1d_v4i8_trap:
4008 case Intrinsic::nvvm_suld_1d_array_i8_trap:
4009 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
4010 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
4011 case Intrinsic::nvvm_suld_2d_i8_trap:
4012 case Intrinsic::nvvm_suld_2d_v2i8_trap:
4013 case Intrinsic::nvvm_suld_2d_v4i8_trap:
4014 case Intrinsic::nvvm_suld_2d_array_i8_trap:
4015 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
4016 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
4017 case Intrinsic::nvvm_suld_3d_i8_trap:
4018 case Intrinsic::nvvm_suld_3d_v2i8_trap:
4019 case Intrinsic::nvvm_suld_3d_v4i8_trap:
4020 case Intrinsic::nvvm_suld_1d_i8_zero:
4021 case Intrinsic::nvvm_suld_1d_v2i8_zero:
4022 case Intrinsic::nvvm_suld_1d_v4i8_zero:
4023 case Intrinsic::nvvm_suld_1d_array_i8_zero:
4024 case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
4025 case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
4026 case Intrinsic::nvvm_suld_2d_i8_zero:
4027 case Intrinsic::nvvm_suld_2d_v2i8_zero:
4028 case Intrinsic::nvvm_suld_2d_v4i8_zero:
4029 case Intrinsic::nvvm_suld_2d_array_i8_zero:
4030 case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
4031 case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
4032 case Intrinsic::nvvm_suld_3d_i8_zero:
4033 case Intrinsic::nvvm_suld_3d_v2i8_zero:
4034 case Intrinsic::nvvm_suld_3d_v4i8_zero:
4037 Info.ptrVal =
nullptr;
4043 case Intrinsic::nvvm_suld_1d_i16_clamp:
4044 case Intrinsic::nvvm_suld_1d_v2i16_clamp:
4045 case Intrinsic::nvvm_suld_1d_v4i16_clamp:
4046 case Intrinsic::nvvm_suld_1d_array_i16_clamp:
4047 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
4048 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
4049 case Intrinsic::nvvm_suld_2d_i16_clamp:
4050 case Intrinsic::nvvm_suld_2d_v2i16_clamp:
4051 case Intrinsic::nvvm_suld_2d_v4i16_clamp:
4052 case Intrinsic::nvvm_suld_2d_array_i16_clamp:
4053 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
4054 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
4055 case Intrinsic::nvvm_suld_3d_i16_clamp:
4056 case Intrinsic::nvvm_suld_3d_v2i16_clamp:
4057 case Intrinsic::nvvm_suld_3d_v4i16_clamp:
4058 case Intrinsic::nvvm_suld_1d_i16_trap:
4059 case Intrinsic::nvvm_suld_1d_v2i16_trap:
4060 case Intrinsic::nvvm_suld_1d_v4i16_trap:
4061 case Intrinsic::nvvm_suld_1d_array_i16_trap:
4062 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
4063 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
4064 case Intrinsic::nvvm_suld_2d_i16_trap:
4065 case Intrinsic::nvvm_suld_2d_v2i16_trap:
4066 case Intrinsic::nvvm_suld_2d_v4i16_trap:
4067 case Intrinsic::nvvm_suld_2d_array_i16_trap:
4068 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
4069 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
4070 case Intrinsic::nvvm_suld_3d_i16_trap:
4071 case Intrinsic::nvvm_suld_3d_v2i16_trap:
4072 case Intrinsic::nvvm_suld_3d_v4i16_trap:
4073 case Intrinsic::nvvm_suld_1d_i16_zero:
4074 case Intrinsic::nvvm_suld_1d_v2i16_zero:
4075 case Intrinsic::nvvm_suld_1d_v4i16_zero:
4076 case Intrinsic::nvvm_suld_1d_array_i16_zero:
4077 case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
4078 case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
4079 case Intrinsic::nvvm_suld_2d_i16_zero:
4080 case Intrinsic::nvvm_suld_2d_v2i16_zero:
4081 case Intrinsic::nvvm_suld_2d_v4i16_zero:
4082 case Intrinsic::nvvm_suld_2d_array_i16_zero:
4083 case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
4084 case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
4085 case Intrinsic::nvvm_suld_3d_i16_zero:
4086 case Intrinsic::nvvm_suld_3d_v2i16_zero:
4087 case Intrinsic::nvvm_suld_3d_v4i16_zero:
4090 Info.ptrVal =
nullptr;
4096 case Intrinsic::nvvm_suld_1d_i32_clamp:
4097 case Intrinsic::nvvm_suld_1d_v2i32_clamp:
4098 case Intrinsic::nvvm_suld_1d_v4i32_clamp:
4099 case Intrinsic::nvvm_suld_1d_array_i32_clamp:
4100 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
4101 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
4102 case Intrinsic::nvvm_suld_2d_i32_clamp:
4103 case Intrinsic::nvvm_suld_2d_v2i32_clamp:
4104 case Intrinsic::nvvm_suld_2d_v4i32_clamp:
4105 case Intrinsic::nvvm_suld_2d_array_i32_clamp:
4106 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
4107 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
4108 case Intrinsic::nvvm_suld_3d_i32_clamp:
4109 case Intrinsic::nvvm_suld_3d_v2i32_clamp:
4110 case Intrinsic::nvvm_suld_3d_v4i32_clamp:
4111 case Intrinsic::nvvm_suld_1d_i32_trap:
4112 case Intrinsic::nvvm_suld_1d_v2i32_trap:
4113 case Intrinsic::nvvm_suld_1d_v4i32_trap:
4114 case Intrinsic::nvvm_suld_1d_array_i32_trap:
4115 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
4116 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
4117 case Intrinsic::nvvm_suld_2d_i32_trap:
4118 case Intrinsic::nvvm_suld_2d_v2i32_trap:
4119 case Intrinsic::nvvm_suld_2d_v4i32_trap:
4120 case Intrinsic::nvvm_suld_2d_array_i32_trap:
4121 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
4122 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
4123 case Intrinsic::nvvm_suld_3d_i32_trap:
4124 case Intrinsic::nvvm_suld_3d_v2i32_trap:
4125 case Intrinsic::nvvm_suld_3d_v4i32_trap:
4126 case Intrinsic::nvvm_suld_1d_i32_zero:
4127 case Intrinsic::nvvm_suld_1d_v2i32_zero:
4128 case Intrinsic::nvvm_suld_1d_v4i32_zero:
4129 case Intrinsic::nvvm_suld_1d_array_i32_zero:
4130 case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
4131 case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
4132 case Intrinsic::nvvm_suld_2d_i32_zero:
4133 case Intrinsic::nvvm_suld_2d_v2i32_zero:
4134 case Intrinsic::nvvm_suld_2d_v4i32_zero:
4135 case Intrinsic::nvvm_suld_2d_array_i32_zero:
4136 case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
4137 case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
4138 case Intrinsic::nvvm_suld_3d_i32_zero:
4139 case Intrinsic::nvvm_suld_3d_v2i32_zero:
4140 case Intrinsic::nvvm_suld_3d_v4i32_zero:
4143 Info.ptrVal =
nullptr;
4149 case Intrinsic::nvvm_suld_1d_i64_clamp:
4150 case Intrinsic::nvvm_suld_1d_v2i64_clamp:
4151 case Intrinsic::nvvm_suld_1d_array_i64_clamp:
4152 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
4153 case Intrinsic::nvvm_suld_2d_i64_clamp:
4154 case Intrinsic::nvvm_suld_2d_v2i64_clamp:
4155 case Intrinsic::nvvm_suld_2d_array_i64_clamp:
4156 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
4157 case Intrinsic::nvvm_suld_3d_i64_clamp:
4158 case Intrinsic::nvvm_suld_3d_v2i64_clamp:
4159 case Intrinsic::nvvm_suld_1d_i64_trap:
4160 case Intrinsic::nvvm_suld_1d_v2i64_trap:
4161 case Intrinsic::nvvm_suld_1d_array_i64_trap:
4162 case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
4163 case Intrinsic::nvvm_suld_2d_i64_trap:
4164 case Intrinsic::nvvm_suld_2d_v2i64_trap:
4165 case Intrinsic::nvvm_suld_2d_array_i64_trap:
4166 case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
4167 case Intrinsic::nvvm_suld_3d_i64_trap:
4168 case Intrinsic::nvvm_suld_3d_v2i64_trap:
4169 case Intrinsic::nvvm_suld_1d_i64_zero:
4170 case Intrinsic::nvvm_suld_1d_v2i64_zero:
4171 case Intrinsic::nvvm_suld_1d_array_i64_zero:
4172 case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
4173 case Intrinsic::nvvm_suld_2d_i64_zero:
4174 case Intrinsic::nvvm_suld_2d_v2i64_zero:
4175 case Intrinsic::nvvm_suld_2d_array_i64_zero:
4176 case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
4177 case Intrinsic::nvvm_suld_3d_i64_zero:
4178 case Intrinsic::nvvm_suld_3d_v2i64_zero:
4181 Info.ptrVal =
nullptr;
4234 if (Constraint.
size() == 1) {
4235 switch (Constraint[0]) {
4253 std::pair<unsigned, const TargetRegisterClass *>
4257 if (Constraint.
size() == 1) {
4258 switch (Constraint[0]) {
4260 return std::make_pair(0U, &NVPTX::Int1RegsRegClass);
4262 return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
4264 return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
4266 return std::make_pair(0U, &NVPTX::Int32RegsRegClass);
4269 return std::make_pair(0U, &NVPTX::Int64RegsRegClass);
4271 return std::make_pair(0U, &NVPTX::Float32RegsRegClass);
4273 return std::make_pair(0U, &NVPTX::Float64RegsRegClass);
4307 return F.getFnAttribute(
"unsafe-fp-math").getValueAsBool();
4356 int nonAddCount = 0;
4368 int orderNo =
N->getIROrder();
4374 if (orderNo - orderNo2 < 500)
4379 bool opIsLive =
false;
4383 if (isa<ConstantSDNode>(left) || isa<ConstantSDNode>(
right))
4389 int orderNo3 =
User->getIROrder();
4390 if (orderNo3 > orderNo) {
4399 int orderNo3 =
User->getIROrder();
4400 if (orderNo3 > orderNo) {
4446 if (isa<ConstantSDNode>(Val)) {
4470 if (MaskVal != 0xff) {
4475 MemSDNode *Mem = dyn_cast<MemSDNode>(Val);
4497 if (AExt.
getNode() !=
nullptr) {