84#include "llvm/IR/IntrinsicsARM.h"
122#define DEBUG_TYPE "arm-isel"
125STATISTIC(NumMovwMovt,
"Number of GAs materialized with movw + movt");
126STATISTIC(NumLoopByVals,
"Number of loops generated for byval arguments");
128 "Number of constants with their storage promoted into constant pools");
132 cl::desc(
"Enable / disable ARM interworking (for debugging only)"),
137 cl::desc(
"Enable / disable promotion of unnamed_addr constants into "
142 cl::desc(
"Maximum size of constant to promote into a constant pool"),
146 cl::desc(
"Maximum size of ALL constants to promote into a constant pool"),
151 cl::desc(
"Maximum interleave factor for MVE VLDn to generate."),
156 ARM::R0, ARM::R1, ARM::R2, ARM::R3
159void ARMTargetLowering::addTypeForNEON(
MVT VT,
MVT PromotedLdStVT) {
160 if (VT != PromotedLdStVT) {
169 if (ElemTy != MVT::f64)
173 if (ElemTy == MVT::i32) {
209 VT != MVT::v2i64 && VT != MVT::v1i64)
217void ARMTargetLowering::addDRTypeForNEON(
MVT VT) {
219 addTypeForNEON(VT, MVT::f64);
222void ARMTargetLowering::addQRTypeForNEON(
MVT VT) {
224 addTypeForNEON(VT, MVT::v2f64);
227void ARMTargetLowering::setAllExpand(
MVT VT) {
240void ARMTargetLowering::addAllExtLoads(
const MVT From,
const MVT To,
241 LegalizeAction Action) {
247void ARMTargetLowering::addMVEVectorTypes(
bool HasMVEFP) {
248 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
250 for (
auto VT : IntTypes) {
324 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
325 for (
auto VT : FloatTypes) {
393 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
394 for (
auto VT : LongTypes) {
411 addAllExtLoads(MVT::v8i16, MVT::v8i8,
Legal);
412 addAllExtLoads(MVT::v4i32, MVT::v4i16,
Legal);
413 addAllExtLoads(MVT::v4i32, MVT::v4i8,
Legal);
430 for (
auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
439 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
440 for (
auto VT : pTypes) {
496 for (
int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
504 if (Subtarget->isThumb() && Subtarget->
hasVFP2Base() &&
505 Subtarget->
hasARMOps() && !Subtarget->useSoftFloat()) {
506 static const struct {
508 const char *
const Name;
530 { RTLIB::UO_F32,
"__unordsf2vfp",
ISD::SETNE },
539 { RTLIB::UO_F64,
"__unorddf2vfp",
ISD::SETNE },
564 for (
const auto &LC : LibraryCalls) {
584 static const struct {
586 const char *
const Name;
671 for (
const auto &LC : LibraryCalls) {
681 static const struct {
683 const char *
const Name;
686 } MemOpsLibraryCalls[] = {
694 for (
const auto &LC : MemOpsLibraryCalls) {
704 static const struct {
706 const char *
const Name;
719 for (
const auto &LC : LibraryCalls) {
751 static const struct {
753 const char *
const Name;
761 for (
const auto &LC : LibraryCalls) {
772 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only() &&
773 Subtarget->hasFPRegs()) {
783 setAllExpand(MVT::f32);
784 if (!Subtarget->hasFP64())
785 setAllExpand(MVT::f64);
788 if (Subtarget->hasFullFP16()) {
797 if (Subtarget->hasBF16()) {
799 setAllExpand(MVT::bf16);
800 if (!Subtarget->hasFullFP16())
807 addAllExtLoads(VT, InnerVT,
Expand);
822 if (Subtarget->hasMVEIntegerOps())
823 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
826 if (Subtarget->hasLOB()) {
830 if (Subtarget->hasNEON()) {
831 addDRTypeForNEON(MVT::v2f32);
832 addDRTypeForNEON(MVT::v8i8);
833 addDRTypeForNEON(MVT::v4i16);
834 addDRTypeForNEON(MVT::v2i32);
835 addDRTypeForNEON(MVT::v1i64);
837 addQRTypeForNEON(MVT::v4f32);
838 addQRTypeForNEON(MVT::v2f64);
839 addQRTypeForNEON(MVT::v16i8);
840 addQRTypeForNEON(MVT::v8i16);
841 addQRTypeForNEON(MVT::v4i32);
842 addQRTypeForNEON(MVT::v2i64);
844 if (Subtarget->hasFullFP16()) {
845 addQRTypeForNEON(MVT::v8f16);
846 addDRTypeForNEON(MVT::v4f16);
849 if (Subtarget->hasBF16()) {
850 addQRTypeForNEON(MVT::v8bf16);
851 addDRTypeForNEON(MVT::v4bf16);
855 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
894 if (Subtarget->hasNEON()) {
1006 for (
MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
1015 for (
auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
1024 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
1032 if (Subtarget->hasMVEIntegerOps()) {
1037 if (Subtarget->hasMVEFloatOps()) {
1041 if (!Subtarget->hasFP64()) {
1088 if (Subtarget->hasFullFP16()) {
1094 if (!Subtarget->hasFP16()) {
1142 if (Subtarget->hasDSP()) {
1164 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6Ops()
1165 || (Subtarget->
isThumb2() && !Subtarget->hasDSP()))
1180 if (Subtarget->hasMVEIntegerOps())
1190 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops())
1201 if (!Subtarget->hasV5TOps() || Subtarget->
isThumb1Only()) {
1210 if (Subtarget->hasPerfMon())
1214 if (!Subtarget->hasV6Ops())
1217 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1218 : Subtarget->hasDivideInARMMode();
1225 if (Subtarget->
isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
1242 HasStandaloneRem =
false;
1247 const char *
const Name;
1249 } LibraryCalls[] = {
1261 for (
const auto &LC : LibraryCalls) {
1268 const char *
const Name;
1270 } LibraryCalls[] = {
1282 for (
const auto &LC : LibraryCalls) {
1326 InsertFencesForAtomic =
false;
1328 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1332 if (!Subtarget->isThumb() || !Subtarget->
isMClass())
1337 if (!Subtarget->hasAcquireRelease() ||
1340 InsertFencesForAtomic =
true;
1346 if (Subtarget->hasDataBarrier())
1347 InsertFencesForAtomic =
true;
1367 if (!InsertFencesForAtomic) {
1375 (!Subtarget->
isMClass() && Subtarget->hasV6Ops())) {
1387 }
else if ((Subtarget->
isMClass() && Subtarget->hasV8MBaselineOps()) ||
1388 Subtarget->hasForced32BitAtomics()) {
1402 if (!Subtarget->hasV6Ops()) {
1408 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1440 if (Subtarget->hasFullFP16()) {
1450 if (Subtarget->hasFullFP16())
1465 if (!Subtarget->useSoftFloat() && Subtarget->
hasVFP2Base() &&
1479 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only()) {
1487 if (!Subtarget->hasFP16()) {
1518 if (Subtarget->hasNEON()) {
1525 if (Subtarget->hasFP64()) {
1538 if (Subtarget->hasFullFP16()) {
1556 if (Subtarget->hasNEON()) {
1572 if (Subtarget->hasFullFP16()) {
1590 if (Subtarget->hasMVEIntegerOps())
1593 if (Subtarget->hasV6Ops())
1598 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1605 if (Subtarget->useSoftFloat() || Subtarget->
isThumb1Only() ||
1631 if (Subtarget->isThumb() || Subtarget->
isThumb2())
1636 return Subtarget->useSoftFloat();
1649std::pair<const TargetRegisterClass *, uint8_t>
1660 case MVT::f32:
case MVT::f64:
case MVT::v8i8:
case MVT::v4i16:
1661 case MVT::v2i32:
case MVT::v1i64:
case MVT::v2f32:
1662 RRC = &ARM::DPRRegClass;
1670 case MVT::v16i8:
case MVT::v8i16:
case MVT::v4i32:
case MVT::v2i64:
1671 case MVT::v4f32:
case MVT::v2f64:
1672 RRC = &ARM::DPRRegClass;
1676 RRC = &ARM::DPRRegClass;
1680 RRC = &ARM::DPRRegClass;
1684 return std::make_pair(RRC,
Cost);
1688#define MAKE_CASE(V) \
1908 if ((Subtarget->hasMVEIntegerOps() &&
1909 (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
1910 VT == MVT::v16i8)) ||
1911 (Subtarget->hasMVEFloatOps() &&
1912 (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16)))
1926 if (Subtarget->hasNEON()) {
1927 if (VT == MVT::v4i64)
1928 return &ARM::QQPRRegClass;
1929 if (VT == MVT::v8i64)
1930 return &ARM::QQQQPRRegClass;
1932 if (Subtarget->hasMVEIntegerOps()) {
1933 if (VT == MVT::v4i64)
1934 return &ARM::MQQPRRegClass;
1935 if (VT == MVT::v8i64)
1936 return &ARM::MQQQQPRRegClass;
1945 Align &PrefAlign)
const {
1946 if (!isa<MemIntrinsic>(CI))
1964 unsigned NumVals =
N->getNumValues();
1968 for (
unsigned i = 0; i != NumVals; ++i) {
1969 EVT VT =
N->getValueType(i);
1970 if (VT == MVT::Glue || VT == MVT::Other)
1976 if (!
N->isMachineOpcode())
2000 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2001 return Const->getZExtValue() == 16;
2008 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2009 return Const->getZExtValue() == 16;
2016 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2017 return Const->getZExtValue() == 16;
2086 bool isVarArg)
const {
2107 else if (Subtarget->hasFPRegs() && !Subtarget->
isThumb1Only() &&
2128 bool isVarArg)
const {
2129 return CCAssignFnForNode(
CC,
false, isVarArg);
2133 bool isVarArg)
const {
2134 return CCAssignFnForNode(
CC,
true, isVarArg);
2141 bool isVarArg)
const {
2142 switch (getEffectiveCallingConv(
CC, isVarArg)) {
2168 if (Subtarget->hasFullFP16()) {
2181 if (Subtarget->hasFullFP16()) {
2195SDValue ARMTargetLowering::LowerCallResult(
2207 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
2212 if (i == 0 && isThisReturn) {
2214 "unexpected return calling convention register assignment");
2225 Chain =
Lo.getValue(1);
2226 InGlue =
Lo.getValue(2);
2230 Chain =
Hi.getValue(1);
2231 InGlue =
Hi.getValue(2);
2243 Chain =
Lo.getValue(1);
2244 InGlue =
Lo.getValue(2);
2247 Chain =
Hi.getValue(1);
2248 InGlue =
Hi.getValue(2);
2283std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
2285 bool IsTailCall,
int SPDiff)
const {
2307 return std::make_pair(DstAddr, DstInfo);
2312 RegsToPassVector &RegsToPass,
2319 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
2320 unsigned id = Subtarget->
isLittle() ? 0 : 1;
2333 std::tie(DstAddr, DstInfo) =
2334 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2366 bool isStructRet = (Outs.
empty()) ?
false : Outs[0].
Flags.isSRet();
2367 bool isThisReturn =
false;
2368 bool isCmseNSCall =
false;
2369 bool isSibCall =
false;
2370 bool PreferIndirect =
false;
2371 bool GuardWithBTI =
false;
2375 !Subtarget->noBTIAtReturnTwice())
2380 isCmseNSCall =
true;
2392 if (isa<GlobalAddressSDNode>(Callee)) {
2396 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2399 PreferIndirect = Subtarget->isThumb() && Subtarget->
hasMinSize() &&
2401 return isa<Instruction>(U) &&
2402 cast<Instruction>(U)->getParent() == BB;
2408 isTailCall = IsEligibleForTailCallOptimization(
2409 Callee, CallConv, isVarArg, isStructRet,
2425 "site marked musttail");
2433 unsigned NumBytes = CCInfo.getStackSize();
2442 if (isTailCall && !isSibCall) {
2449 NumBytes =
alignTo(NumBytes, StackAlign);
2454 SPDiff = NumReusableBytes - NumBytes;
2458 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (
unsigned)-SPDiff)
2474 RegsToPassVector RegsToPass;
2482 bool AfterFormalArgLoads =
false;
2486 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
2488 ++i, ++realArgIdx) {
2490 SDValue Arg = OutVals[realArgIdx];
2492 bool isByVal =
Flags.isByVal();
2512 if (isTailCall && VA.
isMemLoc() && !AfterFormalArgLoads) {
2514 AfterFormalArgLoads =
true;
2526 auto ArgVT = Outs[realArgIdx].ArgVT;
2527 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2545 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2546 StackPtr, MemOpChains, isTailCall, SPDiff);
2550 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2551 StackPtr, MemOpChains, isTailCall, SPDiff);
2556 std::tie(DstAddr, DstInfo) =
2557 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2561 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2562 StackPtr, MemOpChains, isTailCall, SPDiff);
2564 if (realArgIdx == 0 &&
Flags.isReturned() && !
Flags.isSwiftSelf() &&
2565 Outs[0].VT == MVT::i32) {
2567 "unexpected calling convention register assignment");
2568 assert(!
Ins.empty() && Ins[0].VT == MVT::i32 &&
2569 "unexpected use of 'returned'");
2570 isThisReturn =
true;
2575 RegsToPass.push_back(std::make_pair(VA.
getLocReg(), Arg));
2576 }
else if (isByVal) {
2578 unsigned offset = 0;
2582 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2583 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2585 if (CurByValIdx < ByValArgsCount) {
2587 unsigned RegBegin, RegEnd;
2588 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2593 for (i = 0, j = RegBegin;
j < RegEnd; i++,
j++) {
2600 RegsToPass.push_back(std::make_pair(j, Load));
2605 offset = RegEnd - RegBegin;
2607 CCInfo.nextInRegsParam();
2610 if (
Flags.getByValSize() > 4*offset) {
2614 std::tie(Dst, DstInfo) =
2615 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2624 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2632 std::tie(DstAddr, DstInfo) =
2633 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2640 if (!MemOpChains.
empty())
2646 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2647 Chain = DAG.
getCopyToReg(Chain, dl, RegsToPass[i].first,
2648 RegsToPass[i].second, InGlue);
2655 bool isDirect =
false;
2660 GVal =
G->getGlobal();
2661 bool isStub = !
TM.shouldAssumeDSOLocal(GVal) && Subtarget->
isTargetMachO();
2663 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->
isMClass());
2664 bool isLocalARMFunc =
false;
2667 if (Subtarget->genLongCalls()) {
2669 "long-calls codegen is not position independent!");
2673 if (isa<GlobalAddressSDNode>(Callee)) {
2674 if (Subtarget->genExecuteOnly()) {
2693 const char *
Sym = S->getSymbol();
2695 if (Subtarget->genExecuteOnly()) {
2714 }
else if (isa<GlobalAddressSDNode>(Callee)) {
2715 if (!PreferIndirect) {
2720 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !
ARMInterworking);
2722 if (isStub && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2734 "Windows is the only supported COFF target");
2738 else if (!
TM.shouldAssumeDSOLocal(GVal))
2754 const char *
Sym = S->getSymbol();
2755 if (isARMFunc && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2759 ARMPCLabelIndex, 4);
2773 assert(!isARMFunc && !isDirect &&
2774 "Cannot handle call to ARM function or direct call");
2777 "call to non-secure function would "
2778 "require passing arguments on stack",
2785 "call to non-secure function would return value through pointer",
2793 if (Subtarget->isThumb()) {
2796 else if (isCmseNSCall)
2798 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2803 if (!isDirect && !Subtarget->hasV5TOps())
2805 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2818 if (isTailCall && !isSibCall) {
2823 std::vector<SDValue> Ops;
2824 Ops.push_back(Chain);
2825 Ops.push_back(Callee);
2833 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2834 Ops.push_back(DAG.
getRegister(RegsToPass[i].first,
2835 RegsToPass[i].second.getValueType()));
2847 isThisReturn =
false;
2853 assert(Mask &&
"Missing call preserved mask for calling convention");
2857 Ops.push_back(InGlue);
2869 Chain = DAG.
getNode(CallOpc, dl, NodeTys, Ops);
2881 Chain = DAG.
getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
2887 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2888 InVals, isThisReturn,
2889 isThisReturn ? OutVals[0] :
SDValue());
2896void ARMTargetLowering::HandleByVal(
CCState *State,
unsigned &
Size,
2897 Align Alignment)
const {
2899 Alignment = std::max(Alignment,
Align(4));
2905 unsigned AlignInRegs = Alignment.
value() / 4;
2906 unsigned Waste = (ARM::R4 -
Reg) % AlignInRegs;
2907 for (
unsigned i = 0; i < Waste; ++i)
2913 unsigned Excess = 4 * (ARM::R4 -
Reg);
2920 if (NSAAOffset != 0 &&
Size > Excess) {
2932 unsigned ByValRegBegin =
Reg;
2933 unsigned ByValRegEnd = std::min<unsigned>(Reg +
Size / 4, ARM::R4);
2937 for (
unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2943 Size = std::max<int>(
Size - Excess, 0);
2954 int FI = std::numeric_limits<int>::max();
2962 if (!Flags.isByVal()) {
2968 }
else if (
LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2969 if (Flags.isByVal())
2984 assert(FI != std::numeric_limits<int>::max());
2993bool ARMTargetLowering::IsEligibleForTailCallOptimization(
2995 bool isCalleeStructRet,
bool isCallerStructRet,
2999 const bool isIndirect)
const {
3012 if (Outs.
size() >= 4 &&
3013 (!isa<GlobalAddressSDNode>(
Callee.getNode()) || isIndirect)) {
3031 return CalleeCC == CallerCC;
3035 if (isCalleeStructRet || isCallerStructRet)
3049 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO()))
3056 getEffectiveCallingConv(CalleeCC, isVarArg),
3057 getEffectiveCallingConv(CallerCC, CallerF.
isVarArg()), MF,
C, Ins,
3063 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
3064 if (CalleeCC != CallerCC) {
3065 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
3066 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3079 if (!Outs.
empty()) {
3083 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs,
C);
3085 if (CCInfo.getStackSize()) {
3091 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
3093 ++i, ++realArgIdx) {
3096 SDValue Arg = OutVals[realArgIdx];
3100 if (VA.
needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
3107 if (!ArgLocs[++i].isRegLoc())
3109 if (RegVT == MVT::v2f64) {
3110 if (!ArgLocs[++i].isRegLoc())
3112 if (!ArgLocs[++i].isRegLoc())
3137 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
3146 StringRef IntKind =
F.getFnAttribute(
"interrupt").getValueAsString();
3159 if (IntKind ==
"" || IntKind ==
"IRQ" || IntKind ==
"FIQ" ||
3162 else if (IntKind ==
"SWI" || IntKind ==
"UNDEF")
3166 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
3193 bool isLittleEndian = Subtarget->
isLittle();
3205 "secure entry function would return value through pointer",
3211 for (
unsigned i = 0, realRVLocIdx = 0;
3213 ++i, ++realRVLocIdx) {
3217 SDValue Arg = OutVals[realRVLocIdx];
3218 bool ReturnF16 =
false;
3253 auto RetVT = Outs[realRVLocIdx].ArgVT;
3275 DAG.
getVTList(MVT::i32, MVT::i32), Half);
3279 HalfGPRs.
getValue(isLittleEndian ? 0 : 1), Glue);
3285 HalfGPRs.
getValue(isLittleEndian ? 1 : 0), Glue);
3297 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
3299 fmrrd.
getValue(isLittleEndian ? 0 : 1), Glue);
3304 fmrrd.
getValue(isLittleEndian ? 1 : 0), Glue);
3348 return DAG.
getNode(RetNode, dl, MVT::Other, RetOps);
3351bool ARMTargetLowering::isUsedByReturnOnly(
SDNode *
N,
SDValue &Chain)
const {
3352 if (
N->getNumValues() != 1)
3354 if (!
N->hasNUsesOfValue(1, 0))
3362 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3364 TCChain =
Copy->getOperand(0);
3378 SDValue UseChain =
U->getOperand(0);
3386 if (
U->getOperand(
U->getNumOperands() - 1).getValueType() == MVT::Glue)
3394 if (!
Copy->hasOneUse())
3401 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3403 TCChain =
Copy->getOperand(0);
3408 bool HasRet =
false;
3423bool ARMTargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
3441 &&
"LowerWRITE_REGISTER called for non-i64 type argument.");
3457 EVT PtrVT =
Op.getValueType();
3467 if (Subtarget->genExecuteOnly()) {
3469 auto T =
const_cast<Type*
>(
CP->getType());
3470 auto C =
const_cast<Constant*
>(
CP->getConstVal());
3481 return LowerGlobalAddress(GA, DAG);
3486 Align CPAlign =
CP->getAlign();
3488 CPAlign = std::max(CPAlign,
Align(4));
3489 if (
CP->isMachineConstantPoolEntry())
3501 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3510 unsigned ARMPCLabelIndex = 0;
3513 const BlockAddress *BA = cast<BlockAddressSDNode>(
Op)->getBlockAddress();
3516 if (!IsPositionIndependent) {
3519 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3530 if (!IsPositionIndependent)
3561ARMTargetLowering::LowerGlobalTLSAddressDarwin(
SDValue Op,
3564 "This function expects a Darwin target");
3569 SDValue DescAddr = LowerGlobalAddressDarwin(
Op, DAG);
3575 MVT::i32,
DL, Chain, DescAddr,
3599 Chain, FuncTLVGet, DAG.
getRegister(ARM::R0, MVT::i32),
3605ARMTargetLowering::LowerGlobalTLSAddressWindows(
SDValue Op,
3622 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
3649 const auto *GA = cast<GlobalAddressSDNode>(
Op);
3666 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3688 Args.push_back(Entry);
3696 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
3697 return CallResult.first;
3719 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3727 PtrVT, dl, Chain,
Offset,
3729 Chain =
Offset.getValue(1);
3735 PtrVT, dl, Chain,
Offset,
3745 PtrVT, dl, Chain,
Offset,
3761 return LowerGlobalTLSAddressDarwin(
Op, DAG);
3764 return LowerGlobalTLSAddressWindows(
Op, DAG);
3773 return LowerToTLSGeneralDynamicModel(GA, DAG);
3776 return LowerToTLSExecModels(GA, DAG, model);
3785 while (!Worklist.
empty()) {
3787 if (isa<ConstantExpr>(U)) {
3792 auto *
I = dyn_cast<Instruction>(U);
3793 if (!
I ||
I->getParent()->getParent() !=
F)
3821 auto *GVar = dyn_cast<GlobalVariable>(GV);
3822 if (!GVar || !GVar->hasInitializer() ||
3823 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3824 !GVar->hasLocalLinkage())
3829 auto *
Init = GVar->getInitializer();
3831 Init->needsDynamicRelocation())
3840 auto *CDAInit = dyn_cast<ConstantDataArray>(
Init);
3843 unsigned RequiredPadding = 4 - (
Size % 4);
3844 bool PaddingPossible =
3845 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3850 unsigned PaddedSize =
Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3874 if (RequiredPadding != 4) {
3879 while (RequiredPadding--)
3891 ++NumConstpoolPromoted;
3896 if (
const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
3897 if (!(GV = GA->getAliaseeObject()))
3899 if (
const auto *V = dyn_cast<GlobalVariable>(GV))
3900 return V->isConstant();
3901 return isa<Function>(GV);
3909 return LowerGlobalAddressWindows(
Op, DAG);
3911 return LowerGlobalAddressELF(
Op, DAG);
3913 return LowerGlobalAddressDarwin(
Op, DAG);
3921 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
3925 if (GV->
isDSOLocal() && !Subtarget->genExecuteOnly())
3938 }
else if (Subtarget->
isROPI() && IsRO) {
3943 }
else if (Subtarget->
isRWPI() && !IsRO) {
3968 if (Subtarget->
useMovt() || Subtarget->genExecuteOnly()) {
3987 "ROPI/RWPI not currently supported for Darwin");
3990 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4013 "Windows on ARM expects to use movw/movt");
4015 "ROPI/RWPI not currently supported for Windows");
4018 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4022 else if (!
TM.shouldAssumeDSOLocal(GV))
4046 DAG.
getVTList(MVT::i32, MVT::Other),
Op.getOperand(0),
4047 Op.getOperand(1), Val);
4064SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
4067 Op.getConstantOperandVal(
Op.getOperand(0).getValueType() == MVT::Other);
4071 case Intrinsic::arm_gnu_eabi_mcount: {
4080 assert(Mask &&
"Missing call preserved mask for calling convention");
4085 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
4089 if (Subtarget->isThumb())
4092 ARM::tBL_PUSHLR, dl, ResultTys,
4093 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
4094 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
4098 {ReturnAddress, Callee, RegisterMask, Chain}),
4107 unsigned IntNo =
Op.getConstantOperandVal(0);
4111 case Intrinsic::thread_pointer: {
4115 case Intrinsic::arm_cls: {
4116 const SDValue &Operand =
Op.getOperand(1);
4117 const EVT VTy =
Op.getValueType();
4128 case Intrinsic::arm_cls64: {
4131 const SDValue &Operand =
Op.getOperand(1);
4132 const EVT VTy =
Op.getValueType();
4155 case Intrinsic::eh_sjlj_lsda: {
4162 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
4172 if (IsPositionIndependent) {
4178 case Intrinsic::arm_neon_vabs:
4181 case Intrinsic::arm_neon_vmulls:
4182 case Intrinsic::arm_neon_vmullu: {
4183 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
4186 Op.getOperand(1),
Op.getOperand(2));
4188 case Intrinsic::arm_neon_vminnm:
4189 case Intrinsic::arm_neon_vmaxnm: {
4190 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
4193 Op.getOperand(1),
Op.getOperand(2));
4195 case Intrinsic::arm_neon_vminu:
4196 case Intrinsic::arm_neon_vmaxu: {
4197 if (
Op.getValueType().isFloatingPoint())
4199 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
4202 Op.getOperand(1),
Op.getOperand(2));
4204 case Intrinsic::arm_neon_vmins:
4205 case Intrinsic::arm_neon_vmaxs: {
4207 if (!
Op.getValueType().isFloatingPoint()) {
4208 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4211 Op.getOperand(1),
Op.getOperand(2));
4213 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4216 Op.getOperand(1),
Op.getOperand(2));
4218 case Intrinsic::arm_neon_vtbl1:
4220 Op.getOperand(1),
Op.getOperand(2));
4221 case Intrinsic::arm_neon_vtbl2:
4223 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4224 case Intrinsic::arm_mve_pred_i2v:
4225 case Intrinsic::arm_mve_pred_v2i:
4228 case Intrinsic::arm_mve_vreinterpretq:
4231 case Intrinsic::arm_mve_lsll:
4233 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4234 case Intrinsic::arm_mve_asrl:
4236 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4247 if (!Subtarget->hasDataBarrier()) {
4251 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4252 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4263 }
else if (Subtarget->preferISHSTBarriers() &&
4272 DAG.
getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4280 (!Subtarget->
isThumb1Only() && Subtarget->hasV5TEOps())))
4282 return Op.getOperand(0);
4285 unsigned isRead =
~Op.getConstantOperandVal(2) & 1;
4287 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4289 return Op.getOperand(0);
4291 unsigned isData =
Op.getConstantOperandVal(4);
4292 if (Subtarget->isThumb()) {
4294 isRead = ~isRead & 1;
4295 isData = ~isData & 1;
4312 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
4313 return DAG.
getStore(
Op.getOperand(0), dl, FR,
Op.getOperand(1),
4321 const SDLoc &dl)
const {
4327 RC = &ARM::tGPRRegClass;
4329 RC = &ARM::GPRRegClass;
4343 MVT::i32, dl, Root, FIN,
4364 const Value *OrigArg,
4365 unsigned InRegsParamRecordIdx,
4366 int ArgOffset,
unsigned ArgSize)
const {
4381 unsigned RBegin, REnd;
4391 ArgOffset = -4 * (ARM::R4 - RBegin);
4401 for (
unsigned Reg = RBegin, i = 0;
Reg < REnd; ++
Reg, ++i) {
4410 if (!MemOps.
empty())
4419 unsigned TotalArgRegsSaveSize,
4420 bool ForceMutable)
const {
4431 CCInfo.
getStackSize(), std::max(4U, TotalArgRegsSaveSize));
4435bool ARMTargetLowering::splitValueIntoRegisterParts(
4437 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4439 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4451SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4453 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
4454 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4467SDValue ARMTargetLowering::LowerFormalArguments(
4485 unsigned CurArgIdx = 0;
4497 unsigned ArgRegBegin = ARM::R4;
4498 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4505 if (!
Flags.isByVal())
4509 unsigned RBegin, REnd;
4511 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4517 int lastInsIndex = -1;
4521 ArgRegBegin = std::min(ArgRegBegin, (
unsigned)
GPRArgRegs[RegIdx]);
4524 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4528 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4530 if (Ins[VA.
getValNo()].isOrigArg()) {
4531 std::advance(CurOrigArg,
4532 Ins[VA.
getValNo()].getOrigArgIndex() - CurArgIdx);
4543 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4550 MVT::f64, dl, Chain, FIN,
4553 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4561 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4565 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4566 RC = &ARM::HPRRegClass;
4567 else if (RegVT == MVT::f32)
4568 RC = &ARM::SPRRegClass;
4569 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4570 RegVT == MVT::v4bf16)
4571 RC = &ARM::DPRRegClass;
4572 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4573 RegVT == MVT::v8bf16)
4574 RC = &ARM::QPRRegClass;
4575 else if (RegVT == MVT::i32)
4577 : &ARM::GPRRegClass;
4624 assert(VA.
getValVT() != MVT::i64 &&
"i64 should already be lowered");
4630 if (index != lastInsIndex)
4638 if (
Flags.isByVal()) {
4639 assert(Ins[index].isOrigArg() &&
4640 "Byval arguments cannot be implicit");
4644 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4659 lastInsIndex = index;
4666 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.
getStackSize(),
4667 TotalArgRegsSaveSize);
4671 "secure entry function must not be variadic", dl.
getDebugLoc());
4682 StackArgSize =
alignTo(StackArgSize,
DL.getStackAlignment());
4691 "secure entry function requires arguments on stack", dl.
getDebugLoc());
4701 return CFP->getValueAPF().isPosZero();
4705 SDValue WrapperOp =
Op.getOperand(1).getOperand(0);
4707 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
4708 return CFP->getValueAPF().isPosZero();
4711 Op->getValueType(0) == MVT::f64) {
4726 const SDLoc &dl)
const {
4728 unsigned C = RHSC->getZExtValue();
4794 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
4796 unsigned Mask =
LHS.getConstantOperandVal(1);
4797 auto *RHSC = cast<ConstantSDNode>(
RHS.getNode());
4798 uint64_t RHSV = RHSC->getZExtValue();
4799 if (
isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4801 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4816 isa<ConstantSDNode>(RHS) &&
RHS->getAsZExtVal() == 0x80000000U &&
4818 LHS.getConstantOperandVal(1) < 31) {
4819 unsigned ShiftAmt =
LHS.getConstantOperandVal(1) + 1;
4859 return DAG.
getNode(CompareType, dl, MVT::Glue, LHS, RHS);
4865 bool Signaling)
const {
4866 assert(Subtarget->hasFP64() ||
RHS.getValueType() != MVT::f64);
4870 dl, MVT::Glue, LHS, RHS);
4873 dl, MVT::Glue, LHS);
4881 unsigned Opc =
Cmp.getOpcode();
4884 return DAG.
getNode(Opc,
DL, MVT::Glue,
Cmp.getOperand(0),
Cmp.getOperand(1));
4888 Opc =
Cmp.getOpcode();
4902std::pair<SDValue, SDValue>
4905 assert(
Op.getValueType() == MVT::i32 &&
"Unsupported value type");
4917 switch (
Op.getOpcode()) {
4930 DAG.
getVTList(
Op.getValueType(), MVT::i32), LHS, RHS)
4969 return std::make_pair(
Value, OverflowCmp);
4980 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Op, DAG, ARMcc);
4986 EVT VT =
Op.getValueType();
4989 ARMcc, CCR, OverflowCmp);
5029 EVT VT =
Op.getValueType();
5033 switch (
Op.getOpcode()) {
5058 EVT VT =
Op.getValueType();
5059 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->
isThumb1Only())
5069 switch (
Op->getOpcode()) {
5085 switch (
Op->getOpcode()) {
5104 DAG.
getNode(NewOpcode, dl, MVT::i32,
5115 unsigned Opc =
Cond.getOpcode();
5117 if (
Cond.getResNo() == 1 &&
5125 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5127 EVT VT =
Op.getValueType();
5129 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
5140 dyn_cast<ConstantSDNode>(
Cond.getOperand(0));
5142 dyn_cast<ConstantSDNode>(
Cond.getOperand(1));
5144 if (CMOVTrue && CMOVFalse) {
5150 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5152 False = SelectFalse;
5153 }
else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5159 EVT VT =
Op.getValueType();
5164 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
5180 bool &swpCmpOps,
bool &swpVselOps) {
5208 swpCmpOps = !swpCmpOps;
5209 swpVselOps = !swpVselOps;
5232 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5234 DAG.
getVTList(MVT::i32, MVT::i32), FalseVal);
5236 DAG.
getVTList(MVT::i32, MVT::i32), TrueVal);
5246 ARMcc, CCR, duplicateCmp(Cmp, DAG));
5273 ((K ==
LHS && K == TrueVal) || (K ==
RHS && K == FalseVal))) ||
5275 ((K ==
RHS && K == TrueVal) || (K ==
LHS && K == FalseVal)));
5296 EVT VT =
Op.getValueType();
5303 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5318 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5326 if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
5329 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
5330 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
5331 int64_t PosVal = std::max(Val1, Val2);
5332 int64_t NegVal = std::min(Val1, Val2);
5378 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5379 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5384 if (*K != KTmp || V != VTmp)
5395bool ARMTargetLowering::isUnsupportedFloatingType(
EVT VT)
const {
5399 return !Subtarget->hasFP64();
5401 return !Subtarget->hasFullFP16();
5406 EVT VT =
Op.getValueType();
5410 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->
isThumb2())
5422 if (VT == MVT::i32 &&
5442 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
5443 LHS.getValueType() == MVT::i32 &&
RHS.getValueType() == MVT::i32) {
5446 unsigned Opcode = 0;
5448 if (TVal == ~FVal) {
5450 }
else if (TVal == ~FVal + 1) {
5452 }
else if (TVal + 1 == FVal) {
5454 }
else if (TVal == FVal + 1) {
5486 return DAG.
getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5490 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5492 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5496 if (!
RHS.getNode()) {
5502 if (
LHS.getValueType() == MVT::i32) {
5514 TrueVal.getValueType() == MVT::f32 ||
5515 TrueVal.getValueType() == MVT::f64)) {
5530 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5542 (
TrueVal.getValueType() == MVT::f16 ||
5543 TrueVal.getValueType() == MVT::f32 ||
5544 TrueVal.getValueType() == MVT::f64)) {
5545 bool swpCmpOps =
false;
5546 bool swpVselOps =
false;
5561 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5565 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5566 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
5576 if (!
N->hasOneUse())
5579 if (!
N->getNumValues())
5581 EVT VT =
Op.getValueType();
5582 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5599 return DAG.
getLoad(MVT::i32,
SDLoc(
Op), Ld->getChain(), Ld->getBasePtr(),
5600 Ld->getPointerInfo(), Ld->getAlign(),
5601 Ld->getMemOperand()->getFlags());
5619 DAG.
getLoad(MVT::i32, dl, Ld->getChain(),
Ptr, Ld->getPointerInfo(),
5620 Ld->getAlign(), Ld->getMemOperand()->getFlags());
5622 EVT PtrType =
Ptr.getValueType();
5625 RetVal2 = DAG.
getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5626 Ld->getPointerInfo().getWithOffset(4),
5628 Ld->getMemOperand()->getFlags());
5646 bool LHSSeenZero =
false;
5648 bool RHSSeenZero =
false;
5650 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5661 if (
LHS.getValueType() == MVT::f32) {
5669 Chain, Dest, ARMcc, CCR, Cmp);
5681 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5696 unsigned Opc =
Cond.getOpcode();
5699 if (
Cond.getResNo() == 1 &&
5709 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5733 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5735 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5739 if (!
RHS.getNode()) {
5747 unsigned Opc =
LHS.getOpcode();
5761 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
LHS.getValue(0), DAG, ARMcc);
5776 if (
LHS.getValueType() == MVT::i32) {
5781 Chain, Dest, ARMcc, CCR, Cmp);
5787 if (
SDValue Result = OptimizeVFPBrcond(
Op, DAG))
5798 SDValue Ops[] = { Chain, Dest, ARMcc, CCR,
Cmp };
5820 if (Subtarget->
isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5826 Addr,
Op.getOperand(2), JTI);
5832 Chain =
Addr.getValue(1);
5839 Chain =
Addr.getValue(1);
5845 EVT VT =
Op.getValueType();
5848 if (
Op.getValueType().getVectorElementType() == MVT::i32) {
5849 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5857 const EVT OpTy =
Op.getOperand(0).getValueType();
5858 if (OpTy == MVT::v4f32)
5860 else if (OpTy == MVT::v4f16 && HasFullFP16)
5862 else if (OpTy == MVT::v8f16 && HasFullFP16)
5867 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5870 Op = DAG.
getNode(
Op.getOpcode(), dl, NewTy,
Op.getOperand(0));
5875 EVT VT =
Op.getValueType();
5879 bool IsStrict =
Op->isStrictFPOpcode();
5880 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
5882 if (isUnsupportedFloatingType(SrcVal.
getValueType())) {
5892 MakeLibCallOptions CallOptions;
5895 std::tie(Result, Chain) =
makeLibCall(DAG, LC,
Op.getValueType(), SrcVal,
5896 CallOptions, Loc, Chain);
5906 Loc,
Op.getValueType(), SrcVal);
5915 EVT VT =
Op.getValueType();
5916 EVT ToVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
5917 EVT FromVT =
Op.getOperand(0).getValueType();
5919 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5921 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5922 Subtarget->hasFP64())
5924 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5925 Subtarget->hasFullFP16())
5927 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5928 Subtarget->hasMVEFloatOps())
5930 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5931 Subtarget->hasMVEFloatOps())
5934 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5951 EVT VT =
Op.getValueType();
5954 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5960 assert((
Op.getOperand(0).getValueType() == MVT::v4i16 ||
5961 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5962 "Invalid type for custom lowering!");
5967 if (VT == MVT::v4f32)
5968 DestVecType = MVT::v4i32;
5969 else if (VT == MVT::v4f16 && HasFullFP16)
5970 DestVecType = MVT::v4i16;
5971 else if (VT == MVT::v8f16 && HasFullFP16)
5972 DestVecType = MVT::v8i16;
5978 switch (
Op.getOpcode()) {
5990 Op = DAG.
getNode(CastOpc, dl, DestVecType,
Op.getOperand(0));
5995 EVT VT =
Op.getValueType();
5998 if (isUnsupportedFloatingType(VT)) {
6006 MakeLibCallOptions CallOptions;
6008 CallOptions,
SDLoc(
Op)).first;
6019 EVT VT =
Op.getValueType();
6023 bool UseNEON = !InGPR && Subtarget->hasNEON();
6030 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
6037 if (SrcVT == MVT::f32) {
6043 }
else if (VT == MVT::f32)
6059 if (VT == MVT::f32) {
6071 if (SrcVT == MVT::f64)
6080 if (VT == MVT::f32) {
6104 EVT VT =
Op.getValueType();
6106 unsigned Depth =
Op.getConstantOperandVal(0);
6108 SDValue FrameAddr = LowerFRAMEADDR(
Op, DAG);
6127 EVT VT =
Op.getValueType();
6129 unsigned Depth =
Op.getConstantOperandVal(0);
6143 .
Case(
"sp", ARM::SP)
6158 assert(
N->getValueType(0) == MVT::i64
6159 &&
"ExpandREAD_REGISTER called for non-i64 type result.");
6162 DAG.
getVTList(MVT::i32, MVT::i32, MVT::Other),
6168 Results.push_back(Read.getOperand(0));
6202 const APInt &APIntIndex =
Index->getAPIntValue();
6204 NewIndex *= APIntIndex;
6234 EVT SrcVT =
Op.getValueType();
6235 EVT DstVT =
N->getValueType(0);
6237 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6238 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6242 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6243 (SrcVT == MVT::f16 || SrcVT == MVT::bf16))
6248 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6252 if (SrcVT == MVT::i64 && TLI.
isTypeLegal(DstVT)) {
6264 if (DstVT == MVT::i64 && TLI.
isTypeLegal(SrcVT)) {
6301 EVT VT =
Op.getValueType();
6320 SDValue LoBigShift = DAG.
getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6328 ? DAG.
getNode(Opc, dl, VT, ShOpHi,
6345 EVT VT =
Op.getValueType();
6388 DAG.
getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6440 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6468 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6498 EVT VT =
N->getValueType(0);
6499 if (VT.
isVector() && ST->hasNEON()) {
6508 if (ElemTy == MVT::i8) {
6516 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6531 if (ElemTy == MVT::i64) {
6544 if (!ST->hasV6T2Ops())
6553 EVT VT =
N->getValueType(0);
6556 assert(ST->hasNEON() &&
"Custom ctpop lowering requires NEON.");
6557 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6558 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6559 "Unexpected type for custom ctpop lowering");
6567 unsigned EltSize = 8;
6590 Op =
Op.getOperand(0);
6592 APInt SplatBits, SplatUndef;
6593 unsigned SplatBitSize;
6596 !BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6598 SplatBitSize > ElementBits)
6609 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6613 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6624 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6629 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6630 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6639 EVT VT =
N->getValueType(0);
6661 "unexpected vector shift opcode");
6663 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
6664 unsigned VShiftOpc =
6666 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
6672 EVT ShiftVT =
N->getOperand(1).getValueType();
6675 unsigned VShiftOpc =
6677 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0), NegatedCount);
6682 EVT VT =
N->getValueType(0);
6691 "Unknown shift to lower!");
6693 unsigned ShOpc =
N->getOpcode();
6694 if (ST->hasMVEIntegerOps()) {
6724 DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6738 if (ST->isThumb1Only())
6743 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6759 bool Invert =
false;
6766 EVT VT =
Op.getValueType();
6774 assert(ST->hasMVEIntegerOps() &&
6775 "No hardware support for integer vector comparison!");
6777 if (
Op.getValueType().getVectorElementType() != MVT::i1)
6802 Merged = DAG.
getNOT(dl, Merged, CmpVT);
6812 switch (SetCCOpcode) {
6816 if (ST->hasMVEFloatOps()) {
6819 Invert =
true; [[fallthrough]];
6824 case ISD::SETLT: Swap =
true; [[fallthrough]];
6828 case ISD::SETLE: Swap =
true; [[fallthrough]];
6844 Result = DAG.
getNOT(dl, Result, VT);
6847 case ISD::SETUO: Invert =
true; [[fallthrough]];
6856 Result = DAG.
getNOT(dl, Result, VT);
6862 switch (SetCCOpcode) {
6865 if (ST->hasMVEIntegerOps()) {
6868 Invert =
true; [[fallthrough]];
6871 case ISD::SETLT: Swap =
true; [[fallthrough]];
6873 case ISD::SETLE: Swap =
true; [[fallthrough]];
6882 if (ST->hasNEON() && Opc ==
ARMCC::EQ) {
6898 Result = DAG.
getNOT(dl, Result, VT);
6932 Result = DAG.
getNOT(dl, Result, VT);
6944 assert(
LHS.getSimpleValueType().isInteger() &&
"SETCCCARRY is integer only.");
6974 unsigned OpCmode, Imm;
6985 switch (SplatBitSize) {
6990 assert((SplatBits & ~0xff) == 0 &&
"one byte splat value is too big");
6993 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
6998 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
6999 if ((SplatBits & ~0xff) == 0) {
7005 if ((SplatBits & ~0xff00) == 0) {
7008 Imm = SplatBits >> 8;
7018 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
7019 if ((SplatBits & ~0xff) == 0) {
7025 if ((SplatBits & ~0xff00) == 0) {
7028 Imm = SplatBits >> 8;
7031 if ((SplatBits & ~0xff0000) == 0) {
7034 Imm = SplatBits >> 16;
7037 if ((SplatBits & ~0xff000000) == 0) {
7040 Imm = SplatBits >> 24;
7047 if ((SplatBits & ~0xffff) == 0 &&
7048 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
7051 Imm = SplatBits >> 8;
7059 if ((SplatBits & ~0xffffff) == 0 &&
7060 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7063 Imm = SplatBits >> 16;
7079 unsigned ImmMask = 1;
7081 for (
int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7082 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7084 }
else if ((SplatBits & BitMask) != 0) {
7094 unsigned Mask = (1 << BytesPerElem) - 1;
7095 unsigned NumElems = 8 / BytesPerElem;
7096 unsigned NewImm = 0;
7097 for (
unsigned ElemNum = 0; ElemNum < NumElems; ++ElemNum) {
7098 unsigned Elem = ((Imm >> ElemNum * BytesPerElem) & Mask);
7099 NewImm |= Elem << (NumElems - ElemNum - 1) * BytesPerElem;
7106 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7120 EVT VT =
Op.getValueType();
7121 bool IsDouble = (VT == MVT::f64);
7127 if (
ST->genExecuteOnly()) {
7129 assert((!
ST->isThumb1Only() ||
ST->hasV8MBaselineOps()) &&
7130 "Unexpected architecture");
7153 if (!
ST->hasVFP3Base())
7158 if (IsDouble && !Subtarget->hasFP64())
7165 if (IsDouble || !
ST->useNEONForSinglePrecisionFP()) {
7183 if (!
ST->hasNEON() || (!IsDouble && !
ST->useNEONForSinglePrecisionFP()))
7192 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7246 unsigned ExpectedElt = Imm;
7247 for (
unsigned i = 1; i < NumElts; ++i) {
7251 if (ExpectedElt == NumElts)
7254 if (M[i] < 0)
continue;
7255 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7263 bool &ReverseVEXT,
unsigned &Imm) {
7265 ReverseVEXT =
false;
7276 unsigned ExpectedElt = Imm;
7277 for (
unsigned i = 1; i < NumElts; ++i) {
7281 if (ExpectedElt == NumElts * 2) {
7286 if (M[i] < 0)
continue;
7287 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7302 return VT == MVT::v8i8 && M.size() == 8;
7307 if (Mask.size() == Elements * 2)
7308 return Index / Elements;
7309 return Mask[
Index] == 0 ? 0 : 1;
7339 if (M.size() != NumElts && M.size() != NumElts*2)
7347 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7349 for (
unsigned j = 0; j < NumElts; j += 2) {
7350 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7351 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + NumElts + WhichResult))
7356 if (M.size() == NumElts*2)
7371 if (M.size() != NumElts && M.size() != NumElts*2)
7374 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7376 for (
unsigned j = 0; j < NumElts; j += 2) {
7377 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7378 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + WhichResult))
7383 if (M.size() == NumElts*2)
7403 if (M.size() != NumElts && M.size() != NumElts*2)
7406 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7408 for (
unsigned j = 0; j < NumElts; ++j) {
7409 if (M[i+j] >= 0 && (
unsigned) M[i+j] != 2 * j + WhichResult)
7414 if (M.size() == NumElts*2)
7433 if (M.size() != NumElts && M.size() != NumElts*2)
7436 unsigned Half = NumElts / 2;
7437 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7439 for (
unsigned j = 0; j < NumElts; j += Half) {
7440 unsigned Idx = WhichResult;
7441 for (
unsigned k = 0; k < Half; ++k) {
7442 int MIdx = M[i + j + k];
7443 if (MIdx >= 0 && (
unsigned) MIdx !=
Idx)
7450 if (M.size() == NumElts*2)
7474 if (M.size() != NumElts && M.size() != NumElts*2)
7477 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7479 unsigned Idx = WhichResult * NumElts / 2;
7480 for (
unsigned j = 0; j < NumElts; j += 2) {
7481 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7482 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx + NumElts))
7488 if (M.size() == NumElts*2)
7507 if (M.size() != NumElts && M.size() != NumElts*2)
7510 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7512 unsigned Idx = WhichResult * NumElts / 2;
7513 for (
unsigned j = 0; j < NumElts; j += 2) {
7514 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7515 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx))
7521 if (M.size() == NumElts*2)
7534 unsigned &WhichResult,
7537 if (
isVTRNMask(ShuffleMask, VT, WhichResult))
7539 if (
isVUZPMask(ShuffleMask, VT, WhichResult))
7541 if (
isVZIPMask(ShuffleMask, VT, WhichResult))
7559 if (NumElts != M.size())
7563 for (
unsigned i = 0; i != NumElts; ++i)
7564 if (M[i] >= 0 && M[i] != (
int) (NumElts - 1 - i))
7573 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7581 int Ofs = Top ? 1 : 0;
7582 int Upper = SingleSource ? 0 : NumElts;
7583 for (
int i = 0, e = NumElts / 2; i != e; ++i) {
7584 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7586 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs +
Upper)
7595 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7604 unsigned Offset = Top ? 0 : 1;
7605 unsigned N = SingleSource ? 0 : NumElts;
7606 for (
unsigned i = 0; i < NumElts; i += 2) {
7607 if (M[i] >= 0 && M[i] != (
int)i)
7609 if (M[i + 1] >= 0 && M[i + 1] != (
int)(
N + i +
Offset))
7618 if (NumElts != M.size())
7626 unsigned Off0 = rev ? NumElts / 2 : 0;
7627 unsigned Off1 = rev ? 0 : NumElts / 2;
7628 for (
unsigned i = 0; i < NumElts; i += 2) {
7629 if (M[i] >= 0 && M[i] != (
int)(Off0 + i / 2))
7631 if (M[i + 1] >= 0 && M[i + 1] != (
int)(Off1 + i / 2))
7647 if (!ST->hasMVEFloatOps())
7652 if (VT != MVT::v8f16)
7673 for (
unsigned i = 1; i < 4; i++) {
7700 if (!ST->hasMVEFloatOps())
7705 if (VT != MVT::v4f32)
7721 for (
unsigned i = 1; i < 4; i++) {
7742 if (!isa<ConstantSDNode>(
N))
7744 Val =
N->getAsZExtVal();
7746 if (ST->isThumb1Only()) {
7747 if (Val <= 255 || ~Val <= 255)
7759 EVT VT =
Op.getValueType();
7761 assert(ST->hasMVEIntegerOps() &&
"LowerBUILD_VECTOR_i1 called without MVE!");
7765 unsigned BitsPerBool;
7769 }
else if (NumElts == 4) {
7772 }
else if (NumElts == 8) {
7775 }
else if (NumElts == 16) {
7784 if (!isa<ConstantSDNode>(FirstOp) &&
7786 return U.get().isUndef() || U.get() == FirstOp;
7794 unsigned Bits32 = 0;
7795 for (
unsigned i = 0; i < NumElts; ++i) {
7797 if (!isa<ConstantSDNode>(V) && !V.isUndef())
7799 bool BitSet = V.isUndef() ?
false : V->getAsZExtVal();
7801 Bits32 |= BoolMask << (i * BitsPerBool);
7807 for (
unsigned i = 0; i < NumElts; ++i) {
7809 if (isa<ConstantSDNode>(V) || V.isUndef())
7820 if (!ST->hasMVEIntegerOps())
7824 EVT VT =
Op.getValueType();
7834 if (
N != 1 &&
N != 2 &&
N != 4 &&
N != 8)
7838 for (
unsigned I = 2;
I < NumElts;
I++) {
7854 switch (
N->getOpcode()) {
7863 return N->getOperand(1).getNode() ==
Op;
7865 switch (
N->getConstantOperandVal(0)) {
7866 case Intrinsic::arm_mve_add_predicated:
7867 case Intrinsic::arm_mve_mul_predicated:
7868 case Intrinsic::arm_mve_qadd_predicated:
7869 case Intrinsic::arm_mve_vhadd:
7870 case Intrinsic::arm_mve_hadd_predicated:
7871 case Intrinsic::arm_mve_vqdmulh:
7872 case Intrinsic::arm_mve_qdmulh_predicated:
7873 case Intrinsic::arm_mve_vqrdmulh:
7874 case Intrinsic::arm_mve_qrdmulh_predicated:
7875 case Intrinsic::arm_mve_vqdmull:
7876 case Intrinsic::arm_mve_vqdmull_predicated:
7878 case Intrinsic::arm_mve_sub_predicated:
7879 case Intrinsic::arm_mve_qsub_predicated:
7880 case Intrinsic::arm_mve_vhsub:
7881 case Intrinsic::arm_mve_hsub_predicated:
7882 return N->getOperand(2).getNode() ==
Op;
7897 EVT VT =
Op.getValueType();
7905 APInt SplatBits, SplatUndef;
7906 unsigned SplatBitSize;
7908 if (BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7915 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7917 [BVN](
const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7918 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7919 : SplatBitSize == 16 ? MVT::v8i16
7926 if ((
ST->hasNEON() && SplatBitSize <= 64) ||
7927 (
ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7932 SplatBitSize, DAG, dl, VmovVT, VT,
VMOVModImm);
7940 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7942 NegatedImm, SplatUndef.
getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7950 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7960 if (
ST->hasMVEIntegerOps() &&
7961 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7962 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7963 : SplatBitSize == 16 ? MVT::v8i16
7980 bool isOnlyLowElement =
true;
7981 bool usesOnlyOneValue =
true;
7982 bool hasDominantValue =
false;
7989 for (
unsigned i = 0; i < NumElts; ++i) {
7994 isOnlyLowElement =
false;
7995 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
7998 ValueCounts.
insert(std::make_pair(V, 0));
7999 unsigned &Count = ValueCounts[
V];
8002 if (++Count > (NumElts / 2)) {
8003 hasDominantValue =
true;
8007 if (ValueCounts.
size() != 1)
8008 usesOnlyOneValue =
false;
8009 if (!
Value.getNode() && !ValueCounts.
empty())
8012 if (ValueCounts.
empty())
8024 if (hasDominantValue && EltSize <= 32) {
8035 (constIndex = dyn_cast<ConstantSDNode>(
Value->getOperand(1)))) {
8040 if (VT !=
Value->getOperand(0).getValueType()) {
8053 if (!usesOnlyOneValue) {
8056 for (
unsigned I = 0;
I < NumElts; ++
I) {
8071 assert(FVT == MVT::f32 || FVT == MVT::f16);
8072 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
8073 for (
unsigned i = 0; i < NumElts; ++i)
8078 Val = LowerBUILD_VECTOR(Val, DAG, ST);
8082 if (usesOnlyOneValue) {
8109 if (
ST->hasNEON() && VT.
is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
8129 if (EltSize >= 32) {
8135 for (
unsigned i = 0; i < NumElts; ++i)
8149 for (
unsigned i = 0 ; i < NumElts; ++i) {
8168 EVT VT =
Op.getValueType();
8171 struct ShuffleSourceInfo {
8173 unsigned MinElt = std::numeric_limits<unsigned>::max();
8174 unsigned MaxElt = 0;
8184 int WindowScale = 1;
8186 ShuffleSourceInfo(
SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
8194 for (
unsigned i = 0; i < NumElts; ++i) {
8202 }
else if (!isa<ConstantSDNode>(
V.getOperand(1))) {
8209 SDValue SourceVec =
V.getOperand(0);
8211 if (Source == Sources.
end())
8215 unsigned EltNo =
V.getConstantOperandVal(1);
8222 if (Sources.
size() > 2)
8228 for (
auto &Source : Sources) {
8229 EVT SrcEltTy =
Source.Vec.getValueType().getVectorElementType();
8230 if (SrcEltTy.
bitsLT(SmallestEltTy))
8231 SmallestEltTy = SrcEltTy;
8233 unsigned ResMultiplier =
8241 for (
auto &Src : Sources) {
8242 EVT SrcVT = Src.ShuffleVec.getValueType();
8246 if (SrcVTSize == VTSize)
8255 if (SrcVTSize < VTSize) {
8256 if (2 * SrcVTSize != VTSize)
8262 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8266 if (SrcVTSize != 2 * VTSize)
8269 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8274 if (Src.MinElt >= NumSrcElts) {
8279 Src.WindowBase = -NumSrcElts;
8280 }
else if (Src.MaxElt < NumSrcElts) {
8297 Src.WindowBase = -Src.MinElt;
8304 for (
auto &Src : Sources) {
8305 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8306 if (SrcEltTy == SmallestEltTy)
8311 Src.WindowBase *= Src.WindowScale;
8317 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8324 if (Entry.isUndef())
8327 auto Src =
llvm::find(Sources, Entry.getOperand(0));
8328 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
8333 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
8336 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8340 int *LaneMask = &
Mask[i * ResMultiplier];
8342 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8343 ExtractBase += NumElts * (Src - Sources.begin());
8344 for (
int j = 0;
j < LanesDefined; ++
j)
8345 LaneMask[j] = ExtractBase + j;
8351 assert(Sources.size() <= 2 &&
"Too many sources!");
8354 for (
unsigned i = 0; i < Sources.size(); ++i)
8383 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8403 unsigned PFIndexes[4];
8404 for (
unsigned i = 0; i != 4; ++i) {
8408 PFIndexes[i] = M[i];
8412 unsigned PFTableIndex =
8413 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8415 unsigned Cost = (PFEntry >> 30);
8421 bool ReverseVEXT, isV_UNDEF;
8422 unsigned Imm, WhichResult;
8425 if (EltSize >= 32 ||
8432 else if (Subtarget->hasNEON() &&
8437 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8440 else if (Subtarget->hasMVEIntegerOps() &&
8444 else if (Subtarget->hasMVEIntegerOps() &&
8458 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8459 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8460 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8463 if (LHSID == (1*9+2)*9+3)
return LHS;
8464 assert(LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!");
8521 for (
int I : ShuffleMask)
8524 if (V2.getNode()->isUndef())
8534 EVT VT =
Op.getValueType();
8536 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8537 "Expect an v8i16/v16i8 type");
8543 std::vector<int> NewMask;
8547 NewMask.push_back(i);
8587 if (VT != MVT::v16i1)
8603 EVT VT =
Op.getValueType();
8607 assert(ST->hasMVEIntegerOps() &&
8608 "No support for vector shuffle of boolean predicates");
8634 "Expected identical vector type in expanded i1 shuffle!");
8638 PredAsVector2, ShuffleMask);
8643 if (VT == MVT::v2i1) {
8660 EVT VT =
Op.getValueType();
8664 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8665 "Unexpected vector type");
8667 int QuarterSize = NumElts / 4;
8676 for (
int i = 0; i <
Length; i++) {
8677 if (ShuffleMask[Start + i] >= 0) {
8678 if (ShuffleMask[Start + i] %
Length != i)
8680 MovIdx = ShuffleMask[Start + i] /
Length;
8688 for (
int i = 1; i <
Length; i++) {
8689 if (ShuffleMask[Start + i] >= 0 &&
8690 (ShuffleMask[Start + i] /
Length != MovIdx ||
8691 ShuffleMask[Start + i] %
Length != i))
8697 for (
int Part = 0; Part < 4; ++Part) {
8699 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8703 Input =
Op->getOperand(1);
8713 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8718 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8720 for (
int Part = 0; Part < 4; ++Part)
8721 for (
int i = 0; i < QuarterSize; i++)
8723 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8725 VT, dl,
Op->getOperand(0),
Op->getOperand(1), NewShuffleMask);
8728 for (
int Part = 0; Part < 4; ++Part)
8744 EVT VT =
Op.getValueType();
8756 for (
int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8760 if (Mask[i] != i + BaseOffset) {
8761 if (OffElement == -1)
8767 return NonUndef > 2 && OffElement != -1;
8771 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8773 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8784 ShuffleMask[OffElement] < (
int)NumElts ? V1 : V2,
8795 EVT VT =
Op.getValueType();
8799 if (ST->hasMVEIntegerOps() && EltSize == 1)
8810 if (EltSize <= 32) {
8814 if (Lane == -1) Lane = 0;
8825 bool IsScalarToVector =
true;
8828 IsScalarToVector =
false;
8831 if (IsScalarToVector)
8838 bool ReverseVEXT =
false;
8840 if (ST->hasNEON() &&
isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8864 unsigned WhichResult = 0;
8865 bool isV_UNDEF =
false;
8866 if (ST->hasNEON()) {
8868 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8875 if (ST->hasMVEIntegerOps()) {
8910 }) &&
"Unexpected shuffle index into UNDEF operand!");
8913 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8916 assert((WhichResult == 0) &&
8917 "In-place shuffle of concat can only have one result!");
8926 if (ST->hasMVEIntegerOps() && EltSize <= 32) {
8930 for (
bool Top : {
false,
true}) {
8931 for (
bool SingleSource : {
false,
true}) {
8932 if (
isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8937 SingleSource ? V1 : V2);
8953 unsigned PFIndexes[4];
8954 for (
unsigned i = 0; i != 4; ++i) {
8955 if (ShuffleMask[i] < 0)
8958 PFIndexes[i] = ShuffleMask[i];
8962 unsigned PFTableIndex =
8963 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8965 unsigned Cost = (PFEntry >> 30);
8971 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8972 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8982 if (EltSize >= 32) {
8990 for (
unsigned i = 0; i < NumElts; ++i) {
8991 if (ShuffleMask[i] < 0)
8995 ShuffleMask[i] < (
int)NumElts ? V1 : V2,
9003 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
9007 if (ST->hasNEON() && VT == MVT::v8i8)
9011 if (ST->hasMVEIntegerOps())
9020 EVT VecVT =
Op.getOperand(0).getValueType();
9023 assert(ST->hasMVEIntegerOps() &&
9024 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9028 unsigned Lane =
Op.getConstantOperandVal(2);
9029 unsigned LaneWidth =
9031 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
9043 if (!isa<ConstantSDNode>(Lane))
9049 if (Subtarget->hasMVEIntegerOps() &&
9050 Op.getValueType().getScalarSizeInBits() == 1)
9074 IVecIn, IElt, Lane);
9083 EVT VecVT =
Op.getOperand(0).getValueType();
9086 assert(ST->hasMVEIntegerOps() &&
9087 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9091 unsigned Lane =
Op.getConstantOperandVal(1);
9092 unsigned LaneWidth =
9103 if (!isa<ConstantSDNode>(Lane))
9123 assert(
Op.getValueType().getScalarSizeInBits() == 1 &&
9124 "Unexpected custom CONCAT_VECTORS lowering");
9126 "Unexpected custom CONCAT_VECTORS lowering");
9127 assert(ST->hasMVEIntegerOps() &&
9128 "CONCAT_VECTORS lowering only supported for MVE");
9132 EVT Op2VT = V2.getValueType();
9133 assert(Op1VT == Op2VT &&
"Operand types don't match!");
9134 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
9135 "Unexpected i1 concat operations!");
9148 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
9162 auto ExtractInto = [&DAG, &dl](
SDValue NewV,
SDValue ConVec,
unsigned &j) {
9163 EVT NewVT = NewV.getValueType();
9164 EVT ConcatVT = ConVec.getValueType();
9165 unsigned ExtScale = 1;
9166 if (NewVT == MVT::v2f64) {
9180 ConVec = ExtractInto(NewV1, ConVec, j);
9181 ConVec = ExtractInto(NewV2, ConVec, j);
9191 while (ConcatOps.
size() > 1) {
9192 for (
unsigned I = 0,
E = ConcatOps.
size();
I !=
E;
I += 2) {
9195 ConcatOps[
I / 2] = ConcatPair(V1, V2);
9199 return ConcatOps[0];
9204 EVT VT =
Op->getValueType(0);
9211 "unexpected CONCAT_VECTORS");
9232 EVT VT =
Op.getValueType();
9235 unsigned Index = V2->getAsZExtVal();
9238 "Unexpected custom EXTRACT_SUBVECTOR lowering");
9239 assert(ST->hasMVEIntegerOps() &&
9240 "EXTRACT_SUBVECTOR lowering only supported for MVE");
9250 EVT SubVT = MVT::v4i32;
9252 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j += 2) {
9267 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j++) {
9283 assert(ST->hasMVEIntegerOps() &&
"Expected MVE!");
9284 EVT VT =
N->getValueType(0);
9285 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9286 "Expected a vector i1 type!");
9288 EVT FromVT =
Op.getValueType();
9299 if (!Subtarget->hasMVEIntegerOps())
9302 EVT ToVT =
N->getValueType(0);
9345 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9347 EVT FromVT =
N->getOperand(0).getValueType();
9348 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9359 if (!Subtarget->hasMVEIntegerOps())
9364 EVT ToVT =
N->getValueType(0);
9365 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9368 EVT FromVT =
Op.getValueType();
9369 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9380 SDValue Ext1 = Ext.getValue(1);
9383 Ext = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext);
9384 Ext1 = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext1);
9396 EVT VT =
N->getValueType(0);
9398 SDNode *BVN =
N->getOperand(0).getNode();
9403 unsigned HiElt = 1 - LoElt;
9408 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9424 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
9425 SDNode *Elt =
N->getOperand(i).getNode();
9428 unsigned HalfSize = EltSize / 2;
9430 if (!
isIntN(HalfSize,
C->getSExtValue()))
9433 if (!
isUIntN(HalfSize,
C->getZExtValue()))
9472 switch (OrigSimpleTy) {
9488 unsigned ExtOpcode) {
9511 if (ExtendedTy == LD->getMemoryVT())
9512 return DAG.
getLoad(LD->getMemoryVT(),
SDLoc(LD), LD->getChain(),
9513 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9514 LD->getMemOperand()->getFlags());
9520 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9521 LD->getMemoryVT(), LD->getAlign(),
9522 LD->getMemOperand()->getFlags());
9535 N->getOperand(0)->getValueType(0),
9541 "Expected extending load");
9547 DAG.
getNode(Opcode,
SDLoc(newLoad), LD->getValueType(0), newLoad);
9556 SDNode *BVN =
N->getOperand(0).getNode();
9558 BVN->
getValueType(0) == MVT::v4i32 &&
"expected v4i32 BUILD_VECTOR");
9566 EVT VT =
N->getValueType(0);
9572 for (
unsigned i = 0; i != NumElts; ++i) {
9573 const APInt &CInt =
N->getConstantOperandAPInt(i);
9582 unsigned Opcode =
N->getOpcode();
9584 SDNode *N0 =
N->getOperand(0).getNode();
9585 SDNode *N1 =
N->getOperand(1).getNode();
9593 unsigned Opcode =
N->getOpcode();
9595 SDNode *N0 =
N->getOperand(0).getNode();
9596 SDNode *N1 =
N->getOperand(1).getNode();
9606 EVT VT =
Op.getValueType();
9608 "unexpected type for custom-lowering ISD::MUL");
9609 SDNode *N0 =
Op.getOperand(0).getNode();
9610 SDNode *N1 =
Op.getOperand(1).getNode();
9611 unsigned NewOpc = 0;
9615 if (isN0SExt && isN1SExt)
9620 if (isN0ZExt && isN1ZExt)
9622 else if (isN1SExt || isN1ZExt) {
9639 if (VT == MVT::v2i64)
9656 "unexpected types for extended operands to VMULL");
9657 return DAG.
getNode(NewOpc,
DL, VT, Op0, Op1);
9692 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9726 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9729 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9750 EVT VT =
Op.getValueType();
9751 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9752 "unexpected type for custom-lowering ISD::SDIV");
9759 if (VT == MVT::v8i8) {
9787 EVT VT =
Op.getValueType();
9788 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9789 "unexpected type for custom-lowering ISD::UDIV");
9796 if (VT == MVT::v8i8) {
9835 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9838 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9842 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9863 EVT VT =
N->getValueType(0);
9877 Op.getOperand(1), Carry);
9891 Op.getOperand(1), Carry);
9924 bool ShouldUseSRet = Subtarget->
isAPCS_ABI();
9926 if (ShouldUseSRet) {
9936 Entry.IsSExt =
false;
9937 Entry.IsZExt =
false;
9938 Entry.IsSRet =
true;
9939 Args.push_back(Entry);
9946 Entry.IsSExt =
false;
9947 Entry.IsZExt =
false;
9948 Args.push_back(Entry);
9951 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
9961 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
9964 return CallResult.first;
9983 EVT VT =
Op.getValueType();
9984 assert((VT == MVT::i32 || VT == MVT::i64) &&
9985 "unexpected type for custom lowering DIV");
9991 const char *
Name =
nullptr;
9993 Name = (VT == MVT::i32) ?
"__rt_sdiv" :
"__rt_sdiv64";
9995 Name = (VT == MVT::i32) ?
"__rt_udiv" :
"__rt_udiv64";
10001 for (
auto AI : {1, 0}) {
10003 Arg.Node =
Op.getOperand(AI);
10004 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.
getContext());
10005 Args.push_back(Arg);
10008 CallLoweringInfo CLI(DAG);
10012 ES, std::move(Args));
10022ARMTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
10030 const bool MinSize =
ST.hasMinSize();
10031 const bool HasDivide =
ST.isThumb() ?
ST.hasDivideInThumbMode()
10032 :
ST.hasDivideInARMMode();
10036 if (
N->getOperand(0).getValueType().isVector())
10041 if (!(MinSize && HasDivide))
10054 if (Divisor.
sgt(128))
10062 assert(
Op.getValueType() == MVT::i32 &&
10063 "unexpected type for custom lowering DIV");
10069 return LowerWindowsDIVLibCall(
Op, DAG,
Signed, DBZCHK);
10075 if (
N->getValueType(0) == MVT::i32)
10083void ARMTargetLowering::ExpandDIV_Windows(
10089 assert(
Op.getValueType() == MVT::i64 &&
10090 "unexpected type for custom lowering DIV");
10107 EVT MemVT = LD->getMemoryVT();
10108 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10109 MemVT == MVT::v16i1) &&
10110 "Expected a predicate type!");
10111 assert(MemVT ==
Op.getValueType());
10113 "Expected a non-extending load");
10114 assert(LD->isUnindexed() &&
"Expected a unindexed load");
10128 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
10130 LD->getMemOperand());
10137 if (MemVT != MVT::v16i1)
10146 EVT MemVT =
LD->getMemoryVT();
10147 assert(
LD->isUnindexed() &&
"Loads should be unindexed at this point.");
10149 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10155 {LD->getChain(), LD->getBasePtr()}, MemVT,
LD->getMemOperand());
10165 EVT MemVT = ST->getMemoryVT();
10166 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10167 MemVT == MVT::v16i1) &&
10168 "Expected a predicate type!");
10169 assert(MemVT == ST->getValue().getValueType());
10170 assert(!ST->isTruncatingStore() &&
"Expected a non-extending store");
10171 assert(ST->isUnindexed() &&
"Expected a unindexed store");
10176 SDValue Build = ST->getValue();
10177 if (MemVT != MVT::v16i1) {
10196 ST->getChain(), dl, GRP, ST->getBasePtr(),
10198 ST->getMemOperand());
10204 EVT MemVT = ST->getMemoryVT();
10205 assert(ST->isUnindexed() &&
"Stores should be unindexed at this point.");
10207 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10223 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
10224 MemVT, ST->getMemOperand());
10225 }
else if (Subtarget->hasMVEIntegerOps() &&
10226 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10227 MemVT == MVT::v16i1))) {
10242 MVT VT =
Op.getSimpleValueType();
10244 SDValue PassThru =
N->getPassThru();
10255 VT, dl,
N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask, ZeroVec,
10256 N->getMemoryVT(),
N->getMemOperand(),
N->getAddressingMode(),
10257 N->getExtensionType(),
N->isExpandingLoad());
10262 if (!PassThru.
isUndef() && !PassThruIsCastZero)
10269 if (!ST->hasMVEIntegerOps())
10273 unsigned BaseOpcode = 0;
10274 switch (
Op->getOpcode()) {
10290 unsigned NumActiveLanes = NumElts;
10292 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10293 NumActiveLanes == 2) &&
10294 "Only expected a power 2 vector size");
10298 while (NumActiveLanes > 4) {
10301 Op0 = DAG.
getNode(BaseOpcode, dl, VT, Op0, Rev);
10302 NumActiveLanes /= 2;
10306 if (NumActiveLanes == 4) {
10316 SDValue Res0 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10317 SDValue Res1 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext2, Ext3,
Op->getFlags());
10318 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res0, Res1,
Op->getFlags());
10324 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10328 if (EltVT !=
Op->getValueType(0))
10335 if (!ST->hasMVEFloatOps())
10342 if (!ST->hasNEON())
10350 unsigned PairwiseIntrinsic = 0;
10351 switch (
Op->getOpcode()) {
10355 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10358 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10361 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10364 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10370 unsigned NumActiveLanes = NumElts;
10372 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10373 NumActiveLanes == 2) &&
10374 "Only expected a power 2 vector size");
10380 VT =
Lo.getValueType();
10382 NumActiveLanes /= 2;
10386 while (NumActiveLanes > 1) {
10388 NumActiveLanes /= 2;
10395 if (EltVT !=
Op.getValueType()) {
10396 unsigned Extend = 0;
10397 switch (
Op->getOpcode()) {
10409 Res = DAG.
getNode(Extend, dl,
Op.getValueType(), Res);
10431 SDValue Ops[] = {
N->getOperand(0),
10441 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
10448 SDLoc dl(V.getNode());
10449 auto [VLo, VHi] = DAG.
SplitScalar(V, dl, MVT::i32, MVT::i32);
10457 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
10459 DAG.
getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10465 assert(
N->getValueType(0) == MVT::i64 &&
10466 "AtomicCmpSwap on types less than 64 should be legal");
10467 SDValue Ops[] = {
N->getOperand(1),
10472 ARM::CMP_SWAP_64,
SDLoc(
N),
10473 DAG.
getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops);
10492 EVT VT =
Op.getValueType();
10501 if (isUnsupportedFloatingType(
LHS.getValueType())) {
10503 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS, Chain, IsSignaling);
10504 if (!
RHS.getNode()) {
10525 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10526 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
10528 ARMcc = DAG.
getConstant(CondCode2, dl, MVT::i32);
10529 Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10530 Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG);
10546 switch (
Op.getOpcode()) {
10578 case ISD::BITCAST:
return ExpandBITCAST(
Op.getNode(), DAG, Subtarget);
10582 case ISD::SREM:
return LowerREM(
Op.getNode(), DAG);
10583 case ISD::UREM:
return LowerREM(
Op.getNode(), DAG);
10605 return LowerSET_FPMODE(
Op, DAG);
10607 return LowerRESET_FPMODE(
Op, DAG);
10611 return LowerDIV_Windows(
Op, DAG,
true);
10615 return LowerDIV_Windows(
Op, DAG,
false);
10622 return LowerSignedALUO(
Op, DAG);
10625 return LowerUnsignedALUO(
Op, DAG);
10659 return LowerDYNAMIC_STACKALLOC(
Op, DAG);
10668 return LowerSPONENTRY(
Op, DAG);
10675 unsigned IntNo =
N->getConstantOperandVal(0);
10677 if (IntNo == Intrinsic::arm_smlald)
10679 else if (IntNo == Intrinsic::arm_smlaldx)
10681 else if (IntNo == Intrinsic::arm_smlsld)
10683 else if (IntNo == Intrinsic::arm_smlsldx)
10690 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(3), dl, MVT::i32, MVT::i32);
10694 N->getOperand(1),
N->getOperand(2),
10706 switch (
N->getOpcode()) {
10713 Res = ExpandBITCAST(
N, DAG, Subtarget);
10722 Res = LowerREM(
N, DAG);
10726 Res = LowerDivRem(
SDValue(
N, 0), DAG);
10780 "ROPI/RWPI not currently supported with SjLj");
10789 bool isThumb = Subtarget->isThumb();
10790 bool isThumb2 = Subtarget->
isThumb2();
10793 unsigned PCAdj = (
isThumb || isThumb2) ? 4 : 8;
10799 : &ARM::GPRRegClass;
10817 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10823 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10829 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10847 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10852 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10857 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10862 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
10868 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
10883 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10889 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10913 : &ARM::GPRnopcRegClass;
10918 unsigned MaxCSNum = 0;
10926 if (!II.isEHLabel())
10930 if (!MF->hasCallSiteLandingPad(
Sym))
continue;
10933 for (
unsigned Idx : CallSiteIdxs) {
10934 CallSiteNumToLPad[
Idx].push_back(&BB);
10935 MaxCSNum = std::max(MaxCSNum,
Idx);
10942 std::vector<MachineBasicBlock*> LPadList;
10944 LPadList.reserve(CallSiteNumToLPad.
size());
10945 for (
unsigned I = 1;
I <= MaxCSNum; ++
I) {
10948 LPadList.push_back(
MBB);
10953 assert(!LPadList.empty() &&
10954 "No landing pad destinations for the dispatch jump table!");
10968 unsigned trap_opcode;
10969 if (Subtarget->isThumb())
10970 trap_opcode = ARM::tTRAP;
10972 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
10981 MF->insert(MF->end(), DispatchBB);
10982 MF->insert(MF->end(), DispContBB);
10983 MF->insert(MF->end(), TrapBB);
10987 SetupEntryBlockForSjLj(
MI,
MBB, DispatchBB, FI);
10994 MIB =
BuildMI(DispatchBB, dl,
TII->get(ARM::Int_eh_sjlj_dispatchsetup));
11006 unsigned NumLPads = LPadList.size();
11008 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11009 BuildMI(DispatchBB, dl,
TII->get(ARM::t2LDRi12), NewVReg1)
11015 if (NumLPads < 256) {
11016 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPri))
11018 .
addImm(LPadList.size())
11021 Register VReg1 =
MRI->createVirtualRegister(TRC);
11022 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVi16), VReg1)
11023 .
addImm(NumLPads & 0xFFFF)
11026 unsigned VReg2 = VReg1;
11027 if ((NumLPads & 0xFFFF0000) != 0) {
11028 VReg2 =
MRI->createVirtualRegister(TRC);
11029 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVTi16), VReg2)
11035 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPrr))
11041 BuildMI(DispatchBB, dl,
TII->get(ARM::t2Bcc))
11046 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11047 BuildMI(DispContBB, dl,
TII->get(ARM::t2LEApcrelJT), NewVReg3)
11051 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11052 BuildMI(DispContBB, dl,
TII->get(ARM::t2ADDrs), NewVReg4)
11059 BuildMI(DispContBB, dl,
TII->get(ARM::t2BR_JT))
11063 }
else if (Subtarget->isThumb()) {
11064 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11065 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRspi), NewVReg1)
11071 if (NumLPads < 256) {
11072 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPi8))
11082 Align Alignment = MF->getDataLayout().getPrefTypeAlign(
Int32Ty);
11085 Register VReg1 =
MRI->createVirtualRegister(TRC);
11086 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRpci))
11090 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPr))
11096 BuildMI(DispatchBB, dl,
TII->get(ARM::tBcc))
11101 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
11102 BuildMI(DispContBB, dl,
TII->get(ARM::tLSLri), NewVReg2)
11108 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11109 BuildMI(DispContBB, dl,
TII->get(ARM::tLEApcrelJT), NewVReg3)
11113 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11114 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg4)
11124 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11125 BuildMI(DispContBB, dl,
TII->get(ARM::tLDRi), NewVReg5)
11131 unsigned NewVReg6 = NewVReg5;
11132 if (IsPositionIndependent) {
11133 NewVReg6 =
MRI->createVirtualRegister(TRC);
11134 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg6)
11141 BuildMI(DispContBB, dl,
TII->get(ARM::tBR_JTr))
11145 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11146 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRi12), NewVReg1)
11152 if (NumLPads < 256) {
11153 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPri))
11157 }
else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
11158 Register VReg1 =
MRI->createVirtualRegister(TRC);
11159 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVi16), VReg1)
11160 .
addImm(NumLPads & 0xFFFF)
11163 unsigned VReg2 = VReg1;
11164 if ((NumLPads & 0xFFFF0000) != 0) {
11165 VReg2 =
MRI->createVirtualRegister(TRC);
11166 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVTi16), VReg2)
11172 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11182 Align Alignment = MF->getDataLayout().getPrefTypeAlign(
Int32Ty);
11185 Register VReg1 =
MRI->createVirtualRegister(TRC);
11186 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRcp))
11191 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11202 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11203 BuildMI(DispContBB, dl,
TII->get(ARM::MOVsi), NewVReg3)
11208 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11209 BuildMI(DispContBB, dl,
TII->get(ARM::LEApcrelJT), NewVReg4)
11216 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11217 BuildMI(DispContBB, dl,
TII->get(ARM::LDRrs), NewVReg5)
11224 if (IsPositionIndependent) {
11225 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTadd))
11230 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTr))
11239 if (SeenMBBs.
insert(CurMBB).second)
11251 while (!Successors.empty()) {
11260 BB->normalizeSuccProbs();
11267 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
11268 if (!II->isCall())
continue;
11272 OI = II->operands_begin(), OE = II->operands_end();
11274 if (!OI->isReg())
continue;
11275 DefRegs[OI->getReg()] =
true;
11280 for (
unsigned i = 0; SavedRegs[i] != 0; ++i) {
11281 unsigned Reg = SavedRegs[i];
11283 !ARM::tGPRRegClass.contains(Reg) &&
11284 !ARM::hGPRRegClass.contains(Reg))
11286 if (Subtarget->
isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11288 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11301 MBBLPad->setIsEHPad(
false);
11304 MI.eraseFromParent();
11317static unsigned getLdOpcode(
unsigned LdSize,
bool IsThumb1,
bool IsThumb2) {
11319 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11320 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11322 return LdSize == 4 ? ARM::tLDRi
11323 : LdSize == 2 ? ARM::tLDRHi
11324 : LdSize == 1 ? ARM::tLDRBi : 0;
11326 return LdSize == 4 ? ARM::t2LDR_POST
11327 : LdSize == 2 ? ARM::t2LDRH_POST
11328 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11329 return LdSize == 4 ? ARM::LDR_POST_IMM
11330 : LdSize == 2 ? ARM::LDRH_POST
11331 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11336static unsigned getStOpcode(
unsigned StSize,
bool IsThumb1,
bool IsThumb2) {
11338 return StSize == 16 ? ARM::VST1q32wb_fixed
11339 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11341 return StSize == 4 ? ARM::tSTRi
11342 : StSize == 2 ? ARM::tSTRHi
11343 : StSize == 1 ? ARM::tSTRBi : 0;
11345 return StSize == 4 ? ARM::t2STR_POST
11346 : StSize == 2 ? ARM::t2STRH_POST
11347 : StSize == 1 ? ARM::t2STRB_POST : 0;
11348 return StSize == 4 ? ARM::STR_POST_IMM
11349 : StSize == 2 ? ARM::STRH_POST
11350 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11357 unsigned LdSize,
unsigned Data,
unsigned AddrIn,
11358 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11359 unsigned LdOpc =
getLdOpcode(LdSize, IsThumb1, IsThumb2);
11360 assert(LdOpc != 0 &&
"Should have a load opcode");
11367 }
else if (IsThumb1) {
11373 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11378 }
else if (IsThumb2) {
11398 unsigned StSize,
unsigned Data,
unsigned AddrIn,
11399 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11400 unsigned StOpc =
getStOpcode(StSize, IsThumb1, IsThumb2);
11401 assert(StOpc != 0 &&
"Should have a store opcode");
11403 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11408 }
else if (IsThumb1) {
11415 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11420 }
else if (IsThumb2) {
11421 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11427 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11448 unsigned SizeVal =
MI.getOperand(2).getImm();
11449 unsigned Alignment =
MI.getOperand(3).getImm();
11454 unsigned UnitSize = 0;
11459 bool IsThumb2 = Subtarget->
isThumb2();
11460 bool IsThumb = Subtarget->isThumb();
11462 if (Alignment & 1) {
11464 }
else if (Alignment & 2) {
11469 Subtarget->hasNEON()) {
11470 if ((Alignment % 16 == 0) && SizeVal >= 16)
11472 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11481 bool IsNeon = UnitSize >= 8;
11482 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11484 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11485 : UnitSize == 8 ? &ARM::DPRRegClass
11488 unsigned BytesLeft = SizeVal % UnitSize;
11489 unsigned LoopSize = SizeVal - BytesLeft;
11491 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11495 unsigned srcIn = src;
11496 unsigned destIn = dest;
11497 for (
unsigned i = 0; i < LoopSize; i+=UnitSize) {
11498 Register srcOut =
MRI.createVirtualRegister(TRC);
11499 Register destOut =
MRI.createVirtualRegister(TRC);
11500 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11502 IsThumb1, IsThumb2);
11504 IsThumb1, IsThumb2);
11512 for (
unsigned i = 0; i < BytesLeft; i++) {
11513 Register srcOut =
MRI.createVirtualRegister(TRC);
11514 Register destOut =
MRI.createVirtualRegister(TRC);
11515 Register scratch =
MRI.createVirtualRegister(TRC);
11517 IsThumb1, IsThumb2);
11519 IsThumb1, IsThumb2);
11523 MI.eraseFromParent();
11549 MF->
insert(It, loopMBB);
11550 MF->
insert(It, exitMBB);
11553 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
11563 Register varEnd =
MRI.createVirtualRegister(TRC);
11565 BuildMI(BB, dl,
TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11568 }
else if (Subtarget->genExecuteOnly()) {
11569 assert(IsThumb &&
"Non-thumb expected to have used movt");
11605 Register varLoop =
MRI.createVirtualRegister(TRC);
11606 Register varPhi =
MRI.createVirtualRegister(TRC);
11607 Register srcLoop =
MRI.createVirtualRegister(TRC);
11608 Register srcPhi =
MRI.createVirtualRegister(TRC);
11609 Register destLoop =
MRI.createVirtualRegister(TRC);
11610 Register destPhi =
MRI.createVirtualRegister(TRC);
11618 BuildMI(BB, dl,
TII->get(ARM::PHI), destPhi)
11624 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11626 IsThumb1, IsThumb2);
11628 IsThumb1, IsThumb2);
11632 BuildMI(*BB, BB->
end(), dl,
TII->get(ARM::tSUBi8), varLoop)
11640 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11649 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11658 auto StartOfExit = exitMBB->
begin();
11662 unsigned srcIn = srcLoop;
11663 unsigned destIn = destLoop;
11664 for (
unsigned i = 0; i < BytesLeft; i++) {
11665 Register srcOut =
MRI.createVirtualRegister(TRC);
11666 Register destOut =
MRI.createVirtualRegister(TRC);
11667 Register scratch =
MRI.createVirtualRegister(TRC);
11668 emitPostLd(BB, StartOfExit,
TII, dl, 1, scratch, srcIn, srcOut,
11669 IsThumb1, IsThumb2);
11670 emitPostSt(BB, StartOfExit,
TII, dl, 1, scratch, destIn, destOut,
11671 IsThumb1, IsThumb2);
11676 MI.eraseFromParent();
11688 "__chkstk is only supported on Windows");
11689 assert(Subtarget->
isThumb2() &&
"Windows on ARM requires Thumb-2 mode");
11709 switch (
TM.getCodeModel()) {
11751 MI.eraseFromParent();
11775 .
addReg(
MI.getOperand(0).getReg())
11783 MI.eraseFromParent();
11807 if (miI == BB->
end()) {
11809 if (Succ->isLiveIn(ARM::CPSR))
11815 SelectItr->addRegisterKilled(ARM::CPSR,
TRI);
11827 Register AddDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11828 BuildMI(TpEntry, Dl,
TII->get(ARM::t2ADDri), AddDestReg)
11834 Register LsrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11835 BuildMI(TpEntry, Dl,
TII->get(ARM::t2LSRri), LsrDestReg)
11841 Register TotalIterationsReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11842 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11845 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopStart))
11846 .
addUse(TotalIterationsReg)
11853 return TotalIterationsReg;
11864 Register TotalIterationsReg,
bool IsMemcpy) {
11871 SrcPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11872 CurrSrcReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11873 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), SrcPhiReg)
11881 Register DestPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11882 Register CurrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11883 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), DestPhiReg)
11890 Register LoopCounterPhiReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11891 Register RemainingLoopIterationsReg =
11892 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11893 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), LoopCounterPhiReg)
11894 .
addUse(TotalIterationsReg)
11896 .
addUse(RemainingLoopIterationsReg)
11900 Register PredCounterPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11901 Register RemainingElementsReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11902 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), PredCounterPhiReg)
11903 .
addUse(ElementCountReg)
11905 .
addUse(RemainingElementsReg)
11909 Register VccrReg =
MRI.createVirtualRegister(&ARM::VCCRRegClass);
11910 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VCTP8), VccrReg)
11911 .
addUse(PredCounterPhiReg)
11916 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2SUBri), RemainingElementsReg)
11917 .
addUse(PredCounterPhiReg)
11925 SrcValueReg =
MRI.createVirtualRegister(&ARM::MQPRRegClass);
11926 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VLDRBU8_post))
11935 SrcValueReg = OpSrcReg;
11937 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VSTRBU8_post))
11948 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11949 .
addUse(LoopCounterPhiReg)
11952 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopEnd))
11953 .
addUse(RemainingLoopIterationsReg)
11966 bool isThumb2 = Subtarget->
isThumb2();
11967 switch (
MI.getOpcode()) {
11974 case ARM::tLDR_postidx: {
11978 .
add(
MI.getOperand(2))
11979 .
add(
MI.getOperand(3))
11980 .
add(
MI.getOperand(4))
11981 .
add(
MI.getOperand(0))
11983 MI.eraseFromParent();
11987 case ARM::MVE_MEMCPYLOOPINST:
11988 case ARM::MVE_MEMSETLOOPINST: {
12018 Register OpDestReg =
MI.getOperand(0).getReg();
12019 Register OpSrcReg =
MI.getOperand(1).getReg();
12020 Register OpSizeReg =
MI.getOperand(2).getReg();
12040 if (TpExit == BB) {
12042 "block containing memcpy/memset Pseudo");
12055 bool IsMemcpy =
MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
12057 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
12072 MI.eraseFromParent();
12082 case ARM::t2STR_preidx:
12083 MI.setDesc(
TII->get(ARM::t2STR_PRE));
12085 case ARM::t2STRB_preidx:
12086 MI.setDesc(
TII->get(ARM::t2STRB_PRE));
12088 case ARM::t2STRH_preidx:
12089 MI.setDesc(
TII->get(ARM::t2STRH_PRE));
12092 case ARM::STRi_preidx:
12093 case ARM::STRBi_preidx: {
12094 unsigned NewOpc =
MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12095 : ARM::STRB_PRE_IMM;
12097 unsigned Offset =
MI.getOperand(4).getImm();
12105 .
add(
MI.getOperand(0))
12106 .
add(
MI.getOperand(1))
12107 .
add(
MI.getOperand(2))
12109 .
add(
MI.getOperand(5))
12110 .
add(
MI.getOperand(6))
12112 MI.eraseFromParent();
12115 case ARM::STRr_preidx:
12116 case ARM::STRBr_preidx:
12117 case ARM::STRH_preidx: {
12119 switch (
MI.getOpcode()) {
12121 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG;
break;
12122 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG;
break;
12123 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE;
break;
12128 MI.eraseFromParent();
12132 case ARM::tMOVCCr_pseudo: {
12150 F->insert(It, copy0MBB);
12151 F->insert(It, sinkMBB);
12154 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
12160 if (!
MI.killsRegister(ARM::CPSR) &&
12176 .
addImm(
MI.getOperand(3).getImm())
12177 .
addReg(
MI.getOperand(4).getReg());
12192 .
addReg(
MI.getOperand(1).getReg())
12194 .
addReg(
MI.getOperand(2).getReg())
12197 MI.eraseFromParent();
12202 case ARM::BCCZi64: {
12208 bool RHSisZero =
MI.getOpcode() == ARM::BCCZi64;
12213 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12217 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12223 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12227 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12237 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12246 MI.eraseFromParent();
12250 case ARM::Int_eh_sjlj_setjmp:
12251 case ARM::Int_eh_sjlj_setjmp_nofp:
12252 case ARM::tInt_eh_sjlj_setjmp:
12253 case ARM::t2Int_eh_sjlj_setjmp:
12254 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12257 case ARM::Int_eh_sjlj_setup_dispatch:
12258 EmitSjLjDispatchBlock(
MI, BB);
12281 Fn->
insert(BBI, SinkBB);
12283 Register ABSSrcReg =
MI.getOperand(1).getReg();
12284 Register ABSDstReg =
MI.getOperand(0).getReg();
12285 bool ABSSrcKIll =
MI.getOperand(1).isKill();
12286 bool isThumb2 = Subtarget->
isThumb2();
12290 Register NewRsbDstReg =
MRI.createVirtualRegister(
12291 isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
12305 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12312 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).
addMBB(SinkBB)
12319 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
12328 TII->get(ARM::PHI), ABSDstReg)
12333 MI.eraseFromParent();
12338 case ARM::COPY_STRUCT_BYVAL_I32:
12340 return EmitStructByval(
MI, BB);
12341 case ARM::WIN__CHKSTK:
12342 return EmitLowered__chkstk(
MI, BB);
12343 case ARM::WIN__DBZCHK:
12344 return EmitLowered__dbzchk(
MI, BB);
12361 if (!
Node->hasAnyUseOfValue(0)) {
12362 MI.getOperand(0).setIsDead(
true);
12364 if (!
Node->hasAnyUseOfValue(1)) {
12365 MI.getOperand(1).setIsDead(
true);
12369 for (
unsigned I = 0;
I !=
MI.getOperand(4).getImm(); ++
I) {
12370 Register TmpReg =
MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12371 : &ARM::GPRRegClass);
12378 if (
MI.getOpcode() == ARM::MEMCPY) {
12396 MCID = &
TII->get(NewOpc);
12399 MI.getDesc().getNumOperands() + 5 -
MI.getDesc().getSize()
12400 &&
"converted opcode should be the same except for cc_out"
12401 " (and, on Thumb1, pred)");
12411 MI.addOperand(
MI.getOperand(1));
12412 MI.removeOperand(1);
12416 for (
unsigned i =
MI.getNumOperands(); i--;) {
12418 if (
op.isReg() &&
op.isUse()) {
12421 MI.tieOperands(DefIdx, i);
12435 if (!
MI.hasOptionalDef() || !MCID->
operands()[ccOutIdx].isOptionalDef()) {
12436 assert(!NewOpc &&
"Optional cc_out operand required");
12441 bool definesCPSR =
false;
12442 bool deadCPSR =
false;
12443 for (
unsigned i = MCID->
getNumOperands(), e =
MI.getNumOperands(); i != e;
12447 definesCPSR =
true;
12450 MI.removeOperand(i);
12454 if (!definesCPSR) {
12455 assert(!NewOpc &&
"Optional cc_out operand required");
12458 assert(deadCPSR == !Node->hasAnyUseOfValue(1) &&
"inconsistent dead flag");
12460 assert(!
MI.getOperand(ccOutIdx).getReg() &&
12461 "expect uninitialized optional cc_out operand");
12499 switch (
N->getOpcode()) {
12500 default:
return false;
12502 CC =
N->getOperand(0);
12524 EVT VT =
N->getValueType(0);
12525 CC =
N->getOperand(0);
12526 if (
CC.getValueType() != MVT::i1 ||
CC.getOpcode() !=
ISD::SETCC)
12572 EVT VT =
N->getValueType(0);
12575 bool SwapSelectOps;
12577 NonConstantVal, DAG))
12583 OtherOp, NonConstantVal);
12589 CCOp, TrueVal, FalseVal);
12613 if (
N->getOpcode() ==
ARMISD::VTRN &&
N->getValueType(0) == MVT::v2i32)
12628 if (!
N->getValueType(0).is64BitVector())
12636 EVT VT =
N->getValueType(0);
12675 EVT VT =
N->getValueType(0);
12681 Opcode = Intrinsic::arm_neon_vpaddls;
12683 Opcode = Intrinsic::arm_neon_vpaddlu;
12711 EVT VT =
N->getValueType(0);
12726 unsigned nextIndex = 0;
12794 return DAG.
getNode(ExtOp, dl, VT, tmp);
12825 if (SRA.getOpcode() !=
ISD::SRA) {
12831 if (
auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) {
12832 if (Const->getZExtValue() != 31)
12837 if (SRA.getOperand(0) !=
Mul)
12841 SDLoc dl(AddcNode);
12842 unsigned Opcode = 0;
12848 Op0 =
Mul.getOperand(0);
12849 Op1 =
Mul.getOperand(1);
12852 Op0 =
Mul.getOperand(0);
12853 Op1 =
Mul.getOperand(1).getOperand(0);
12856 Op0 =
Mul.getOperand(0).getOperand(0);
12857 Op1 =
Mul.getOperand(1);
12860 Op0 =
Mul->getOperand(0).getOperand(0);
12861 Op1 =
Mul->getOperand(1).getOperand(0);
12870 SDValue HiMLALResult(SMLAL.getNode(), 1);
12871 SDValue LoMLALResult(SMLAL.getNode(), 0);
12877 SDValue resNode(AddcNode, 0);
12906 "Expect an ADDE or SUBE");
12910 "ADDE node has the wrong inputs");
12929 "Expect ADDC with two result values. First: i32");
12949 bool IsLeftOperandMUL =
false;
12954 IsLeftOperandMUL =
true;
12965 SDValue *LowAddSub =
nullptr;
12968 if ((AddeSubeOp0 != MULOp.
getValue(1)) && (AddeSubeOp1 != MULOp.
getValue(1)))
12971 if (IsLeftOperandMUL)
12972 HiAddSub = &AddeSubeOp1;
12974 HiAddSub = &AddeSubeOp0;
12979 if (AddcSubcOp0 == MULOp.
getValue(0)) {
12980 LoMul = &AddcSubcOp0;
12981 LowAddSub = &AddcSubcOp1;
12983 if (AddcSubcOp1 == MULOp.
getValue(0)) {
12984 LoMul = &AddcSubcOp1;
12985 LowAddSub = &AddcSubcOp0;
12993 if (AddcSubcNode == HiAddSub->getNode() ||
13009 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->
useMulOps() &&
13023 return SDValue(AddeSubeNode, 0);
13034 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13044 return SDValue(AddeSubeNode, 0);
13056 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13065 SDNode *UmlalNode =
nullptr;
13090 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13104 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13109 SDNode* AddcNode =
N->getOperand(2).getNode();
13110 SDNode* AddeNode =
N->getOperand(3).getNode();
13118 {N->getOperand(0), N->getOperand(1),
13119 AddcNode->getOperand(0), AddcNode->getOperand(1)});
13143 int32_t imm =
C->getSExtValue();
13144 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13149 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
N->getOperand(0),
RHS);
13164 int64_t imm =
C->getSExtValue();
13175 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
13176 N->getOperand(0),
RHS,
N->getOperand(2));
13188 if (!Subtarget->hasMVEIntegerOps())
13201 SetCC =
N->getOperand(0);
13204 CC = cast<CondCodeSDNode>(SetCC->
getOperand(2))->get();
13205 TrueVal =
N->getOperand(1);
13206 FalseVal =
N->getOperand(2);
13208 LHS =
N->getOperand(0);
13209 RHS =
N->getOperand(1);
13210 CC = cast<CondCodeSDNode>(
N->getOperand(4))->get();
13211 TrueVal =
N->getOperand(2);
13212 FalseVal =
N->getOperand(3);
13217 unsigned int Opcode = 0;
13246 switch (TrueVal->getOpcode()) {
13265 if (TrueVal !=
LHS || FalseVal !=
RHS)
13268 EVT LeftType =
LHS->getValueType(0);
13269 EVT RightType =
RHS->getValueType(0);
13272 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13276 if (VectorScalarType != MVT::i32)
13284 if (VectorScalarType != MVT::i32)
13297 EVT VT =
N->getValueType(0);
13305 Shft =
N->getOperand(0);
13311 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() !=
ISD::SETLT ||
13312 Cmp.getOperand(0) !=
N->getOperand(1) ||
13313 Cmp.getOperand(1) !=
N->getOperand(2))
13315 Shft =
N->getOperand(1);
13327 ScalarType = MVT::i8;
13330 case (1 << 15) - 1:
13331 ScalarType = MVT::i16;
13334 case (1ULL << 31) - 1:
13335 ScalarType = MVT::i32;
13366 unsigned LegalLanes = 128 / (ShftAmt + 1);
13390 for (
unsigned I = 0;
I < NumParts; ++
I) {
13407 if (!Subtarget->hasMVEIntegerOps())
13422 if (
N->getOperand(0).getOpcode() !=
ISD::XOR)
13432 if (!Const || !Const->isOne())
13450 EVT VT =
N->getValueType(0);
13452 if (!Subtarget->hasMVEIntegerOps() ||
13481 Opc = Intrinsic::arm_mve_vctp64;
13484 Opc = Intrinsic::arm_mve_vctp32;
13487 Opc = Intrinsic::arm_mve_vctp16;
13490 Opc = Intrinsic::arm_mve_vctp8;
13556 EVT VT =
N->getValueType(0);
13562 switch (
Op.getOpcode()) {
13579 !isa<ConstantSDNode>(N0) && N1->
hasOneUse()) {
13587 unsigned N0RedOp = 0;
13594 unsigned N1RedOp = 0;
13608 if (
SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13610 if (
SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13617 auto DistrubuteVecReduceLoad = [&](
SDValue N0,
SDValue N1,
bool IsForward) {
13631 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(N0);
13632 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(N1);
13641 if (!BaseLocDecomp0.getBase() ||
13642 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13643 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13645 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13647 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13657 if (IsBefore < 0) {
13660 }
else if (IsBefore > 0) {
13673 }
else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13683 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13693 if (
SDValue R = DistrubuteVecReduceLoad(N0, N1,
true))
13695 if (
SDValue R = DistrubuteVecReduceLoad(N1, N0,
false))
13702 if (!Subtarget->hasMVEIntegerOps())
13708 EVT VT =
N->getValueType(0);
13713 if (VT != MVT::i64)
13724 auto MakeVecReduce = [&](
unsigned Opcode,
unsigned OpcodeA,
SDValue NA,
13742 std::tie(Ops[0], Ops[1]) = DAG.
SplitScalar(NA, dl, MVT::i32, MVT::i32);
13744 unsigned S = VecRed->
getOpcode() == OpcodeA ? 2 : 0;
13793 "Expected shift op");
13811 if (
auto *Const = dyn_cast<ConstantSDNode>(N1->
getOperand(1))) {
13812 if (Const->getAPIntValue().ult(256))
13815 Const->getAPIntValue().sgt(-256))
13831 (
N->getOperand(0).getOpcode() ==
ISD::SHL ||
13832 N->getOperand(0).getOpcode() ==
ISD::SRL) &&
13833 "Expected XOR(SHIFT) pattern");
13836 auto *XorC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13837 auto *ShiftC = dyn_cast<ConstantSDNode>(
N->getOperand(0).getOperand(1));
13838 if (XorC && ShiftC) {
13839 unsigned MaskIdx, MaskLen;
13840 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13841 unsigned ShiftAmt = ShiftC->getZExtValue();
13842 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
13843 if (
N->getOperand(0).getOpcode() ==
ISD::SHL)
13844 return MaskIdx == ShiftAmt && MaskLen == (
BitWidth - ShiftAmt);
13845 return MaskIdx == 0 && MaskLen == (
BitWidth - ShiftAmt);
13855 N->getOperand(0).getOpcode() ==
ISD::SRL) ||
13857 N->getOperand(0).getOpcode() ==
ISD::SHL)) &&
13858 "Expected shift-shift mask");
13871 return Subtarget->hasMVEIntegerOps() &&
isTypeLegal(VT);
13875 if (!Subtarget->hasNEON()) {
13894 return Subtarget->hasFP64();
13897 return Subtarget->hasMVEFloatOps();
13926 if (ST->isThumb() && ST->isThumb1Only())
13930 for (
auto *U :
N->uses()) {
13931 switch(U->getOpcode()) {
13944 if (isa<ConstantSDNode>(U->getOperand(0)) ||
13945 isa<ConstantSDNode>(U->getOperand(1)))
13949 if (U->getOperand(0).getOpcode() ==
ISD::SHL ||
13950 U->getOperand(1).getOpcode() ==
ISD::SHL)
13960 if (
N->getOperand(0).getOpcode() !=
ISD::SHL)
13965 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13966 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1));
13967 if (!C1ShlC2 || !C2)
13970 APInt C2Int = C2->getAPIntValue();
13971 APInt C1Int = C1ShlC2->getAPIntValue();
13973 if (C2Int.
uge(C2Width))
13979 if ((C1Int & Mask) != C1Int)
13986 auto LargeImm = [](
const APInt &Imm) {
13987 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
13988 return Imm.getBitWidth() - Zeros > 8;
13991 if (LargeImm(C1Int) || LargeImm(C2Int))
14003 SHL.dump();
N->dump());
14038 if (CSINC.getOpcode() !=
ARMISD::CSINC || !CSINC.hasOneUse())
14047 CSINC.getOperand(0)),
14048 CSINC.getOperand(1), CSINC.getOperand(2),
14049 CSINC.getOperand(3));
14068 if (!Subtarget->hasMVEIntegerOps() || !
N->getValueType(0).isVector())
14110 if (!Subtarget->hasVMLxForwarding())
14129 EVT VT =
N->getValueType(0);
14140 EVT VT =
N->getValueType(0);
14141 if (VT != MVT::v2i64)
14150 EVT VT = cast<VTSDNode>(
Op->getOperand(1))->getVT();
14152 return Op->getOperand(0);
14166 And =
And->getOperand(0);
14171 Mask = Mask->getOperand(0);
14174 Mask.getValueType() != MVT::v4i32)
14180 return And->getOperand(0);
14185 if (
SDValue Op0 = IsSignExt(N0)) {
14186 if (
SDValue Op1 = IsSignExt(N1)) {
14192 if (
SDValue Op0 = IsZeroExt(N0)) {
14193 if (
SDValue Op1 = IsZeroExt(N1)) {
14208 EVT VT =
N->getValueType(0);
14209 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14220 if (VT != MVT::i32)
14227 int64_t MulAmt =
C->getSExtValue();
14228 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(MulAmt);
14230 ShiftAmt = ShiftAmt & (32 - 1);
14235 MulAmt >>= ShiftAmt;
14238 if (llvm::has_single_bit<uint32_t>(MulAmt - 1)) {
14246 }
else if (llvm::has_single_bit<uint32_t>(MulAmt + 1)) {
14258 if (llvm::has_single_bit<uint32_t>(MulAmtAbs + 1)) {
14266 }
else if (llvm::has_single_bit<uint32_t>(MulAmtAbs - 1)) {
14296 if (
N->getValueType(0) != MVT::i32)
14305 if (C1 == 255 || C1 == 65535)
14308 SDNode *N0 =
N->getOperand(0).getNode();
14322 if (!C2 || C2 >= 32)
14366 if (Trailing == C2 && C2 + C3 < 32) {
14379 if (Leading == C2 && C2 + C3 < 32) {
14399 EVT VT =
N->getValueType(0);
14403 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14406 APInt SplatBits, SplatUndef;
14407 unsigned SplatBitSize;
14409 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14410 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14411 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14412 SplatBitSize == 64) {
14446 if (!Subtarget->hasV6Ops() ||
14447 (Subtarget->isThumb() &&
14448 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14451 SDValue SRL = OR->getOperand(0);
14452 SDValue SHL = OR->getOperand(1);
14455 SRL = OR->getOperand(1);
14456 SHL = OR->getOperand(0);
14463 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14467 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14468 if (SRL.getOperand(0) !=
SDValue(SMULLOHI, 0) ||
14469 SHL.getOperand(0) !=
SDValue(SMULLOHI, 1))
14488 unsigned Opcode = 0;
14489 if (
isS16(OpS16, DAG))
14507 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6T2Ops())
14510 EVT VT =
N->getValueType(0);
14525 if (VT != MVT::i32)
14538 if (Mask == 0xffff)
14545 if ((Val & ~Mask) != Val)
14570 (Mask == ~Mask2)) {
14573 if (Subtarget->hasDSP() &&
14574 (Mask == 0xffff || Mask == 0xffff0000))
14587 (~Mask == Mask2)) {
14590 if (Subtarget->hasDSP() &&
14591 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14664 EVT VT =
N->getValueType(0);
14669 auto IsFreelyInvertable = [&](
SDValue V) {
14676 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14692 EVT VT =
N->getValueType(0);
14698 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14699 VT == MVT::v8i1 || VT == MVT::v16i1))
14702 APInt SplatBits, SplatUndef;
14703 unsigned SplatBitSize;
14705 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14706 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14707 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14708 SplatBitSize == 64) {
14744 unsigned SplatBitSize;
14747 APInt SplatBits0, SplatBits1;
14751 if (BVN0 && BVN0->
isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14752 HasAnyUndefs) && !HasAnyUndefs) {
14753 if (BVN1 && BVN1->
isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14754 HasAnyUndefs) && !HasAnyUndefs) {
14759 SplatBits0 == ~SplatBits1) {
14789 EVT VT =
N->getValueType(0);
14804 if (Subtarget->hasMVEIntegerOps()) {
14835 ToMask = ~N->getConstantOperandAPInt(2);
14841 isa<ConstantSDNode>(
From->getOperand(1))) {
14842 APInt Shift =
From->getConstantOperandAPInt(1);
14855 unsigned LastActiveBitInA =
A.countr_zero();
14856 unsigned FirstActiveBitInB =
B.getBitWidth() -
B.countl_zero() - 1;
14857 return LastActiveBitInA - 1 == FirstActiveBitInB;
14862 APInt ToMask, FromMask;
14870 APInt NewToMask, NewFromMask;
14872 if (NewFrom !=
From)
14876 if ((NewToMask & ToMask).getBoolValue())
14901 unsigned InvMask =
N->getConstantOperandVal(2);
14903 unsigned Width = llvm::bit_width<unsigned>(~InvMask) - LSB;
14905 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14906 "undefined behavior");
14907 unsigned Mask = (1u << Width) - 1;
14909 if ((Mask & (~Mask2)) == 0)
14911 N->getOperand(0), N1.
getOperand(0),
N->getOperand(2));
14918 APInt ToMask1, FromMask1;
14921 APInt ToMask2, FromMask2;
14927 APInt NewFromMask = FromMask1 | FromMask2;
14928 APInt NewToMask = ToMask1 | ToMask2;
14930 EVT VT =
N->getValueType(0);
14933 if (NewFromMask[0] == 0)
14945 APInt ToMask1 = ~N->getConstantOperandAPInt(2);
14946 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
14948 if (!N0.
hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14952 EVT VT =
N->getValueType(0);
14955 N->getOperand(1),
N->getOperand(2));
14969 SDValue CSInc = Cmp->getOperand(0);
15019 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ)
15020 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15023 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
15025 N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15038 SDValue InDouble =
N->getOperand(0);
15047 !cast<LoadSDNode>(InNode)->isVolatile()) {
15053 SDValue BasePtr = LD->getBasePtr();
15055 DAG.
getLoad(MVT::i32,
DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15056 LD->getAlign(), LD->getMemOperand()->getFlags());
15062 LD->getPointerInfo().getWithOffset(4),
15064 LD->getMemOperand()->getFlags());
15076 isa<ConstantSDNode>(InDouble.
getOperand(1))) {
15097 if (!Subtarget->
isLittle() && BVSwap)
15107 if (isa<ConstantSDNode>(BV.
getOperand(2))) {
15115 if (!Subtarget->
isLittle() && BVSwap)
15160 if (Copy.getValueType() == MVT::f32 &&
15162 bool HasGlue = Copy->getNumOperands() == 3;
15163 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1),
15164 HasGlue ? Copy->getOperand(2) :
SDValue()};
15165 EVT OutTys[] = {
N->getValueType(0), MVT::Other, MVT::Glue};
15183 if (
LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Op0)) {
15184 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15185 LN0->getMemoryVT() == MVT::i16) {
15188 LN0->getBasePtr(), LN0->getMemOperand());
15206 EVT VT =
N->getValueType(0);
15240 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
15241 for (
unsigned i = 0; i < NumElts; ++i) {
15242 SDNode *Elt =
N->getOperand(i).getNode();
15259 if (
N->getNumOperands() == 2)
15265 EVT VT =
N->getValueType(0);
15271 for (
unsigned i = 0; i < NumElts; ++i) {
15297 EVT VT =
N->getValueType(0);
15305 assert(EltVT == MVT::f32 &&
"Unexpected type!");
15310 Use->getValueType(0).isFloatingPoint())
15318 unsigned NumOfBitCastedElts = 0;
15320 unsigned NumOfRelevantElts = NumElts;
15321 for (
unsigned Idx = 0;
Idx < NumElts; ++
Idx) {
15326 ++NumOfBitCastedElts;
15327 }
else if (Elt.
isUndef() || isa<ConstantSDNode>(Elt))
15330 --NumOfRelevantElts;
15334 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15352 for (
unsigned Idx = 0 ;
Idx < NumElts; ++
Idx) {
15357 V->getOperand(0).getValueType() == MVT::i32)
15359 V = V.getOperand(0);
15376 EVT VT =
N->getValueType(0);
15383 if (
Op->getOperand(0).getValueType() == VT)
15384 return Op->getOperand(0);
15399 if (
Op.getValueType() == MVT::i32) {
15410 EVT VT =
N->getValueType(0);
15415 if (ST->isLittle())
15425 if (
Op->getOperand(0).getValueType() == VT)
15426 return Op->getOperand(0);
15435 if (!Subtarget->hasMVEIntegerOps())
15438 EVT VT =
N->getValueType(0);
15469 EVT VT =
N->getValueType(0);
15470 SDNode *Elt =
N->getOperand(1).getNode();
15485 Vec, V,
N->getOperand(2));
15495 EVT VT =
N->getValueType(0);
15504 Ext.getOperand(0).getValueType() == MVT::f32)
15505 Ext = Ext.getOperand(0);
15507 !isa<ConstantSDNode>(Ext.getOperand(1)) ||
15508 Ext.getConstantOperandVal(1) % 2 != 0)
15510 if (Ext->use_size() == 1 &&
15515 SDValue Op0 = Ext.getOperand(0);
15518 unsigned Lane = Ext.getConstantOperandVal(1);
15524 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15525 isa<ConstantSDNode>(V->getOperand(1)) &&
15526 V->getConstantOperandVal(1) == Lane + 1 &&
15527 V->getOperand(0).getResNo() == ResNo;
15529 if (OtherIt == Op0->
uses().end())
15534 SDValue OtherExt(*OtherIt, 0);
15547 DCI.
DAG.
getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15559 EVT VT =
N->getValueType(0);
15565 if (VT == MVT::f16 &&
X.getValueType() == MVT::i32)
15567 if (VT == MVT::i32 &&
X.getValueType() == MVT::f16)
15569 if (VT == MVT::f32 &&
X.getValueType() == MVT::i32)
15572 while (
X.getValueType() != VT &&
X->getOpcode() ==
ISD::BITCAST)
15573 X =
X->getOperand(0);
15574 if (
X.getValueType() == VT)
15580 isa<ConstantSDNode>(
N->getOperand(1)) &&
15582 return Op0.
getOperand(
N->getConstantOperandVal(1));
15587 isa<ConstantSDNode>(
N->getOperand(1)) &&
15592 unsigned Offset =
N->getConstantOperandVal(1);
15604 unsigned Idx =
N->getConstantOperandVal(1);
15618 EVT VT =
N->getValueType(0);
15622 cast<VTSDNode>(
N->getOperand(1))->getVT() ==
15623 Op.getOperand(0).getValueType().getScalarType())
15633 SDValue SubVec =
N->getOperand(1);
15634 uint64_t IdxVal =
N->getConstantOperandVal(2);
15645 if (IdxVal == 0 && Vec.
isUndef())
15651 (IdxVal != 0 && IdxVal != NumSubElts))
15724 EVT VT =
N->getValueType(0);
15735 unsigned HalfElts = NumElts/2;
15737 for (
unsigned n = 0; n < NumElts; ++n) {
15740 if (MaskElt < (
int)HalfElts)
15742 else if (MaskElt >= (
int)NumElts && MaskElt < (
int)(NumElts + HalfElts))
15743 NewElt = HalfElts + MaskElt - NumElts;
15770 bool SimpleConstIncOnly,
15778 bool isLoadOp =
true;
15779 bool isLaneOp =
false;
15782 bool hasAlignment =
true;
15783 unsigned NewOpc = 0;
15784 unsigned NumVecs = 0;
15785 if (
Target.isIntrinsic) {
15786 unsigned IntNo =
N->getConstantOperandVal(1);
15790 case Intrinsic::arm_neon_vld1:
15794 case Intrinsic::arm_neon_vld2:
15798 case Intrinsic::arm_neon_vld3:
15802 case Intrinsic::arm_neon_vld4:
15806 case Intrinsic::arm_neon_vld1x2:
15809 hasAlignment =
false;
15811 case Intrinsic::arm_neon_vld1x3:
15814 hasAlignment =
false;
15816 case Intrinsic::arm_neon_vld1x4:
15819 hasAlignment =
false;
15821 case Intrinsic::arm_neon_vld2dup:
15825 case Intrinsic::arm_neon_vld3dup:
15829 case Intrinsic::arm_neon_vld4dup:
15833 case Intrinsic::arm_neon_vld2lane:
15838 case Intrinsic::arm_neon_vld3lane:
15843 case Intrinsic::arm_neon_vld4lane:
15848 case Intrinsic::arm_neon_vst1:
15853 case Intrinsic::arm_neon_vst2:
15858 case Intrinsic::arm_neon_vst3:
15863 case Intrinsic::arm_neon_vst4:
15868 case Intrinsic::arm_neon_vst2lane:
15874 case Intrinsic::arm_neon_vst3lane:
15880 case Intrinsic::arm_neon_vst4lane:
15886 case Intrinsic::arm_neon_vst1x2:
15890 hasAlignment =
false;
15892 case Intrinsic::arm_neon_vst1x3:
15896 hasAlignment =
false;
15898 case Intrinsic::arm_neon_vst1x4:
15902 hasAlignment =
false;
15907 switch (
N->getOpcode()) {
15943 VecTy =
N->getValueType(0);
15944 }
else if (
Target.isIntrinsic) {
15945 VecTy =
N->getOperand(
Target.AddrOpIdx + 1).getValueType();
15948 "Node has to be a load, a store, or an intrinsic!");
15949 VecTy =
N->getOperand(1).getValueType();
15957 if (isLaneOp || isVLDDUPOp)
15960 if (NumBytes >= 3 * 16 &&
User.ConstInc != NumBytes) {
15966 if (SimpleConstIncOnly &&
User.ConstInc != NumBytes)
15972 EVT AlignedVecTy = VecTy;
15989 if (isa<LSBaseSDNode>(
N)) {
15992 assert(NumVecs == 1 &&
"Unexpected multi-element generic load/store.");
15993 assert(!isLaneOp &&
"Unexpected generic load/store lane.");
16004 Alignment =
Align(1);
16010 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16012 for (n = 0; n < NumResultVecs; ++n)
16013 Tys[n] = AlignedVecTy;
16014 Tys[n++] = MVT::i32;
16015 Tys[n] = MVT::Other;
16030 unsigned LastOperand =
16031 hasAlignment ?
N->getNumOperands() - 1 :
N->getNumOperands();
16032 for (
unsigned i =
Target.AddrOpIdx + 1; i < LastOperand; ++i)
16041 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::STORE) {
16052 for (
unsigned i = 0; i < NumResultVecs; ++i)
16057 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::LOAD) {
16058 SDValue &LdVal = NewResults[0];
16094 switch (
N->getOpcode()) {
16097 if (isa<ConstantSDNode>(
N->getOperand(1))) {
16098 *
Ptr =
N->getOperand(0);
16099 *CInc =
N->getOperand(1);
16105 if (isa<ConstantSDNode>(
N->getOperand(2))) {
16106 *
Ptr =
N->getOperand(1);
16107 *CInc =
N->getOperand(2);
16142 const unsigned AddrOpIdx = ((isIntrinsic ||
isStore) ? 2 : 1);
16151 UE =
Addr.getNode()->use_end(); UI != UE; ++UI) {
16153 if (UI.getUse().getResNo() !=
Addr.getResNo() ||
16158 unsigned ConstInc =
16176 if (UI.getUse().getResNo() !=
Base.getResNo() ||
User ==
Addr.getNode() ||
16181 unsigned UserOffset =
16184 if (!UserOffset || UserOffset <=
Offset)
16187 unsigned NewConstInc = UserOffset -
Offset;
16197 unsigned NumValidUpd = BaseUpdates.
size();
16198 for (
unsigned I = 0;
I < NumValidUpd;) {
16202 std::swap(BaseUpdates[
I], BaseUpdates[NumValidUpd]);
16210 BaseUpdates.
resize(NumValidUpd);
16215 std::stable_sort(BaseUpdates.
begin(), BaseUpdates.
end(),
16217 return LHS.ConstInc < RHS.ConstInc;
16246 unsigned IntNo =
N->getConstantOperandVal(1);
16247 if (IntNo == Intrinsic::arm_mve_vst2q &&
N->getConstantOperandVal(5) != 1)
16249 if (IntNo == Intrinsic::arm_mve_vst4q &&
N->getConstantOperandVal(7) != 3)
16254 UE =
Addr.getNode()->use_end();
16258 UI.getUse().getResNo() !=
Addr.getResNo())
16274 bool isLoadOp =
true;
16275 unsigned NewOpc = 0;
16276 unsigned NumVecs = 0;
16280 case Intrinsic::arm_mve_vld2q:
16284 case Intrinsic::arm_mve_vld4q:
16288 case Intrinsic::arm_mve_vst2q:
16293 case Intrinsic::arm_mve_vst4q:
16303 VecTy =
N->getValueType(0);
16305 VecTy =
N->getOperand(3).getValueType();
16319 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16321 for (n = 0; n < NumResultVecs; ++n)
16323 Tys[n++] = MVT::i32;
16324 Tys[n] = MVT::Other;
16333 for (
unsigned i = 3; i <
N->getNumOperands(); ++i)
16341 for (
unsigned i = 0; i < NumResultVecs; ++i)
16360 EVT VT =
N->getValueType(0);
16366 SDNode *VLD =
N->getOperand(0).getNode();
16369 unsigned NumVecs = 0;
16370 unsigned NewOpc = 0;
16372 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16375 }
else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16378 }
else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16391 if (UI.getUse().getResNo() == NumVecs)
16395 VLDLaneNo !=
User->getConstantOperandVal(1))
16402 for (n = 0; n < NumVecs; ++n)
16404 Tys[n] = MVT::Other;
16415 unsigned ResNo = UI.getUse().
getResNo();
16417 if (ResNo == NumVecs)
16425 std::vector<SDValue> VLDDupResults;
16426 for (
unsigned n = 0; n < NumVecs; ++n)
16440 EVT VT =
N->getValueType(0);
16443 if (Subtarget->hasMVEIntegerOps()) {
16447 ExtractVT = MVT::i32;
16449 N->getOperand(0),
N->getOperand(1));
16461 Op =
Op.getOperand(0);
16466 unsigned EltSize =
Op.getScalarValueSizeInBits();
16468 unsigned Imm =
Op.getConstantOperandVal(0);
16484 if (Subtarget->hasMVEIntegerOps()) {
16487 if (
Op.getValueType() == MVT::f32)
16490 else if (
Op.getValueType() == MVT::f16)
16495 if (!Subtarget->hasNEON())
16502 if (LD &&
Op.hasOneUse() && LD->isUnindexed() &&
16503 LD->getMemoryVT() ==
N->getValueType(0).getVectorElementType()) {
16504 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16509 LD->getMemoryVT(), LD->getMemOperand());
16520 EVT VT =
N->getValueType(0);
16542 assert(StVT != VT &&
"Cannot truncate to the same type");
16552 if (0 != (NumElems * FromEltSz) % ToEltSz)
16555 unsigned SizeRatio = FromEltSz / ToEltSz;
16560 NumElems * SizeRatio);
16566 for (
unsigned i = 0; i < NumElems; ++i)
16580 MVT StoreType = MVT::i8;
16582 if (TLI.
isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16602 for (
unsigned I = 0;
I <
E;
I++) {
16633 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16636 unsigned NumElements = 4;
16653 unsigned Off0 = Rev ? NumElts : 0;
16654 unsigned Off1 = Rev ? 0 : NumElts;
16656 for (
unsigned I = 0;
I < NumElts;
I += 2) {
16657 if (M[
I] >= 0 && M[
I] != (
int)(Off0 +
I / 2))
16659 if (M[
I + 1] >= 0 && M[
I + 1] != (
int)(Off1 +
I / 2))
16666 if (
auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.
getOperand(0)))
16667 if (isVMOVNShuffle(Shuffle,
false) || isVMOVNShuffle(Shuffle,
true))
16687 unsigned NewOffset = i * NumElements * ToEltVT.
getSizeInBits() / 8;
16702 NewToVT, Alignment, MMOFlags, AAInfo);
16735 unsigned NewOffset =
16743 NewToVT, Alignment, MMOFlags, AAInfo);
16765 {Extract.getOperand(0), Extract.getOperand(1)});
16796 if (Subtarget->hasNEON())
16800 if (Subtarget->hasMVEFloatOps())
16804 if (Subtarget->hasMVEIntegerOps()) {
16880 if (!Subtarget->hasNEON())
16884 if (!
Op.getValueType().isVector() || !
Op.getValueType().isSimple() ||
16889 if (!isa<BuildVectorSDNode>(ConstVec))
16892 MVT FloatTy =
Op.getSimpleValueType().getVectorElementType();
16894 MVT IntTy =
N->getSimpleValueType(0).getVectorElementType();
16896 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
16897 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16908 if (
C == -1 ||
C == 0 ||
C > 32)
16913 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
16914 Intrinsic::arm_neon_vcvtfp2fxu;
16917 DAG.
getConstant(IntrinsicOpcode, dl, MVT::i32),
Op->getOperand(0),
16920 if (IntBits < FloatBits)
16928 if (!Subtarget->hasMVEFloatOps())
16936 EVT VT =
N->getValueType(0);
16941 auto isIdentitySplat = [&](
SDValue Op,
bool NSZ) {
16945 uint64_t ImmVal =
Op.getOperand(0).getConstantOperandVal(0);
16946 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
16948 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
16961 if (!isIdentitySplat(Op1.
getOperand(2), NSZ))
16972 EVT VT =
N->getValueType(0);
16975 if (!
N->getFlags().hasAllowReassociation())
16982 unsigned Opc =
A.getConstantOperandVal(0);
16983 if (Opc != Intrinsic::arm_mve_vcmlaq)
16988 A.getOperand(3),
A.getOperand(4));
17020 if (!Subtarget->hasNEON())
17024 unsigned OpOpcode =
Op.getNode()->getOpcode();
17025 if (!
N->getValueType(0).isVector() || !
N->getValueType(0).isSimple() ||
17029 SDValue ConstVec =
N->getOperand(1);
17030 if (!isa<BuildVectorSDNode>(ConstVec))
17033 MVT FloatTy =
N->getSimpleValueType(0).getVectorElementType();
17035 MVT IntTy =
Op.getOperand(0).getSimpleValueType().getVectorElementType();
17037 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
17038 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17049 if (
C == -1 ||
C == 0 ||
C > 32)
17055 if (IntBits < FloatBits)
17057 dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
17060 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfxs2fp :
17061 Intrinsic::arm_neon_vcvtfxu2fp;
17070 if (!ST->hasMVEIntegerOps())
17074 EVT ResVT =
N->getValueType(0);
17102 EVT AVT =
A.getValueType();
17108 auto ExtendIfNeeded = [&](
SDValue A,
unsigned ExtendCode) {
17109 EVT AVT =
A.getValueType();
17121 if (ExtTypeMatches(
A, ExtTypes))
17122 return ExtendIfNeeded(
A, ExtendCode);
17125 auto IsPredVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
17132 if (Ext->getOpcode() != ExtendCode)
17135 if (ExtTypeMatches(
A, ExtTypes))
17136 return ExtendIfNeeded(
A, ExtendCode);
17149 if (ResVT !=
RetTy)
17152 if (
Mul->getOpcode() == ExtendCode &&
17153 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17155 Mul =
Mul->getOperand(0);
17164 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17165 A = ExtendIfNeeded(
A, ExtendCode);
17166 B = ExtendIfNeeded(
B, ExtendCode);
17184 if (
Mul->getOpcode() == ExtendCode &&
17185 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17187 Mul =
Mul->getOperand(0);
17196 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17197 A = ExtendIfNeeded(
A, ExtendCode);
17198 B = ExtendIfNeeded(
B, ExtendCode);
17209 EVT VT = Ops[0].getValueType();
17210 if (VT == MVT::v16i8) {
17212 "Unexpected illegal long reduction opcode");
17217 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17220 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17241 if (IsVMLAV(MVT::i64,
ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17244 if (IsVMLAV(MVT::i64,
ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17308 Op =
Op->getOperand(1);
17310 Op->getOperand(0)->getOpcode() ==
ISD::MUL) {
17312 if (
Mul->getOperand(0) ==
Mul->getOperand(1) &&
17329 unsigned VecOp =
N->getOperand(0).getValueType().isVector() ? 0 : 2;
17330 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp));
17331 if (!Shuf || !Shuf->getOperand(1).isUndef())
17336 APInt SetElts(Mask.size(), 0);
17337 for (
int E : Mask) {
17338 if (E < 0 || E >= (
int)Mask.size())
17345 if (
N->getNumOperands() != VecOp + 1) {
17346 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp + 1));
17347 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask)
17353 if (
Op.getValueType().isVector())
17365 unsigned IsTop =
N->getConstantOperandVal(2);
17372 if (Op0->
isUndef() && !IsTop)
17386 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17388 APInt Op0DemandedElts =
17389 IsTop ? Op1DemandedElts
17404 unsigned IsTop =
N->getConstantOperandVal(2);
17406 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17407 APInt Op0DemandedElts =
17419 EVT VT =
N->getValueType(0);
17423 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(
LHS);
17424 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(
RHS);
17426 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
17427 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
17431 LHS.getOperand(0),
RHS.getOperand(0));
17445 if (
auto C = dyn_cast<ConstantSDNode>(
N->getOperand(2))) {
17446 int ShiftAmt =
C->getSExtValue();
17447 if (ShiftAmt == 0) {
17453 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17454 unsigned NewOpcode =
17470 unsigned IntNo =
N->getConstantOperandVal(0);
17481 case Intrinsic::arm_neon_vshifts:
17482 case Intrinsic::arm_neon_vshiftu:
17483 case Intrinsic::arm_neon_vrshifts:
17484 case Intrinsic::arm_neon_vrshiftu:
17485 case Intrinsic::arm_neon_vrshiftn:
17486 case Intrinsic::arm_neon_vqshifts:
17487 case Intrinsic::arm_neon_vqshiftu:
17488 case Intrinsic::arm_neon_vqshiftsu:
17489 case Intrinsic::arm_neon_vqshiftns:
17490 case Intrinsic::arm_neon_vqshiftnu:
17491 case Intrinsic::arm_neon_vqshiftnsu:
17492 case Intrinsic::arm_neon_vqrshiftns:
17493 case Intrinsic::arm_neon_vqrshiftnu:
17494 case Intrinsic::arm_neon_vqrshiftnsu: {
17495 EVT VT =
N->getOperand(1).getValueType();
17497 unsigned VShiftOpc = 0;
17500 case Intrinsic::arm_neon_vshifts:
17501 case Intrinsic::arm_neon_vshiftu:
17506 if (
isVShiftRImm(
N->getOperand(2), VT,
false,
true, Cnt)) {
17513 case Intrinsic::arm_neon_vrshifts:
17514 case Intrinsic::arm_neon_vrshiftu:
17519 case Intrinsic::arm_neon_vqshifts:
17520 case Intrinsic::arm_neon_vqshiftu:
17525 case Intrinsic::arm_neon_vqshiftsu:
17530 case Intrinsic::arm_neon_vrshiftn:
17531 case Intrinsic::arm_neon_vqshiftns:
17532 case Intrinsic::arm_neon_vqshiftnu:
17533 case Intrinsic::arm_neon_vqshiftnsu:
17534 case Intrinsic::arm_neon_vqrshiftns:
17535 case Intrinsic::arm_neon_vqrshiftnu:
17536 case Intrinsic::arm_neon_vqrshiftnsu:
17548 case Intrinsic::arm_neon_vshifts:
17549 case Intrinsic::arm_neon_vshiftu:
17552 case Intrinsic::arm_neon_vrshifts:
17555 case Intrinsic::arm_neon_vrshiftu:
17558 case Intrinsic::arm_neon_vrshiftn:
17561 case Intrinsic::arm_neon_vqshifts:
17564 case Intrinsic::arm_neon_vqshiftu:
17567 case Intrinsic::arm_neon_vqshiftsu:
17570 case Intrinsic::arm_neon_vqshiftns:
17573 case Intrinsic::arm_neon_vqshiftnu:
17576 case Intrinsic::arm_neon_vqshiftnsu:
17579 case Intrinsic::arm_neon_vqrshiftns:
17582 case Intrinsic::arm_neon_vqrshiftnu:
17585 case Intrinsic::arm_neon_vqrshiftnsu:
17591 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17592 N->getOperand(1), DAG.
getConstant(Cnt, dl, MVT::i32));
17595 case Intrinsic::arm_neon_vshiftins: {
17596 EVT VT =
N->getOperand(1).getValueType();
17598 unsigned VShiftOpc = 0;
17602 else if (
isVShiftRImm(
N->getOperand(3), VT,
false,
true, Cnt))
17609 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17610 N->getOperand(1),
N->getOperand(2),
17614 case Intrinsic::arm_neon_vqrshifts:
17615 case Intrinsic::arm_neon_vqrshiftu:
17619 case Intrinsic::arm_mve_vqdmlah:
17620 case Intrinsic::arm_mve_vqdmlash:
17621 case Intrinsic::arm_mve_vqrdmlah:
17622 case Intrinsic::arm_mve_vqrdmlash:
17623 case Intrinsic::arm_mve_vmla_n_predicated:
17624 case Intrinsic::arm_mve_vmlas_n_predicated:
17625 case Intrinsic::arm_mve_vqdmlah_predicated:
17626 case Intrinsic::arm_mve_vqdmlash_predicated:
17627 case Intrinsic::arm_mve_vqrdmlah_predicated:
17628 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17633 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
17640 case Intrinsic::arm_mve_minv:
17641 case Intrinsic::arm_mve_maxv:
17642 case Intrinsic::arm_mve_minav:
17643 case Intrinsic::arm_mve_maxav:
17644 case Intrinsic::arm_mve_minv_predicated:
17645 case Intrinsic::arm_mve_maxv_predicated:
17646 case Intrinsic::arm_mve_minav_predicated:
17647 case Intrinsic::arm_mve_maxav_predicated: {
17650 unsigned BitWidth =
N->getOperand(2)->getValueType(0).getScalarSizeInBits();
17657 case Intrinsic::arm_mve_addv: {
17660 bool Unsigned =
N->getConstantOperandVal(2);
17665 case Intrinsic::arm_mve_addlv:
17666 case Intrinsic::arm_mve_addlv_predicated: {
17669 bool Unsigned =
N->getConstantOperandVal(2);
17670 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17675 for (
unsigned i = 1, e =
N->getNumOperands(); i < e; i++)
17698 EVT VT =
N->getValueType(0);
17700 if (ST->isThumb1Only() &&
N->getOpcode() ==
ISD::SHL && VT == MVT::i32 &&
17701 N->getOperand(0)->getOpcode() ==
ISD::AND &&
17702 N->getOperand(0)->hasOneUse()) {
17710 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17719 if (AndMask == 255 || AndMask == 65535)
17723 if (MaskedBits > ShiftAmt) {
17738 if (ST->hasMVEIntegerOps())
17743 switch (
N->getOpcode()) {
17756 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
17757 unsigned VShiftOpc =
17760 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
17776 if (!LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed() ||
17779 EVT FromVT = LD->getValueType(0);
17780 EVT ToVT =
N->getValueType(0);
17787 unsigned NumElements = 0;
17788 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
17790 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
17792 if (NumElements == 0 ||
17802 SDValue BasePtr = LD->getBasePtr();
17803 Align Alignment = LD->getOriginalAlign();
17824 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
17825 Alignment, MMOFlags, AAInfo);
17831 if (FromEltVT == MVT::f16) {
17834 for (
unsigned i = 0; i < Loads.
size(); i++) {
17860 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
17864 EVT VT =
N->getValueType(0);
17868 if (VT == MVT::i32 &&
17869 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
17871 isa<ConstantSDNode>(Lane)) {
17874 switch (
N->getOpcode()) {
17888 if (ST->hasMVEIntegerOps())
17897 if (ST->hasMVEFloatOps())
17908 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
17912 EVT VT =
Op.getValueType();
17915 if (VT != MVT::i32 ||
17917 !isa<ConstantSDNode>(
Op.getOperand(1)) ||
17928 APInt MaxC = Max.getConstantOperandAPInt(1);
17931 !(MinC + 1).isPowerOf2())
17949 EVT VT =
N->getValueType(0);
17952 if (VT == MVT::i32)
17955 if (!ST->hasMVEIntegerOps())
17961 if (VT != MVT::v4i32 && VT != MVT::v8i16)
17964 auto IsSignedSaturate = [&](
SDNode *Min,
SDNode *Max) {
17972 if (VT == MVT::v4i32)
17973 SaturateC =
APInt(32, (1 << 15) - 1,
true);
17975 SaturateC =
APInt(16, (1 << 7) - 1,
true);
17982 MaxC != ~SaturateC)
17987 if (IsSignedSaturate(
N, N0.
getNode())) {
17990 if (VT == MVT::v4i32) {
17991 HalfVT = MVT::v8i16;
17992 ExtVT = MVT::v4i16;
17994 HalfVT = MVT::v16i8;
18009 auto IsUnsignedSaturate = [&](
SDNode *Min) {
18015 if (VT == MVT::v4i32)
18016 SaturateC =
APInt(32, (1 << 16) - 1,
true);
18018 SaturateC =
APInt(16, (1 << 8) - 1,
true);
18027 if (IsUnsignedSaturate(
N)) {
18031 if (VT == MVT::v4i32) {
18032 HalfVT = MVT::v8i16;
18033 ExtConst = 0x0000FFFF;
18035 HalfVT = MVT::v16i8;
18057 const APInt *CV = &
C->getAPIntValue();
18075 SDValue Op0 = CMOV->getOperand(0);
18076 SDValue Op1 = CMOV->getOperand(1);
18077 auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue();
18078 SDValue CmpZ = CMOV->getOperand(4);
18114 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18121 if ((OrCI & Known.
Zero) != OrCI)
18127 EVT VT =
X.getValueType();
18128 unsigned BitInX = AndC->
logBase2();
18136 for (
unsigned BitInY = 0, NumActiveBits = OrCI.
getActiveBits();
18137 BitInY < NumActiveBits; ++BitInY) {
18138 if (OrCI[BitInY] == 0)
18141 Mask.setBit(BitInY);
18158 switch (
N->getOpcode()) {
18162 if (!isa<ConstantSDNode>(
N.getOperand(1)))
18164 if (!cast<ConstantSDNode>(
N.getOperand(1))->isOne())
18170 auto *Const = dyn_cast<ConstantSDNode>(
N.getOperand(1));
18173 if (Const->isZero())
18175 else if (Const->isOne())
18179 CC = cast<CondCodeSDNode>(
N.getOperand(2))->get();
18183 unsigned IntOp =
N.getConstantOperandVal(1);
18184 if (IntOp != Intrinsic::test_start_loop_iterations &&
18185 IntOp != Intrinsic::loop_decrement_reg)
18211 bool Negate =
false;
18217 Cond =
N->getOperand(1);
18218 Dest =
N->getOperand(2);
18221 CC = cast<CondCodeSDNode>(
N->getOperand(1))->get();
18222 Cond =
N->getOperand(2);
18223 Dest =
N->getOperand(4);
18224 if (
auto *Const = dyn_cast<ConstantSDNode>(
N->getOperand(3))) {
18225 if (!Const->isOne() && !Const->isZero())
18227 Imm = Const->getZExtValue();
18255 assert((IsTrueIfZero(
CC, Imm) || IsFalseIfZero(
CC, Imm)) &&
18256 "unsupported condition");
18261 unsigned IntOp =
Int->getConstantOperandVal(1);
18263 &&
"expected single br user");
18264 SDNode *Br = *
N->use_begin();
18274 if (IntOp == Intrinsic::test_start_loop_iterations) {
18278 if (IsTrueIfZero(
CC, Imm)) {
18279 SDValue Ops[] = {Chain, Setup, Dest};
18284 UpdateUncondBr(Br, Dest, DAG);
18286 SDValue Ops[] = {Chain, Setup, OtherTarget};
18299 DAG.
getVTList(MVT::i32, MVT::Other), Args);
18307 if (
Target == OtherTarget)
18308 UpdateUncondBr(Br, Dest, DAG);
18327 EVT VT =
N->getValueType(0);
18346 LHS->getOperand(0)->getOperand(3),
LHS->getOperand(0)->getOperand(4));
18360 EVT VT =
N->getValueType(0);
18364 SDValue FalseVal =
N->getOperand(0);
18365 SDValue TrueVal =
N->getOperand(1);
18370 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops()) {
18396 N->getOperand(3), Cmp);
18401 N->getOperand(3), NewCmp);
18410 LHS->getOperand(2),
LHS->getOperand(3),
18411 LHS->getOperand(4));
18421 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ ||
18425 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
18427 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
18430 N->getOperand(3),
C);
18437 if (!Subtarget->
isThumb1Only() && Subtarget->hasV5TOps()) {
18473 N->getOperand(3), CPSRGlue.
getValue(1));
18488 N->getOperand(3), CPSRGlue.
getValue(1));
18506 const APInt *TrueConst;
18509 FalseVal.getOperand(0) ==
LHS && FalseVal.getOperand(1) ==
RHS) ||
18513 unsigned ShiftAmount = TrueConst->
logBase2();
18528 if (Known.
Zero == 0xfffffffe)
18531 else if (Known.
Zero == 0xffffff00)
18534 else if (Known.
Zero == 0xffff0000)
18547 EVT DstVT =
N->getValueType(0);
18550 if (ST->hasMVEIntegerOps() && Src.getOpcode() ==
ARMISD::VDUP) {
18551 EVT SrcVT = Src.getValueType();
18559 Src = Src.getOperand(0);
18563 EVT SrcVT = Src.getValueType();
18583 EVT VT =
N->getValueType(0);
18591 if (
N->getNumOperands() == 2 &&
18595 N->getOperand(0).getOperand(1),
18596 N->getOperand(1).getOperand(0),
18597 N->getOperand(1).getOperand(1));
18600 if (
N->getNumOperands() == 2 &&
18603 auto *S0 = cast<ShuffleVectorSDNode>(
N->getOperand(0).getNode());
18604 auto *
S1 = cast<ShuffleVectorSDNode>(
N->getOperand(1).getNode());
18606 if (S0->getOperand(0) ==
S1->getOperand(0) &&
18607 S0->getOperand(1) ==
S1->getOperand(1)) {
18610 Mask.append(
S1->getMask().begin(),
S1->getMask().end());
18630 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18631 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18632 (Op.getOpcode() == ISD::BITCAST &&
18633 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
18636 for (
unsigned Op = 0;
Op <
N->getNumOperands();
Op++) {
18638 for (
unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18655 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18656 int NumIns =
N->getNumOperands();
18657 assert((NumIns == 2 || NumIns == 4) &&
18658 "Expected 2 or 4 inputs to an MVETrunc");
18660 if (
N->getNumOperands() == 4)
18664 for (
int I = 0;
I < NumIns;
I++) {
18666 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18686 if (!LD || !LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed())
18689 EVT FromVT = LD->getMemoryVT();
18690 EVT ToVT =
N->getValueType(0);
18697 unsigned NumElements = 0;
18698 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18700 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18702 assert(NumElements != 0);
18708 LD->getExtensionType() != NewExtType)
18715 SDValue BasePtr = LD->getBasePtr();
18716 Align Alignment = LD->getOriginalAlign();
18735 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18736 Alignment, MMOFlags, AAInfo);
18752 EVT VT =
N->getValueType(0);
18754 assert(
N->getNumValues() == 2 &&
"Expected MVEEXT with 2 elements");
18755 assert((VT == MVT::v4i32 || VT == MVT::v8i16) &&
"Unexpected MVEEXT type");
18757 EVT ExtVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18759 auto Extend = [&](
SDValue V) {
18769 SDValue Ext = Extend(
N->getOperand(0));
18774 if (
auto *SVN = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(0))) {
18777 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
18782 auto CheckInregMask = [&](
int Start,
int Offset) {
18790 if (CheckInregMask(0, 0))
18792 else if (CheckInregMask(0, 1))
18793 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18794 else if (CheckInregMask(0, Mask.size()))
18796 else if (CheckInregMask(0, Mask.size() + 1))
18797 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18802 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18806 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18813 if (
N->getOperand(0)->getOpcode() ==
ISD::LOAD)
18823 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18824 int NumOuts =
N->getNumValues();
18825 assert((NumOuts == 2 || NumOuts == 4) &&
18826 "Expected 2 or 4 outputs to an MVEEXT");
18827 EVT LoadVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18829 if (
N->getNumOperands() == 4)
18835 StackPtr, MPI,
Align(4));
18838 for (
int I = 0;
I < NumOuts;
I++) {
18840 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18841 DAG.
getConstant(
I * 16 / NumOuts,
DL, StackPtr.getValueType()));
18846 VT, Chain,
Ptr, MPI, LoadVT,
Align(4));
18855 switch (
N->getOpcode()) {
18973 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
18980 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
18991 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
18999 unsigned LowWidth =
N->getOperand(0).getValueType().getSizeInBits();
19001 unsigned HighWidth =
N->getOperand(1).getValueType().getSizeInBits();
19009 unsigned HighWidth =
N->getOperand(0).getValueType().getSizeInBits();
19011 unsigned LowWidth =
N->getOperand(1).getValueType().getSizeInBits();
19019 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19030 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19039 switch (
N->getConstantOperandVal(1)) {
19040 case Intrinsic::arm_neon_vld1:
19041 case Intrinsic::arm_neon_vld1x2:
19042 case Intrinsic::arm_neon_vld1x3:
19043 case Intrinsic::arm_neon_vld1x4:
19044 case Intrinsic::arm_neon_vld2:
19045 case Intrinsic::arm_neon_vld3:
19046 case Intrinsic::arm_neon_vld4:
19047 case Intrinsic::arm_neon_vld2lane:
19048 case Intrinsic::arm_neon_vld3lane:
19049 case Intrinsic::arm_neon_vld4lane:
19050 case Intrinsic::arm_neon_vld2dup:
19051 case Intrinsic::arm_neon_vld3dup:
19052 case Intrinsic::arm_neon_vld4dup:
19053 case Intrinsic::arm_neon_vst1:
19054 case Intrinsic::arm_neon_vst1x2:
19055 case Intrinsic::arm_neon_vst1x3:
19056 case Intrinsic::arm_neon_vst1x4:
19057 case Intrinsic::arm_neon_vst2:
19058 case Intrinsic::arm_neon_vst3:
19059 case Intrinsic::arm_neon_vst4:
19060 case Intrinsic::arm_neon_vst2lane:
19061 case Intrinsic::arm_neon_vst3lane:
19062 case Intrinsic::arm_neon_vst4lane:
19064 case Intrinsic::arm_mve_vld2q:
19065 case Intrinsic::arm_mve_vld4q:
19066 case Intrinsic::arm_mve_vst2q:
19067 case Intrinsic::arm_mve_vst4q:
19084 unsigned *
Fast)
const {
19093 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19095 if (AllowsUnaligned) {
19097 *
Fast = Subtarget->hasV7Ops();
19102 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19106 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->
isLittle())) {
19113 if (!Subtarget->hasMVEIntegerOps())
19117 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19118 Ty == MVT::v2i1)) {
19126 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19142 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19143 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19144 Ty == MVT::v2f64) {
19157 if ((
Op.isMemcpy() ||
Op.isZeroMemset()) && Subtarget->hasNEON() &&
19158 !FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat)) {
19160 if (
Op.size() >= 16 &&
19166 }
else if (
Op.size() >= 8 &&
19187 return (SrcBits == 64 && DestBits == 32);
19196 return (SrcBits == 64 && DestBits == 32);
19232 return Subtarget->hasFullFP16();
19242 return Ext->getType()->getScalarSizeInBits() ==
19243 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
19248 !areExtDoubled(cast<Instruction>(Ext1)) ||
19249 !areExtDoubled(cast<Instruction>(Ext2)))
19260 if (!
I->getType()->isVectorTy())
19263 if (Subtarget->hasNEON()) {
19264 switch (
I->getOpcode()) {
19265 case Instruction::Sub:
19266 case Instruction::Add: {
19278 if (!Subtarget->hasMVEIntegerOps())
19282 if (!
I->hasOneUse())
19284 auto *Sub = cast<Instruction>(*
I->users().begin());
19285 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) ==
I;
19295 switch (
I->getOpcode()) {
19296 case Instruction::Add:
19297 case Instruction::Mul:
19298 case Instruction::FAdd:
19299 case Instruction::ICmp:
19300 case Instruction::FCmp:
19302 case Instruction::FMul:
19303 return !IsFMSMul(
I);
19304 case Instruction::Sub:
19305 case Instruction::FSub:
19306 case Instruction::Shl:
19307 case Instruction::LShr:
19308 case Instruction::AShr:
19309 return Operand == 1;
19310 case Instruction::Call:
19311 if (
auto *II = dyn_cast<IntrinsicInst>(
I)) {
19312 switch (II->getIntrinsicID()) {
19313 case Intrinsic::fma:
19315 case Intrinsic::sadd_sat:
19316 case Intrinsic::uadd_sat:
19317 case Intrinsic::arm_mve_add_predicated:
19318 case Intrinsic::arm_mve_mul_predicated:
19319 case Intrinsic::arm_mve_qadd_predicated:
19320 case Intrinsic::arm_mve_vhadd:
19321 case Intrinsic::arm_mve_hadd_predicated:
19322 case Intrinsic::arm_mve_vqdmull:
19323 case Intrinsic::arm_mve_vqdmull_predicated:
19324 case Intrinsic::arm_mve_vqdmulh:
19325 case Intrinsic::arm_mve_qdmulh_predicated:
19326 case Intrinsic::arm_mve_vqrdmulh:
19327 case Intrinsic::arm_mve_qrdmulh_predicated:
19328 case Intrinsic::arm_mve_fma_predicated:
19330 case Intrinsic::ssub_sat:
19331 case Intrinsic::usub_sat:
19332 case Intrinsic::arm_mve_sub_predicated:
19333 case Intrinsic::arm_mve_qsub_predicated:
19334 case Intrinsic::arm_mve_hsub_predicated:
19335 case Intrinsic::arm_mve_vhsub:
19336 return Operand == 1;
19347 for (
auto OpIdx :
enumerate(
I->operands())) {
19348 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
19350 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
19354 if (Shuffle->
getOpcode() == Instruction::BitCast)
19355 Shuffle = dyn_cast<Instruction>(Shuffle->
getOperand(0));
19362 if (!IsSinker(
I, OpIdx.index()))
19367 for (
Use &U :
Op->uses()) {
19369 if (!IsSinker(
Insn, U.getOperandNo()))
19382 if (!Subtarget->hasMVEIntegerOps())
19400 if (
auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal.
getOperand(0))) {
19401 if (Ld->isExpandingLoad())
19405 if (Subtarget->hasMVEIntegerOps())
19450bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(
const MachineFunction &MF,
19458 return Subtarget->hasMVEFloatOps();
19476 unsigned Scale = 1;
19493 if ((V & (Scale - 1)) != 0)
19495 return isUInt<5>(V / Scale);
19502 if (VT.
isVector() && Subtarget->hasNEON())
19505 !Subtarget->hasMVEFloatOps())
19508 bool IsNeg =
false;
19514 unsigned NumBytes = std::max((
unsigned)VT.
getSizeInBits() / 8, 1U);
19517 if (VT.
isVector() && Subtarget->hasMVEIntegerOps()) {
19521 return isShiftedUInt<7,2>(V);
19524 return isShiftedUInt<7,1>(V);
19526 return isUInt<7>(V);
19533 if (VT.
isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19534 return isShiftedUInt<8, 1>(V);
19537 return isShiftedUInt<8, 2>(V);
19539 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19542 return isUInt<8>(V);
19543 return isUInt<12>(V);
19569 default:
return false;
19574 return isUInt<12>(V);
19577 return isUInt<8>(V);
19582 return isShiftedUInt<8, 2>(V);
19588 int Scale = AM.
Scale;
19593 default:
return false;
19601 Scale = Scale & ~1;
19602 return Scale == 2 || Scale == 4 || Scale == 8;
19619 if (Scale & 1)
return false;
19626 const int Scale = AM.
Scale;
19636 return (Scale == 1) || (!AM.
HasBaseReg && Scale == 2);
19652 switch (AM.
Scale) {
19669 int Scale = AM.
Scale;
19671 default:
return false;
19675 if (Scale < 0) Scale = -Scale;
19683 if (Scale == 1 || (AM.
HasBaseReg && Scale == -1))
19696 if (Scale & 1)
return false;
19709 if (!Subtarget->isThumb())
19716 return Imm >= 0 && Imm <= 255;
19725 int64_t AbsImm = std::abs(Imm);
19726 if (!Subtarget->isThumb())
19731 return AbsImm >= 0 && AbsImm <= 255;
19766 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19770 int RHSC = (int)
RHS->getZExtValue();
19771 if (RHSC < 0 && RHSC > -256) {
19781 }
else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19784 int RHSC = (int)
RHS->getZExtValue();
19785 if (RHSC < 0 && RHSC > -0x1000) {
19827 int RHSC = (int)
RHS->getZExtValue();
19828 if (RHSC < 0 && RHSC > -0x100) {
19833 }
else if (RHSC > 0 && RHSC < 0x100) {
19844 bool isSEXTLoad,
bool IsMasked,
bool isLE,
19849 if (!isa<ConstantSDNode>(
Ptr->getOperand(1)))
19855 bool CanChangeType = isLE && !IsMasked;
19858 int RHSC = (int)
RHS->getZExtValue();
19860 auto IsInRange = [&](
int RHSC,
int Limit,
int Scale) {
19861 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
19866 }
else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
19877 if (VT == MVT::v4i16) {
19878 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
19880 }
else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
19881 if (IsInRange(RHSC, 0x80, 1))
19883 }
else if (Alignment >= 4 &&
19884 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
19885 IsInRange(RHSC, 0x80, 4))
19887 else if (Alignment >= 2 &&
19888 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
19889 IsInRange(RHSC, 0x80, 2))
19891 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
19910 bool isSEXTLoad =
false;
19911 bool IsMasked =
false;
19913 Ptr = LD->getBasePtr();
19914 VT = LD->getMemoryVT();
19915 Alignment = LD->getAlign();
19917 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19918 Ptr = ST->getBasePtr();
19919 VT = ST->getMemoryVT();
19920 Alignment = ST->getAlign();
19922 Ptr = LD->getBasePtr();
19923 VT = LD->getMemoryVT();
19924 Alignment = LD->getAlign();
19928 Ptr = ST->getBasePtr();
19929 VT = ST->getMemoryVT();
19930 Alignment = ST->getAlign();
19936 bool isLegal =
false;
19938 isLegal = Subtarget->hasMVEIntegerOps() &&
19940 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
19968 bool isSEXTLoad =
false, isNonExt;
19969 bool IsMasked =
false;
19971 VT = LD->getMemoryVT();
19972 Ptr = LD->getBasePtr();
19973 Alignment = LD->getAlign();
19976 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19977 VT = ST->getMemoryVT();
19978 Ptr = ST->getBasePtr();
19979 Alignment = ST->getAlign();
19980 isNonExt = !ST->isTruncatingStore();
19982 VT = LD->getMemoryVT();
19983 Ptr = LD->getBasePtr();
19984 Alignment = LD->getAlign();
19989 VT = ST->getMemoryVT();
19990 Ptr = ST->getBasePtr();
19991 Alignment = ST->getAlign();
19992 isNonExt = !ST->isTruncatingStore();
20000 assert(
Op->getValueType(0) == MVT::i32 &&
"Non-i32 post-inc op?!");
20001 if (
Op->getOpcode() !=
ISD::ADD || !isNonExt)
20003 auto *
RHS = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
20004 if (!
RHS ||
RHS->getZExtValue() != 4)
20006 if (Alignment <
Align(4))
20010 Base =
Op->getOperand(0);
20016 bool isLegal =
false;
20018 isLegal = Subtarget->hasMVEIntegerOps() &&
20051 const APInt &DemandedElts,
20053 unsigned Depth)
const {
20056 switch (
Op.getOpcode()) {
20063 if (
Op.getResNo() == 0) {
20089 case Intrinsic::arm_ldaex:
20090 case Intrinsic::arm_ldrex: {
20091 EVT VT = cast<MemIntrinsicSDNode>(
Op)->getMemoryVT();
20105 const APInt &Mask =
Op.getConstantOperandAPInt(2);
20106 Known.
Zero &= Mask;
20112 const SDValue &SrcSV =
Op.getOperand(0);
20118 "VGETLANE index out of bounds");
20123 EVT VT =
Op.getValueType();
20130 Known = Known.
sext(DstSz);
20132 Known = Known.
zext(DstSz);
20140 Known = KnownOp.
zext(32);
20155 true,
false,
false, KnownOp1,
20181 EVT VT =
Op.getValueType();
20187 assert(VT == MVT::i32 &&
"Unexpected integer type");
20194 unsigned Mask =
C->getZExtValue();
20197 unsigned ShrunkMask = Mask & Demanded;
20198 unsigned ExpandedMask = Mask | ~Demanded;
20202 if (ShrunkMask == 0)
20208 if (ExpandedMask == ~0U)
20211 auto IsLegalMask = [ShrunkMask, ExpandedMask](
unsigned Mask) ->
bool {
20212 return (ShrunkMask & Mask) == ShrunkMask && (~ExpandedMask & Mask) == 0;
20214 auto UseMask = [Mask,
Op, VT, &TLO](
unsigned NewMask) ->
bool {
20215 if (NewMask == Mask)
20224 if (IsLegalMask(0xFF))
20225 return UseMask(0xFF);
20228 if (IsLegalMask(0xFFFF))
20229 return UseMask(0xFFFF);
20233 if (ShrunkMask < 256)
20234 return UseMask(ShrunkMask);
20238 if ((
int)ExpandedMask <= -2 && (
int)ExpandedMask >= -256)
20239 return UseMask(ExpandedMask);
20254 unsigned Depth)
const {
20255 unsigned Opc =
Op.getOpcode();
20263 if (
Op.getResNo() == 0 && !
Op->hasAnyUseOfValue(1) &&
20264 isa<ConstantSDNode>(
Op->getOperand(2))) {
20265 unsigned ShAmt =
Op->getConstantOperandVal(2);
20277 unsigned ModImm =
Op.getConstantOperandVal(1);
20278 unsigned EltBits = 0;
20280 if ((OriginalDemandedBits & Mask) == 0)
20286 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO,
Depth);
20295 if (!Subtarget->hasV6Ops())
20301 SplitString(AsmStr, AsmPieces,
";\n");
20303 switch (AsmPieces.
size()) {
20304 default:
return false;
20306 AsmStr = AsmPieces[0];
20308 SplitString(AsmStr, AsmPieces,
" \t,");
20311 if (AsmPieces.
size() == 3 &&
20312 AsmPieces[0] ==
"rev" && AsmPieces[1] ==
"$0" && AsmPieces[2] ==
"$1" &&
20313 IA->getConstraintString().compare(0, 4,
"=l,l") == 0) {
20336 if (ConstraintVT.
isVector() && Subtarget->hasNEON() &&
20348 unsigned S = Constraint.
size();
20350 switch (Constraint[0]) {
20362 }
else if (S == 2) {
20363 switch (Constraint[0]) {
20380 Value *CallOperandVal =
info.CallOperandVal;
20383 if (!CallOperandVal)
20387 switch (*constraint) {
20393 if (Subtarget->isThumb())
20407using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20411 switch (Constraint.
size()) {
20414 switch (Constraint[0]) {
20416 if (Subtarget->isThumb())
20417 return RCPair(0U, &ARM::tGPRRegClass);
20418 return RCPair(0U, &ARM::GPRRegClass);
20420 if (Subtarget->isThumb())
20421 return RCPair(0U, &ARM::hGPRRegClass);
20425 return RCPair(0U, &ARM::tGPRRegClass);
20426 return RCPair(0U, &ARM::GPRRegClass);
20428 if (VT == MVT::Other)
20430 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20431 return RCPair(0U, &ARM::SPRRegClass);
20433 return RCPair(0U, &ARM::DPRRegClass);
20435 return RCPair(0U, &ARM::QPRRegClass);
20438 if (VT == MVT::Other)
20440 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20441 return RCPair(0U, &ARM::SPR_8RegClass);
20443 return RCPair(0U, &ARM::DPR_8RegClass);
20445 return RCPair(0U, &ARM::QPR_8RegClass);
20448 if (VT == MVT::Other)
20450 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20451 return RCPair(0U, &ARM::SPRRegClass);
20453 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20455 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20461 if (Constraint[0] ==
'T') {
20462 switch (Constraint[1]) {
20466 return RCPair(0U, &ARM::tGPREvenRegClass);
20468 return RCPair(0U, &ARM::tGPROddRegClass);
20477 if (
StringRef(
"{cc}").equals_insensitive(Constraint))
20478 return std::make_pair(
unsigned(ARM::CPSR), &ARM::CCRRegClass);
20487 std::vector<SDValue> &Ops,
20492 if (Constraint.
size() != 1)
20495 char ConstraintLetter = Constraint[0];
20496 switch (ConstraintLetter) {
20499 case 'I':
case 'J':
case 'K':
case 'L':
20500 case 'M':
case 'N':
case 'O':
20505 int64_t CVal64 =
C->getSExtValue();
20506 int CVal = (int) CVal64;
20509 if (CVal != CVal64)
20512 switch (ConstraintLetter) {
20516 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20517 if (CVal >= 0 && CVal <= 65535)
20524 if (CVal >= 0 && CVal <= 255)
20526 }
else if (Subtarget->
isThumb2()) {
20545 if (CVal >= -255 && CVal <= -1)
20551 if (CVal >= -4095 && CVal <= 4095)
20564 }
else if (Subtarget->
isThumb2()) {
20587 if (CVal >= -7 && CVal < 7)
20589 }
else if (Subtarget->
isThumb2()) {
20612 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20618 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20626 if (CVal >= 0 && CVal <= 31)
20635 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20644 if (Result.getNode()) {
20645 Ops.push_back(Result);
20655 "Unhandled Opcode in getDivRemLibcall");
20661 case MVT::i8: LC =
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
20662 case MVT::i16: LC =
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
20663 case MVT::i32: LC =
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
20664 case MVT::i64: LC =
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
20673 "Unhandled Opcode in getDivRemArgList");
20678 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
20679 EVT ArgVT =
N->getOperand(i).getValueType();
20681 Entry.Node =
N->getOperand(i);
20685 Args.push_back(Entry);
20696 "Register-based DivRem lowering only");
20697 unsigned Opcode =
Op->getOpcode();
20699 "Invalid opcode for Div/Rem lowering");
20701 EVT VT =
Op->getValueType(0);
20704 if (VT == MVT::i64 && isa<ConstantSDNode>(
Op.getOperand(1))) {
20723 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20724 : Subtarget->hasDivideInARMMode();
20725 if (hasDivide &&
Op->getValueType(0).isSimple() &&
20726 Op->getSimpleValueType(0) == MVT::i32) {
20728 const SDValue Dividend =
Op->getOperand(0);
20729 const SDValue Divisor =
Op->getOperand(1);
20730 SDValue Div = DAG.
getNode(DivOpcode, dl, VT, Dividend, Divisor);
20734 SDValue Values[2] = {Div, Rem};
20766 EVT VT =
N->getValueType(0);
20768 if (VT == MVT::i64 && isa<ConstantSDNode>(
N->getOperand(1))) {
20772 Result[0], Result[1]);
20776 std::vector<Type*> RetTyParams;
20777 Type *RetTyElement;
20787 RetTyParams.push_back(RetTyElement);
20788 RetTyParams.push_back(RetTyElement);
20805 CallLoweringInfo CLI(DAG);
20809 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
20812 SDNode *ResNode = CallResult.first.getNode();
20827 "no-stack-arg-probe")) {
20829 cast<ConstantSDNode>(
Op.getOperand(2))->getMaybeAlignValue();
20838 SDValue Ops[2] = { SP, Chain };
20855 SDValue Ops[2] = { NewSP, Chain };
20860 bool IsStrict =
Op->isStrictFPOpcode();
20861 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20862 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20864 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
20865 "Unexpected type for custom-lowering FP_EXTEND");
20868 "With both FP DP and 16, any FP conversion is legal!");
20870 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
20871 "With FP16, 16 to 32 conversion is legal!");
20874 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
20879 Loc,
Op.getValueType(), SrcVal);
20892 MakeLibCallOptions CallOptions;
20894 for (
unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
20895 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
20896 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
20897 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
20901 {DstVT, MVT::Other}, {Chain, SrcVal});
20908 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20909 "Unexpected type for custom-lowering FP_EXTEND");
20910 std::tie(SrcVal, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20915 return IsStrict ? DAG.
getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
20919 bool IsStrict =
Op->isStrictFPOpcode();
20921 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20923 EVT DstVT =
Op.getValueType();
20924 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20927 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
20928 "Unexpected type for custom-lowering FP_ROUND");
20931 "With both FP DP and 16, any FP conversion is legal!");
20936 if (SrcSz == 32 && Subtarget->hasFP16())
20941 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20942 "Unexpected type for custom-lowering FP_ROUND");
20943 MakeLibCallOptions CallOptions;
20946 std::tie(Result, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20958 if (v == 0xffffffff)
20970 bool ForCodeSize)
const {
20973 if (VT == MVT::f16 && Subtarget->hasFullFP16())
20975 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
20978 if (VT == MVT::f32)
20980 if (VT == MVT::f64 && Subtarget->hasFP64())
20991 unsigned Intrinsic)
const {
20992 switch (Intrinsic) {
20993 case Intrinsic::arm_neon_vld1:
20994 case Intrinsic::arm_neon_vld2:
20995 case Intrinsic::arm_neon_vld3:
20996 case Intrinsic::arm_neon_vld4:
20997 case Intrinsic::arm_neon_vld2lane:
20998 case Intrinsic::arm_neon_vld3lane:
20999 case Intrinsic::arm_neon_vld4lane:
21000 case Intrinsic::arm_neon_vld2dup:
21001 case Intrinsic::arm_neon_vld3dup:
21002 case Intrinsic::arm_neon_vld4dup: {
21005 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21006 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21008 Info.ptrVal =
I.getArgOperand(0);
21010 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21011 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21016 case Intrinsic::arm_neon_vld1x2:
21017 case Intrinsic::arm_neon_vld1x3:
21018 case Intrinsic::arm_neon_vld1x4: {
21021 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21022 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21024 Info.ptrVal =
I.getArgOperand(
I.arg_size() - 1);
21026 Info.align.reset();
21031 case Intrinsic::arm_neon_vst1:
21032 case Intrinsic::arm_neon_vst2:
21033 case Intrinsic::arm_neon_vst3:
21034 case Intrinsic::arm_neon_vst4:
21035 case Intrinsic::arm_neon_vst2lane:
21036 case Intrinsic::arm_neon_vst3lane:
21037 case Intrinsic::arm_neon_vst4lane: {
21040 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21041 unsigned NumElts = 0;
21042 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21043 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21046 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21049 Info.ptrVal =
I.getArgOperand(0);
21051 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21052 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21057 case Intrinsic::arm_neon_vst1x2:
21058 case Intrinsic::arm_neon_vst1x3:
21059 case Intrinsic::arm_neon_vst1x4: {
21062 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21063 unsigned NumElts = 0;
21064 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21065 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21068 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21071 Info.ptrVal =
I.getArgOperand(0);
21073 Info.align.reset();
21078 case Intrinsic::arm_mve_vld2q:
21079 case Intrinsic::arm_mve_vld4q: {
21082 Type *VecTy = cast<StructType>(
I.getType())->getElementType(1);
21083 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21085 Info.ptrVal =
I.getArgOperand(0);
21092 case Intrinsic::arm_mve_vst2q:
21093 case Intrinsic::arm_mve_vst4q: {
21096 Type *VecTy =
I.getArgOperand(1)->getType();
21097 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21099 Info.ptrVal =
I.getArgOperand(0);
21106 case Intrinsic::arm_mve_vldr_gather_base:
21107 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21109 Info.ptrVal =
nullptr;
21115 case Intrinsic::arm_mve_vldr_gather_base_wb:
21116 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21118 Info.ptrVal =
nullptr;
21124 case Intrinsic::arm_mve_vldr_gather_offset:
21125 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21127 Info.ptrVal =
nullptr;
21129 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
21136 case Intrinsic::arm_mve_vstr_scatter_base:
21137 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21139 Info.ptrVal =
nullptr;
21145 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21146 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21148 Info.ptrVal =
nullptr;
21154 case Intrinsic::arm_mve_vstr_scatter_offset:
21155 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21157 Info.ptrVal =
nullptr;
21159 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(3))->getZExtValue();
21166 case Intrinsic::arm_ldaex:
21167 case Intrinsic::arm_ldrex: {
21168 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21169 Type *ValTy =
I.getParamElementType(0);
21172 Info.ptrVal =
I.getArgOperand(0);
21174 Info.align =
DL.getABITypeAlign(ValTy);
21178 case Intrinsic::arm_stlex:
21179 case Intrinsic::arm_strex: {
21180 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21181 Type *ValTy =
I.getParamElementType(1);
21184 Info.ptrVal =
I.getArgOperand(1);
21186 Info.align =
DL.getABITypeAlign(ValTy);
21190 case Intrinsic::arm_stlexd:
21191 case Intrinsic::arm_strexd:
21193 Info.memVT = MVT::i64;
21194 Info.ptrVal =
I.getArgOperand(2);
21200 case Intrinsic::arm_ldaexd:
21201 case Intrinsic::arm_ldrexd:
21203 Info.memVT = MVT::i64;
21204 Info.ptrVal =
I.getArgOperand(0);
21224 if (Bits == 0 || Bits > 32)
21230 unsigned Index)
const {
21242 if (!Subtarget->hasDataBarrier()) {
21246 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21283 if (Subtarget->preferISHSTBarriers())
21316 bool has64BitAtomicStore;
21318 has64BitAtomicStore =
false;
21319 else if (Subtarget->isThumb())
21320 has64BitAtomicStore = Subtarget->hasV7Ops();
21322 has64BitAtomicStore = Subtarget->hasV6Ops();
21324 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21338 bool has64BitAtomicLoad;
21340 has64BitAtomicLoad =
false;
21341 else if (Subtarget->isThumb())
21342 has64BitAtomicLoad = Subtarget->hasV7Ops();
21344 has64BitAtomicLoad = Subtarget->hasV6Ops();
21361 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21362 else if (Subtarget->isThumb())
21363 hasAtomicRMW = Subtarget->hasV7Ops();
21365 hasAtomicRMW = Subtarget->hasV6Ops();
21366 if (
Size <= (Subtarget->
isMClass() ? 32U : 64U) && hasAtomicRMW) {
21389 bool HasAtomicCmpXchg;
21391 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21392 else if (Subtarget->isThumb())
21393 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21395 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21397 HasAtomicCmpXchg &&
Size <= (Subtarget->
isMClass() ? 32U : 64U))
21404 return InsertFencesForAtomic;
21417 M.getOrInsertGlobal(
"__security_cookie",
21425 F->addParamAttr(0, Attribute::AttrKind::InReg);
21431 return M.getGlobalVariable(
"__security_cookie");
21438 return M.getFunction(
"__security_check_cookie");
21443 unsigned &
Cost)
const {
21445 if (!Subtarget->hasNEON())
21457 if (!isa<ConstantInt>(
Idx))
21472 return Subtarget->hasV6T2Ops();
21476 return Subtarget->hasV6T2Ops();
21481 if (!Subtarget->hasV7Ops())
21487 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21489 auto MaskVal =
unsigned(Mask->getValue().getZExtValue());
21514 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21526 Lo, Builder.
CreateShl(
Hi, ConstantInt::get(ValueTy, 32)),
"val64");
21529 Type *Tys[] = {
Addr->getType() };
21530 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21535 0,
Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
21541 if (!Subtarget->hasV7Ops())
21558 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21569 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21570 Type *Tys[] = {
Addr->getType() };
21592 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
21599 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
21602 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21610 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21618 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21621 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21626 if (Subtarget->hasNEON() && VecSize == 64)
21628 return VecSize % 128 == 0;
21632 if (Subtarget->hasNEON())
21634 if (Subtarget->hasMVEIntegerOps())
21654 "Invalid interleave factor");
21655 assert(!Shuffles.
empty() &&
"Empty shufflevector input");
21657 "Unmatched number of shufflevectors and indices");
21659 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21660 Type *EltTy = VecTy->getElementType();
21683 if (NumLoads > 1) {
21687 VecTy->getNumElements() / NumLoads);
21692 auto createLoadIntrinsic = [&](
Value *BaseAddr) {
21693 if (Subtarget->hasNEON()) {
21695 Type *Tys[] = {VecTy, PtrTy};
21696 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21697 Intrinsic::arm_neon_vld3,
21698 Intrinsic::arm_neon_vld4};
21706 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21708 assert((Factor == 2 || Factor == 4) &&
21709 "expected interleave factor of 2 or 4 for MVE");
21711 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21713 Type *Tys[] = {VecTy, PtrTy};
21719 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21728 for (
unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
21733 VecTy->getNumElements() * Factor);
21735 CallInst *VldN = createLoadIntrinsic(BaseAddr);
21739 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21741 unsigned Index = Indices[i];
21751 SubVecs[SV].push_back(SubVec);
21760 auto &SubVec = SubVecs[SVI];
21763 SVI->replaceAllUsesWith(WideVec);
21797 unsigned Factor)
const {
21799 "Invalid interleave factor");
21801 auto *VecTy = cast<FixedVectorType>(SVI->
getType());
21802 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
21804 unsigned LaneLen = VecTy->getNumElements() / Factor;
21805 Type *EltTy = VecTy->getElementType();
21808 const DataLayout &
DL = SI->getModule()->getDataLayout();
21809 Align Alignment = SI->getAlign();
21826 Type *IntTy =
DL.getIntPtrType(EltTy);
21838 Value *BaseAddr = SI->getPointerOperand();
21840 if (NumStores > 1) {
21843 LaneLen /= NumStores;
21851 auto createStoreIntrinsic = [&](
Value *BaseAddr,
21853 if (Subtarget->hasNEON()) {
21854 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
21855 Intrinsic::arm_neon_vst3,
21856 Intrinsic::arm_neon_vst4};
21857 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21858 Type *Tys[] = {PtrTy, SubVecTy};
21861 SI->getModule(), StoreInts[Factor - 2], Tys);
21869 assert((Factor == 2 || Factor == 4) &&
21870 "expected interleave factor of 2 or 4 for MVE");
21872 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
21873 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21874 Type *Tys[] = {PtrTy, SubVecTy};
21881 for (
unsigned F = 0;
F < Factor;
F++) {
21889 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
21892 if (StoreCount > 0)
21894 BaseAddr, LaneLen * Factor);
21899 for (
unsigned i = 0; i < Factor; i++) {
21900 unsigned IdxI = StoreCount * LaneLen * Factor + i;
21901 if (Mask[IdxI] >= 0) {
21905 unsigned StartMask = 0;
21906 for (
unsigned j = 1; j < LaneLen; j++) {
21907 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
21908 if (Mask[IdxJ * Factor + IdxI] >= 0) {
21909 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
21924 createStoreIntrinsic(BaseAddr, Shuffles);
21939 if (
auto *ST = dyn_cast<StructType>(Ty)) {
21940 for (
unsigned i = 0; i < ST->getNumElements(); ++i) {
21944 Members += SubMembers;
21946 }
else if (
auto *AT = dyn_cast<ArrayType>(Ty)) {
21950 Members += SubMembers * AT->getNumElements();
21961 }
else if (
auto *VT = dyn_cast<VectorType>(Ty)) {
21968 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
21970 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
21972 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
21985 return (Members > 0 && Members <= 4);
21991 const Align ABITypeAlign =
DL.getABITypeAlign(ArgTy);
21993 return ABITypeAlign;
21997 return std::min(ABITypeAlign,
DL.getStackAlignment());
22006 if (getEffectiveCallingConv(CallConv, isVarArg) !=
22016 return IsHA || IsIntArray;
22020 const Constant *PersonalityFn)
const {
22027 const Constant *PersonalityFn)
const {
22039void ARMTargetLowering::insertCopiesSplitCSR(
22043 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
22053 RC = &ARM::GPRRegClass;
22054 else if (ARM::DPRRegClass.
contains(*
I))
22055 RC = &ARM::DPRRegClass;
22065 assert(Entry->getParent()->getFunction().hasFnAttribute(
22066 Attribute::NoUnwind) &&
22067 "Function should be nounwind in insertCopiesSplitCSR!");
22068 Entry->addLiveIn(*
I);
22073 for (
auto *Exit : Exits)
22075 TII->get(TargetOpcode::COPY), *
I)
22086 return Subtarget->hasMVEIntegerOps();
22091 auto *VTy = dyn_cast<FixedVectorType>(Ty);
22096 unsigned NumElements = VTy->getNumElements();
22103 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22104 return Subtarget->hasMVEFloatOps();
22109 return Subtarget->hasMVEIntegerOps() &&
22110 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
22111 ScalarTy->isIntegerTy(32));
22123 assert(TyWidth >= 128 &&
"Width of vector type must be at least 128 bits");
22125 if (TyWidth > 128) {
22130 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22132 auto *LowerSplitA =
B.CreateShuffleVector(InputA, LowerSplitMask);
22133 auto *LowerSplitB =
B.CreateShuffleVector(InputB, LowerSplitMask);
22134 auto *UpperSplitA =
B.CreateShuffleVector(InputA, UpperSplitMask);
22135 auto *UpperSplitB =
B.CreateShuffleVector(InputB, UpperSplitMask);
22136 Value *LowerSplitAcc =
nullptr;
22137 Value *UpperSplitAcc =
nullptr;
22140 LowerSplitAcc =
B.CreateShuffleVector(
Accumulator, LowerSplitMask);
22141 UpperSplitAcc =
B.CreateShuffleVector(
Accumulator, UpperSplitMask);
22145 B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
22147 B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
22150 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
22157 ConstRotation = ConstantInt::get(IntTy, (
int)Rotation);
22160 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
22162 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
22163 {ConstRotation, InputB, InputA});
22168 auto *ConstHalving = ConstantInt::get(IntTy, 1);
22171 ConstRotation = ConstantInt::get(IntTy, 0);
22173 ConstRotation = ConstantInt::get(IntTy, 1);
22175 if (!ConstRotation)
22178 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
22179 {ConstHalving, ConstRotation, InputA, InputB});
unsigned const MachineRegisterInfo * MRI
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG, bool isSigned)
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static const MCPhysReg GPRArgRegs[]
static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1, SDValue V2, unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
static const unsigned PerfectShuffleTable[6561+1]
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
static bool isStore(int Opcode)
static bool isThumb(const MCSubtargetInfo &STI)
static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const TargetInstrInfo *TII)
MatchingStackOffset - Return true if the given stack call argument is already available in the same p...
static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG)
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total value size to 64 bits.
static cl::opt< unsigned > ConstpoolPromotionMaxSize("arm-promote-constant-max-size", cl::Hidden, cl::desc("Maximum size of constant to promote into a constant pool"), cl::init(64))
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isVTBLMask(ArrayRef< int > M, EVT VT)
static SDValue PerformSUBCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
static cl::opt< bool > EnableConstpoolPromotion("arm-promote-constant", cl::Hidden, cl::desc("Enable / disable promotion of unnamed_addr constants into " "constant pools"), cl::init(false))
static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG)
static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
static SDValue PerformExtractEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static const APInt * isPowerOf2Constant(SDValue V)
static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool isValidMVECond(unsigned CC, bool IsFloat)
static SDValue PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC)
IntCCToARMCC - Convert a DAG integer condition code to an ARM CC.
static SDValue PerformSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSTORECombine - Target-specific dag combine xforms for ISD::STORE.
static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry, SelectionDAG &DAG)
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isGTorGE(ISD::CondCode CC)
static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a vldN-lane (N > 1) intrinsic,...
static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask)
static bool isReverseMask(ArrayRef< int > M, EVT VT)
static bool isVZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of "vector_shuffle v,...
static SDValue PerformSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG)
static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc)
static bool isVTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool CanInvertMVEVCMP(SDValue N)
static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG)
static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformShiftCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
PerformShiftCombine - Checks for immediate versions of vector shifts and lowers them.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, ARMCC::CondCodes &CondCode2)
FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static EVT getVectorTyFromPredicateVector(EVT VT)
static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
static bool isSRL16(const SDValue &Op)
static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC)
static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr, SDValue Inc, const SelectionDAG &DAG)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static Register genTPEntry(MachineBasicBlock *TpEntry, MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpExit, Register OpSizeReg, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI)
Adds logic in loop entry MBB to calculate loop iteration count and adds t2WhileLoopSetup and t2WhileL...
static bool isLTorLE(ISD::CondCode CC)
static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, SelectionDAG &DAG)
static SDValue PerformBITCASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG)
static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG)
static bool hasNormalLoadOperand(SDNode *N)
hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node are normal,...
static SDValue PerformInsertEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
PerformInsertEltCombine - Target-specific dag combine xforms for ISD::INSERT_VECTOR_ELT.
static SDValue PerformVDUPLANECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVDUPLANECombine - Target-specific dag combine xforms for ARMISD::VDUPLANE.
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static cl::opt< unsigned > ConstpoolPromotionMaxTotal("arm-promote-constant-max-total", cl::Hidden, cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::init(128))
static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static RTLIB::Libcall getDivRemLibcall(const SDNode *N, MVT::SimpleValueType SVT)
static SDValue PerformABSCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG &DAG)
SkipLoadExtensionForVMULL - return a load of the original vector size that does not do any sign/zero ...
static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static const MCPhysReg GPRArgRegs[]
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, SelectionDAG &DAG)
static bool isVZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformORCombineToSMULWBT(SDNode *OR, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isVTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of "vector_shuffle v,...
static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue FindBFIToCombineWith(SDNode *N)
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, SelectionDAG &DAG)
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, bool &swpCmpOps, bool &swpVselOps)
static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isS16(const SDValue &Op, SelectionDAG &DAG)
static bool isSRA16(const SDValue &Op)
static SDValue AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue LowerInterruptReturn(SmallVectorImpl< SDValue > &RetOps, const SDLoc &DL, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, SelectionDAG &DAG)
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, SDValue &RetVal1, SDValue &RetVal2)
static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isSHL16(const SDValue &Op)
static bool isVEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseVEXT, unsigned &Imm)
static SDValue PerformMVEVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isTruncMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2)
Return the load opcode for a given load size.
static bool isLegalT2AddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
static bool isLegalMVEShuffleOp(unsigned PFEntry)
static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, SelectionDAG &DAG)
static bool isVUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG)
PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for ISD::VECTOR_SHUFFLE.
static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG)
SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, ANY_EXTEND,...
static bool isVMOVNTruncMask(ArrayRef< int > M, EVT ToVT, bool rev)
static SDValue PerformVQMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformAddcSubcCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static TargetLowering::ArgListTy getDivRemArgList(const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget)
static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static ARMCC::CondCodes getVCMPCondCode(SDValue N)
static cl::opt< bool > ARMInterworking("arm-interworking", cl::Hidden, cl::desc("Enable / disable ARM interworking (for debugging only)"), cl::init(true))
static void ReplaceREADCYCLECOUNTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformORCombineToBFI(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC, bool &Invert, SDValue &OtherOp, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVSetCCToVCTPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isZeroVector(SDValue N)
static SDValue PerformAddeSubeCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void ReplaceCMP_SWAP_64Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, const SDValue TrueVal, const SDValue FalseVal, const ISD::CondCode CC, const SDValue K)
static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG)
static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned StSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment store operation with given size.
static bool isVMOVNMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue CombineBaseUpdate(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, NEON load/store intrinsics,...
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVRRDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMOVRRDCombine - Target-specific dag combine xforms for ARMISD::VMOVRRD.
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain)
static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue PerformVMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMULCombine Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the special multi...
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformORCombine - Target-specific dag combine xforms for ISD::OR.
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, SelectionDAG &DAG)
static unsigned SelectPairHalf(unsigned Elements, ArrayRef< int > Mask, unsigned Index)
static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned LdSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment load operation with given size.
static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG)
static bool isValidBaseUpdate(SDNode *N, SDNode *User)
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, const ARMSubtarget *ST, const SDLoc &dl)
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op)
static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformXORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, bool isSEXTLoad, bool IsMasked, bool isLE, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
std::pair< unsigned, const TargetRegisterClass * > RCPair
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,...
static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
cl::opt< unsigned > MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::init(2))
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, unsigned SplatBitSize, SelectionDAG &DAG, const SDLoc &dl, EVT &VT, EVT VectorVT, VMOVModImmType type)
isVMOVModifiedImm - Check if the specified splat value corresponds to a valid vector constant for a N...
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, SelectionDAG &DAG)
BC is a bitcast that is about to be turned into a VMOVDRR.
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, const GlobalValue *GV, SelectionDAG &DAG, EVT PtrVT, const SDLoc &dl)
static unsigned isNEONTwoResultShuffleMask(ArrayRef< int > ShuffleMask, EVT VT, unsigned &WhichResult, bool &isV_UNDEF)
Check if ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), and return the corresponding AR...
static bool BitsProperlyConcatenate(const APInt &A, const APInt &B)
static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG)
static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target, struct BaseUpdateUser &User, bool SimpleConstIncOnly, TargetLowering::DAGCombinerInfo &DCI)
static bool allUsersAreInFunction(const Value *V, const Function *F)
Return true if all users of V are within function F, looking through ConstantExprs.
static bool isSingletonVEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG)
PerformVMOVDRRCombine - Target-specific dag combine xforms for ARMISD::VMOVDRR.
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, SDValue &SatK)
static bool isLegalAddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
isLegalAddressImmediate - Return true if the integer value can be used as the offset of the target ad...
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isLegalT1AddressImmediate(int64_t V, EVT VT)
static SDValue CombineANDShift(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSHLSimplify(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformADDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDECombine - Target-specific dag combine transform from ARMISD::ADDC, ARMISD::ADDE,...
static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformHWLoopCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool isVUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of "vector_shuffle v,...
static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, uint64_t &Members)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
static SDValue PerformVDIVCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDIVCombine - VCVT (fixed-point to floating-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG)
static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, bool &Negate)
static bool canChangeToInt(SDValue Op, bool &SeenZero, const ARMSubtarget *Subtarget)
canChangeToInt - Given the fp compare operand, return true if it is suitable to morph to an integer c...
static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2)
Return the store opcode for a given store size.
static bool IsVUZPShuffleNode(SDNode *N)
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, MachineInstr &MI, const SDNode *Node)
Attaches vregs to MEMCPY that it will use as scratch registers when it is expanded into LDM/STM.
static bool isFloatingPointZero(SDValue Op)
isFloatingPointZero - Return true if this is +0.0.
static SDValue findMUL_LOHI(SDValue V)
static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG)
static void genTPLoopBody(MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI, Register OpSrcReg, Register OpDestReg, Register ElementCountReg, Register TotalIterationsReg, bool IsMemcpy)
Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and t2DoLoopEnd.
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformMinMaxCombine - Target-specific DAG combining for creating truncating saturates.
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static Function * getFunction(Constant *C)
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
loop Loop Strength Reduction
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static constexpr int Concat[]
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
virtual const ARMBaseRegisterInfo & getRegisterInfo() const =0
const uint32_t * getSjLjDispatchPreservedMask(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
Register getFrameRegister(const MachineFunction &MF) const override
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
const uint32_t * getTLSCallPreservedMask(const MachineFunction &MF) const
const uint32_t * getThisReturnPreservedMask(const MachineFunction &MF, CallingConv::ID) const
getThisReturnPreservedMask - Returns a call preserved mask specific to the case that 'returned' is on...
static ARMConstantPoolConstant * Create(const Constant *C, unsigned ID)
static ARMConstantPoolMBB * Create(LLVMContext &C, const MachineBasicBlock *mbb, unsigned ID, unsigned char PCAdj)
static ARMConstantPoolSymbol * Create(LLVMContext &C, StringRef s, unsigned ID, unsigned char PCAdj)
ARMConstantPoolValue - ARM specific constantpool value.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
int getVarArgsFrameIndex() const
int getPromotedConstpoolIncrease() const
SmallPtrSet< const GlobalVariable *, 2 > & getGlobalsPromotedToConstantPool()
void setArgumentStackToRestore(unsigned v)
bool branchTargetEnforcement() const
unsigned createPICLabelUId()
void setPromotedConstpoolIncrease(int Sz)
bool isThumb1OnlyFunction() const
void setArgRegsSaveSize(unsigned s)
bool shouldSignReturnAddress() const
bool isCmseNSEntryFunction() const
void setReturnRegsCount(unsigned s)
void setVarArgsFrameIndex(int Index)
unsigned getArgRegsSaveSize() const
void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV)
Indicate to the backend that GV has had its storage changed to inside a constant pool.
void setIsSplitCSR(bool s)
void setArgumentStackSize(unsigned size)
unsigned getArgumentStackSize() const
bool isTargetMachO() const
bool isTargetAEABI() const
bool supportsTailCall() const
const Triple & getTargetTriple() const
const ARMBaseInstrInfo * getInstrInfo() const override
bool isThumb1Only() const
bool hasFPARMv8Base() const
bool isTargetWindows() const
bool isGVIndirectSymbol(const GlobalValue *GV) const
True if the GV will be accessed via an indirect symbol.
const ARMTargetLowering * getTargetLowering() const override
bool isTargetDarwin() const
const ARMBaseRegisterInfo * getRegisterInfo() const override
bool isTargetAndroid() const
bool isTargetCOFF() const
bool isTargetGNUAEABI() const
bool isTargetWatchOS() const
bool useNEONForSinglePrecisionFP() const
const InstrItineraryData * getInstrItineraryData() const override
getInstrItins - Return the instruction itineraries based on subtarget selection.
bool isTargetWatchABI() const
bool hasAnyDataBarrier() const
bool isTargetDriverKit() const
bool allowsUnalignedMem() const
bool isTargetMuslAEABI() const
bool isTargetLinux() const
unsigned getPrefLoopLogAlignment() const
bool isTargetHardFloat() const
Align getDualLoadStoreAlignment() const
bool isReadOnly(const GlobalValue *GV) const
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const override
Return the correct alignment for the current calling convention.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
const ARMSubtarget * getSubtarget() const
bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const
bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const
Returns true if the addressing mode representing by AM is legal for the Thumb1 target,...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPreIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mod...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
ReplaceNodeResults - Replace the results of node with an illegal result type with new values built ou...
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool isLegalAddImmediate(int64_t Imm) const override
isLegalAddImmediate - Return true if the specified immediate is legal add immediate,...
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const override
isFPImmLegal - Returns true if the target can instruction select the specified FP immediate natively.
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const
PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const
PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
getSetCCResultType - Return the value type to use for ISD::SETCC.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
createFastISel - This method returns a target specific FastISel object, or null if the target does no...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
isShuffleMaskLegal - Targets can use this to indicate that they only support some VECTOR_SHUFFLE oper...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const override
getRegClassFor - Return the register class that should be used for the specified value type.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vstN intrinsic.
ARMTargetLowering(const TargetMachine &TM, const ARMSubtarget &STI)
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const
PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const override
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
Instruction * makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const
bool isLegalICmpImmediate(int64_t Imm) const override
isLegalICmpImmediate - Return true if the specified immediate is legal icmp immediate,...
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
allowsMisalignedMemoryAccesses - Returns true if the target allows unaligned memory accesses of the s...
bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy, Align Alignment, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
bool isVectorLoadExtDesirable(SDValue ExtVal) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override
Return true if the target can combine store(extractelement VectorTy, Idx).
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vldN intrinsic.
bool useSoftFloat() const override
bool alignLoopsWithOptSize() const override
Should loops be aligned even when the function is marked OptSize (but not MinSize).
SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
LowerAsmOperandForConstraint - Lower the specified operand into the Ops vector.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
Returns true if an argument of type Ty needs to be passed in a contiguous block of registers in calli...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPostIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mo...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
The address of a basic block.
static BranchProbability getZero()
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
void getInRegsParamInfo(unsigned InRegsParamRecordIndex, unsigned &BeginReg, unsigned &EndReg) const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
void rewindByValRegsInfo()
unsigned getInRegsParamsProcessed() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void addInRegsParamInfo(unsigned RegBegin, unsigned RegEnd)
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getInRegsParamsCount() const
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
unsigned getValNo() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getCalledOperand() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
const APFloat & getValueAPF() const
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
Align getStackAlignment() const
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPreferredAlign(const GlobalVariable *GV) const
Returns the preferred alignment of the specified global.
StringRef getPrivateGlobalPrefix() const
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
const GlobalValue * getGlobal() const
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
@ InternalLinkage
Rename collisions when linking (static functions).
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
Common base class shared among various IRBuilders.
Value * CreateZExtOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
BasicBlock * GetInsertBlock() const
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTruncOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::optional< unsigned > getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const
Return the cycle for the given class and operand.
bool isEmpty() const
Returns true if there are no itineraries.
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
const BasicBlock * getParent() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
static bool LowerToByteSwap(CallInst *CI)
Try to replace a call instruction with a call to a bswap intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Describe properties that are true of each instruction in the target description file.
unsigned getSchedClass() const
Return the scheduling class for this instruction.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
pred_iterator pred_begin()
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void moveAfter(MachineBasicBlock *NewBefore)
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
void computeMaxCallFrameSize(const MachineFunction &MF)
Computes the maximum size of a callframe and the AdjustsStack property.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool hasVAStart() const
Returns true if the function calls the llvm.va_start intrinsic.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getFunctionContextIndex() const
Return the index for the function context object.
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & reset(Property P)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i....
Representation of each machine instruction.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr fully defines the specified register.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr reads the specified register.
const MachineOperand & getOperand(unsigned i) const
unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
void setIsDef(bool Val=true)
Change a def to a use, or a use to a def.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< use_iterator > uses()
size_t use_size() const
Return the number of uses of this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
void setFlags(SDNodeFlags NewFlags)
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
void addCallSiteInfo(const SDNode *Node, CallSiteInfoImpl &&CallInfo)
Set CallSiteInfo to be associated with Node.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
StringRef - Represent a constant reference to a string, i.e.
const unsigned char * bytes_end() const
constexpr size_t size() const
size - Get the string size.
const unsigned char * bytes_begin() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC)
Override the default CondCode to be used to test the result of the comparison libcall against zero.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
const TargetMachine & getTargetMachine() const
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
bool isPositionIndependent() const
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
bool isOSVersionLT(unsigned Major, unsigned Minor=0, unsigned Micro=0) const
Helper function for doing comparisons against version numbers included in the target triple.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isPointerTy() const
True if this is an instance of PointerType.
Type * getArrayElementType() const
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static CondCodes getOppositeCondition(CondCodes CC)
@ SECREL
Thread Pointer Offset.
@ SBREL
Section Relative (Windows TLS)
@ GOTTPOFF
Global Offset Table, PC Relative.
@ TPOFF
Global Offset Table, Thread Pointer Offset.
TOF
Target Operand Flag enum.
@ MO_NONLAZY
MO_NONLAZY - This is an independent flag, on a symbol operand "FOO" it represents a symbol which,...
@ MO_SBREL
MO_SBREL - On a symbol operand, this represents a static base relative relocation.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_GOT
MO_GOT - On a symbol operand, this represents a GOT relative relocation.
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
unsigned getAM2Offset(unsigned AM2Opc)
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned createVMOVModImm(unsigned OpCmode, unsigned Val)
int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
int getFP32FP16Imm(const APInt &Imm)
If this is a FP16Imm encoded as a fp32 value, return the 8-bit encoding for it.
AddrOpc getAM2Op(unsigned AM2Opc)
bool isBitFieldInvertedMask(unsigned v)
const unsigned FPStatusBits
const unsigned FPReservedBits
const unsigned RoundingBitsPos
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Swift
Calling convention for Swift.
@ ARM_APCS
ARM Procedure Calling Standard (obsolete, but still used on some targets).
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ ARM_AAPCS
ARM Architecture Procedure Calling Standard calling convention (aka EABI).
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns true if Val1 has a lower Constant Materialization Cost than Val2.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
ComplexDeinterleavingOperation
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
ComplexDeinterleavingRotation
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
bool isVREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isVREVMask - Check if a vector shuffle corresponds to a VREV instruction with the specified blocksize...
unsigned gettBLXrOpcode(const MachineFunction &MF)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
unsigned convertAddSubFlagsOpcode(unsigned OldOpc)
Map pseudo instructions that imply an 'S' bit onto real opcodes.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Load/store instruction that can be merged with a base address update.
SDNode * N
Instruction that updates a pointer.
unsigned ConstInc
Pointer increment value if it is a constant, or 0 otherwise.
SDValue Inc
Pointer increment operand.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
bool isUnknown() const
Returns true if we don't know any bits.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)