84#include "llvm/IR/IntrinsicsARM.h"
122#define DEBUG_TYPE "arm-isel"
125STATISTIC(NumMovwMovt,
"Number of GAs materialized with movw + movt");
126STATISTIC(NumLoopByVals,
"Number of loops generated for byval arguments");
128 "Number of constants with their storage promoted into constant pools");
132 cl::desc(
"Enable / disable ARM interworking (for debugging only)"),
137 cl::desc(
"Enable / disable promotion of unnamed_addr constants into "
142 cl::desc(
"Maximum size of constant to promote into a constant pool"),
146 cl::desc(
"Maximum size of ALL constants to promote into a constant pool"),
151 cl::desc(
"Maximum interleave factor for MVE VLDn to generate."),
156 ARM::R0, ARM::R1, ARM::R2, ARM::R3
159void ARMTargetLowering::addTypeForNEON(
MVT VT,
MVT PromotedLdStVT) {
160 if (VT != PromotedLdStVT) {
169 if (ElemTy != MVT::f64)
173 if (ElemTy == MVT::i32) {
209 VT != MVT::v2i64 && VT != MVT::v1i64)
217void ARMTargetLowering::addDRTypeForNEON(
MVT VT) {
219 addTypeForNEON(VT, MVT::f64);
222void ARMTargetLowering::addQRTypeForNEON(
MVT VT) {
224 addTypeForNEON(VT, MVT::v2f64);
227void ARMTargetLowering::setAllExpand(
MVT VT) {
240void ARMTargetLowering::addAllExtLoads(
const MVT From,
const MVT To,
241 LegalizeAction Action) {
247void ARMTargetLowering::addMVEVectorTypes(
bool HasMVEFP) {
248 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
250 for (
auto VT : IntTypes) {
324 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
325 for (
auto VT : FloatTypes) {
393 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
394 for (
auto VT : LongTypes) {
411 addAllExtLoads(MVT::v8i16, MVT::v8i8,
Legal);
412 addAllExtLoads(MVT::v4i32, MVT::v4i16,
Legal);
413 addAllExtLoads(MVT::v4i32, MVT::v4i8,
Legal);
430 for (
auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
439 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
440 for (
auto VT : pTypes) {
496 for (
int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
504 if (Subtarget->isThumb() && Subtarget->
hasVFP2Base() &&
505 Subtarget->
hasARMOps() && !Subtarget->useSoftFloat()) {
506 static const struct {
508 const char *
const Name;
530 { RTLIB::UO_F32,
"__unordsf2vfp",
ISD::SETNE },
539 { RTLIB::UO_F64,
"__unorddf2vfp",
ISD::SETNE },
564 for (
const auto &LC : LibraryCalls) {
584 static const struct {
586 const char *
const Name;
671 for (
const auto &LC : LibraryCalls) {
681 static const struct {
683 const char *
const Name;
686 } MemOpsLibraryCalls[] = {
694 for (
const auto &LC : MemOpsLibraryCalls) {
704 static const struct {
706 const char *
const Name;
719 for (
const auto &LC : LibraryCalls) {
751 static const struct {
753 const char *
const Name;
761 for (
const auto &LC : LibraryCalls) {
772 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only() &&
773 Subtarget->hasFPRegs()) {
783 setAllExpand(MVT::f32);
784 if (!Subtarget->hasFP64())
785 setAllExpand(MVT::f64);
788 if (Subtarget->hasFullFP16()) {
797 if (Subtarget->hasBF16()) {
799 setAllExpand(MVT::bf16);
800 if (!Subtarget->hasFullFP16())
807 addAllExtLoads(VT, InnerVT,
Expand);
822 if (Subtarget->hasMVEIntegerOps())
823 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
826 if (Subtarget->hasLOB()) {
830 if (Subtarget->hasNEON()) {
831 addDRTypeForNEON(MVT::v2f32);
832 addDRTypeForNEON(MVT::v8i8);
833 addDRTypeForNEON(MVT::v4i16);
834 addDRTypeForNEON(MVT::v2i32);
835 addDRTypeForNEON(MVT::v1i64);
837 addQRTypeForNEON(MVT::v4f32);
838 addQRTypeForNEON(MVT::v2f64);
839 addQRTypeForNEON(MVT::v16i8);
840 addQRTypeForNEON(MVT::v8i16);
841 addQRTypeForNEON(MVT::v4i32);
842 addQRTypeForNEON(MVT::v2i64);
844 if (Subtarget->hasFullFP16()) {
845 addQRTypeForNEON(MVT::v8f16);
846 addDRTypeForNEON(MVT::v4f16);
849 if (Subtarget->hasBF16()) {
850 addQRTypeForNEON(MVT::v8bf16);
851 addDRTypeForNEON(MVT::v4bf16);
855 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
894 if (Subtarget->hasNEON()) {
1006 for (
MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
1015 for (
auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
1024 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
1032 if (Subtarget->hasMVEIntegerOps()) {
1037 if (Subtarget->hasMVEFloatOps()) {
1041 if (!Subtarget->hasFP64()) {
1088 if (Subtarget->hasFullFP16()) {
1094 if (!Subtarget->hasFP16()) {
1142 if (Subtarget->hasDSP()) {
1164 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6Ops()
1165 || (Subtarget->
isThumb2() && !Subtarget->hasDSP()))
1180 if (Subtarget->hasMVEIntegerOps())
1190 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops())
1201 if (!Subtarget->hasV5TOps() || Subtarget->
isThumb1Only()) {
1210 if (Subtarget->hasPerfMon())
1214 if (!Subtarget->hasV6Ops())
1217 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1218 : Subtarget->hasDivideInARMMode();
1225 if (Subtarget->
isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
1242 HasStandaloneRem =
false;
1247 const char *
const Name;
1249 } LibraryCalls[] = {
1261 for (
const auto &LC : LibraryCalls) {
1268 const char *
const Name;
1270 } LibraryCalls[] = {
1282 for (
const auto &LC : LibraryCalls) {
1326 InsertFencesForAtomic =
false;
1328 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1332 if (!Subtarget->isThumb() || !Subtarget->
isMClass())
1337 if (!Subtarget->hasAcquireRelease() ||
1340 InsertFencesForAtomic =
true;
1346 if (Subtarget->hasDataBarrier())
1347 InsertFencesForAtomic =
true;
1367 if (!InsertFencesForAtomic) {
1375 (!Subtarget->
isMClass() && Subtarget->hasV6Ops())) {
1387 }
else if ((Subtarget->
isMClass() && Subtarget->hasV8MBaselineOps()) ||
1388 Subtarget->hasForced32BitAtomics()) {
1402 if (!Subtarget->hasV6Ops()) {
1408 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1440 if (Subtarget->hasFullFP16()) {
1450 if (Subtarget->hasFullFP16())
1465 if (!Subtarget->useSoftFloat() && Subtarget->
hasVFP2Base() &&
1479 if (!Subtarget->useSoftFloat() && !Subtarget->
isThumb1Only()) {
1487 if (!Subtarget->hasFP16()) {
1518 if (Subtarget->hasNEON()) {
1525 if (Subtarget->hasFP64()) {
1538 if (Subtarget->hasFullFP16()) {
1556 if (Subtarget->hasNEON()) {
1568 if (Subtarget->hasFullFP16()) {
1586 if (Subtarget->hasMVEIntegerOps())
1589 if (Subtarget->hasV6Ops())
1594 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
1601 if (Subtarget->useSoftFloat() || Subtarget->
isThumb1Only() ||
1627 if (Subtarget->isThumb() || Subtarget->
isThumb2())
1632 return Subtarget->useSoftFloat();
1645std::pair<const TargetRegisterClass *, uint8_t>
1656 case MVT::f32:
case MVT::f64:
case MVT::v8i8:
case MVT::v4i16:
1657 case MVT::v2i32:
case MVT::v1i64:
case MVT::v2f32:
1658 RRC = &ARM::DPRRegClass;
1666 case MVT::v16i8:
case MVT::v8i16:
case MVT::v4i32:
case MVT::v2i64:
1667 case MVT::v4f32:
case MVT::v2f64:
1668 RRC = &ARM::DPRRegClass;
1672 RRC = &ARM::DPRRegClass;
1676 RRC = &ARM::DPRRegClass;
1680 return std::make_pair(RRC,
Cost);
1684#define MAKE_CASE(V) \
1904 if ((Subtarget->hasMVEIntegerOps() &&
1905 (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
1906 VT == MVT::v16i8)) ||
1907 (Subtarget->hasMVEFloatOps() &&
1908 (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16)))
1922 if (Subtarget->hasNEON()) {
1923 if (VT == MVT::v4i64)
1924 return &ARM::QQPRRegClass;
1925 if (VT == MVT::v8i64)
1926 return &ARM::QQQQPRRegClass;
1928 if (Subtarget->hasMVEIntegerOps()) {
1929 if (VT == MVT::v4i64)
1930 return &ARM::MQQPRRegClass;
1931 if (VT == MVT::v8i64)
1932 return &ARM::MQQQQPRRegClass;
1941 Align &PrefAlign)
const {
1942 if (!isa<MemIntrinsic>(CI))
1960 unsigned NumVals =
N->getNumValues();
1964 for (
unsigned i = 0; i != NumVals; ++i) {
1965 EVT VT =
N->getValueType(i);
1966 if (VT == MVT::Glue || VT == MVT::Other)
1972 if (!
N->isMachineOpcode())
1996 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
1997 return Const->getZExtValue() == 16;
2004 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2005 return Const->getZExtValue() == 16;
2012 if (
auto Const = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
2013 return Const->getZExtValue() == 16;
2082 bool isVarArg)
const {
2103 else if (Subtarget->hasFPRegs() && !Subtarget->
isThumb1Only() &&
2124 bool isVarArg)
const {
2125 return CCAssignFnForNode(
CC,
false, isVarArg);
2129 bool isVarArg)
const {
2130 return CCAssignFnForNode(
CC,
true, isVarArg);
2137 bool isVarArg)
const {
2138 switch (getEffectiveCallingConv(
CC, isVarArg)) {
2164 if (Subtarget->hasFullFP16()) {
2177 if (Subtarget->hasFullFP16()) {
2191SDValue ARMTargetLowering::LowerCallResult(
2203 for (
unsigned i = 0; i != RVLocs.
size(); ++i) {
2208 if (i == 0 && isThisReturn) {
2210 "unexpected return calling convention register assignment");
2221 Chain =
Lo.getValue(1);
2222 InGlue =
Lo.getValue(2);
2226 Chain =
Hi.getValue(1);
2227 InGlue =
Hi.getValue(2);
2239 Chain =
Lo.getValue(1);
2240 InGlue =
Lo.getValue(2);
2243 Chain =
Hi.getValue(1);
2244 InGlue =
Hi.getValue(2);
2279std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
2281 bool IsTailCall,
int SPDiff)
const {
2303 return std::make_pair(DstAddr, DstInfo);
2308 RegsToPassVector &RegsToPass,
2315 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
2316 unsigned id = Subtarget->
isLittle() ? 0 : 1;
2329 std::tie(DstAddr, DstInfo) =
2330 computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
2362 bool isStructRet = (Outs.
empty()) ?
false : Outs[0].
Flags.isSRet();
2363 bool isThisReturn =
false;
2364 bool isCmseNSCall =
false;
2365 bool isSibCall =
false;
2366 bool PreferIndirect =
false;
2367 bool GuardWithBTI =
false;
2377 !Subtarget->noBTIAtReturnTwice())
2382 isCmseNSCall =
true;
2394 if (isa<GlobalAddressSDNode>(Callee)) {
2398 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2401 PreferIndirect = Subtarget->isThumb() && Subtarget->
hasMinSize() &&
2403 return isa<Instruction>(U) &&
2404 cast<Instruction>(U)->getParent() == BB;
2411 IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, PreferIndirect);
2425 "site marked musttail");
2428 unsigned NumBytes = CCInfo.getStackSize();
2437 if (isTailCall && !isSibCall) {
2444 NumBytes =
alignTo(NumBytes, StackAlign);
2449 SPDiff = NumReusableBytes - NumBytes;
2453 if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (
unsigned)-SPDiff)
2469 RegsToPassVector RegsToPass;
2477 bool AfterFormalArgLoads =
false;
2481 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
2483 ++i, ++realArgIdx) {
2485 SDValue Arg = OutVals[realArgIdx];
2487 bool isByVal =
Flags.isByVal();
2507 if (isTailCall && VA.
isMemLoc() && !AfterFormalArgLoads) {
2509 AfterFormalArgLoads =
true;
2521 auto ArgVT = Outs[realArgIdx].ArgVT;
2522 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2540 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2541 StackPtr, MemOpChains, isTailCall, SPDiff);
2545 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2546 StackPtr, MemOpChains, isTailCall, SPDiff);
2551 std::tie(DstAddr, DstInfo) =
2552 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2556 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2557 StackPtr, MemOpChains, isTailCall, SPDiff);
2559 if (realArgIdx == 0 &&
Flags.isReturned() && !
Flags.isSwiftSelf() &&
2560 Outs[0].VT == MVT::i32) {
2562 "unexpected calling convention register assignment");
2563 assert(!
Ins.empty() && Ins[0].VT == MVT::i32 &&
2564 "unexpected use of 'returned'");
2565 isThisReturn =
true;
2569 CSInfo.ArgRegPairs.emplace_back(VA.
getLocReg(), i);
2570 RegsToPass.push_back(std::make_pair(VA.
getLocReg(), Arg));
2571 }
else if (isByVal) {
2573 unsigned offset = 0;
2577 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2578 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2580 if (CurByValIdx < ByValArgsCount) {
2582 unsigned RegBegin, RegEnd;
2583 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2588 for (i = 0, j = RegBegin;
j < RegEnd; i++,
j++) {
2595 RegsToPass.push_back(std::make_pair(j, Load));
2600 offset = RegEnd - RegBegin;
2602 CCInfo.nextInRegsParam();
2605 if (
Flags.getByValSize() > 4*offset) {
2609 std::tie(Dst, DstInfo) =
2610 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2619 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2627 std::tie(DstAddr, DstInfo) =
2628 computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
2635 if (!MemOpChains.
empty())
2641 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2642 Chain = DAG.
getCopyToReg(Chain, dl, RegsToPass[i].first,
2643 RegsToPass[i].second, InGlue);
2650 bool isDirect =
false;
2655 GVal =
G->getGlobal();
2656 bool isStub = !
TM.shouldAssumeDSOLocal(GVal) && Subtarget->
isTargetMachO();
2658 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->
isMClass());
2659 bool isLocalARMFunc =
false;
2662 if (Subtarget->genLongCalls()) {
2664 "long-calls codegen is not position independent!");
2668 if (isa<GlobalAddressSDNode>(Callee)) {
2669 if (Subtarget->genExecuteOnly()) {
2688 const char *
Sym = S->getSymbol();
2690 if (Subtarget->genExecuteOnly()) {
2709 }
else if (isa<GlobalAddressSDNode>(Callee)) {
2710 if (!PreferIndirect) {
2715 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !
ARMInterworking);
2717 if (isStub && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2729 "Windows is the only supported COFF target");
2733 else if (!
TM.shouldAssumeDSOLocal(GVal))
2749 const char *
Sym = S->getSymbol();
2750 if (isARMFunc && Subtarget->
isThumb1Only() && !Subtarget->hasV5TOps()) {
2754 ARMPCLabelIndex, 4);
2768 assert(!isARMFunc && !isDirect &&
2769 "Cannot handle call to ARM function or direct call");
2772 "call to non-secure function would "
2773 "require passing arguments on stack",
2780 "call to non-secure function would return value through pointer",
2788 if (Subtarget->isThumb()) {
2791 else if (isCmseNSCall)
2793 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2798 if (!isDirect && !Subtarget->hasV5TOps())
2800 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2813 if (isTailCall && !isSibCall) {
2818 std::vector<SDValue> Ops;
2819 Ops.push_back(Chain);
2820 Ops.push_back(Callee);
2828 for (
unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2829 Ops.push_back(DAG.
getRegister(RegsToPass[i].first,
2830 RegsToPass[i].second.getValueType()));
2842 isThisReturn =
false;
2848 assert(Mask &&
"Missing call preserved mask for calling convention");
2852 Ops.push_back(InGlue);
2864 Chain = DAG.
getNode(CallOpc, dl, NodeTys, Ops);
2876 Chain = DAG.
getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
2882 return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
2883 InVals, isThisReturn,
2884 isThisReturn ? OutVals[0] :
SDValue());
2891void ARMTargetLowering::HandleByVal(
CCState *State,
unsigned &
Size,
2892 Align Alignment)
const {
2894 Alignment = std::max(Alignment,
Align(4));
2900 unsigned AlignInRegs = Alignment.
value() / 4;
2901 unsigned Waste = (ARM::R4 -
Reg) % AlignInRegs;
2902 for (
unsigned i = 0; i < Waste; ++i)
2908 unsigned Excess = 4 * (ARM::R4 -
Reg);
2915 if (NSAAOffset != 0 &&
Size > Excess) {
2927 unsigned ByValRegBegin =
Reg;
2928 unsigned ByValRegEnd = std::min<unsigned>(Reg +
Size / 4, ARM::R4);
2932 for (
unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2938 Size = std::max<int>(
Size - Excess, 0);
2949 int FI = std::numeric_limits<int>::max();
2957 if (!Flags.isByVal()) {
2963 }
else if (
LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2964 if (Flags.isByVal())
2979 assert(FI != std::numeric_limits<int>::max());
2990bool ARMTargetLowering::IsEligibleForTailCallOptimization(
3012 if (Outs.
size() >= 4 &&
3013 (!isa<GlobalAddressSDNode>(
Callee.getNode()) || isIndirect)) {
3031 return CalleeCC == CallerCC;
3035 bool isCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
3037 if (isCalleeStructRet || isCallerStructRet)
3051 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO()))
3058 getEffectiveCallingConv(CalleeCC, isVarArg),
3059 getEffectiveCallingConv(CallerCC, CallerF.
isVarArg()), MF,
C, Ins,
3065 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
3066 if (CalleeCC != CallerCC) {
3067 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
3068 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3081 if (!Outs.
empty()) {
3088 for (
unsigned i = 0, realArgIdx = 0, e = ArgLocs.
size();
3090 ++i, ++realArgIdx) {
3093 SDValue Arg = OutVals[realArgIdx];
3097 if (VA.
needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
3104 if (!ArgLocs[++i].isRegLoc())
3106 if (RegVT == MVT::v2f64) {
3107 if (!ArgLocs[++i].isRegLoc())
3109 if (!ArgLocs[++i].isRegLoc())
3134 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
3143 StringRef IntKind =
F.getFnAttribute(
"interrupt").getValueAsString();
3156 if (IntKind ==
"" || IntKind ==
"IRQ" || IntKind ==
"FIQ" ||
3159 else if (IntKind ==
"SWI" || IntKind ==
"UNDEF")
3163 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
3190 bool isLittleEndian = Subtarget->
isLittle();
3202 "secure entry function would return value through pointer",
3208 for (
unsigned i = 0, realRVLocIdx = 0;
3210 ++i, ++realRVLocIdx) {
3214 SDValue Arg = OutVals[realRVLocIdx];
3215 bool ReturnF16 =
false;
3250 auto RetVT = Outs[realRVLocIdx].ArgVT;
3272 DAG.
getVTList(MVT::i32, MVT::i32), Half);
3276 HalfGPRs.
getValue(isLittleEndian ? 0 : 1), Glue);
3282 HalfGPRs.
getValue(isLittleEndian ? 1 : 0), Glue);
3294 DAG.
getVTList(MVT::i32, MVT::i32), Arg);
3296 fmrrd.
getValue(isLittleEndian ? 0 : 1), Glue);
3301 fmrrd.
getValue(isLittleEndian ? 1 : 0), Glue);
3345 return DAG.
getNode(RetNode, dl, MVT::Other, RetOps);
3348bool ARMTargetLowering::isUsedByReturnOnly(
SDNode *
N,
SDValue &Chain)
const {
3349 if (
N->getNumValues() != 1)
3351 if (!
N->hasNUsesOfValue(1, 0))
3359 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3361 TCChain =
Copy->getOperand(0);
3375 SDValue UseChain =
U->getOperand(0);
3383 if (
U->getOperand(
U->getNumOperands() - 1).getValueType() == MVT::Glue)
3391 if (!
Copy->hasOneUse())
3398 if (
Copy->getOperand(
Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3400 TCChain =
Copy->getOperand(0);
3405 bool HasRet =
false;
3420bool ARMTargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
3438 &&
"LowerWRITE_REGISTER called for non-i64 type argument.");
3454 EVT PtrVT =
Op.getValueType();
3464 if (Subtarget->genExecuteOnly()) {
3466 auto T =
const_cast<Type*
>(
CP->getType());
3467 auto C =
const_cast<Constant*
>(
CP->getConstVal());
3478 return LowerGlobalAddress(GA, DAG);
3483 Align CPAlign =
CP->getAlign();
3485 CPAlign = std::max(CPAlign,
Align(4));
3486 if (
CP->isMachineConstantPoolEntry())
3498 if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
3507 unsigned ARMPCLabelIndex = 0;
3510 const BlockAddress *BA = cast<BlockAddressSDNode>(
Op)->getBlockAddress();
3513 if (!IsPositionIndependent) {
3516 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3527 if (!IsPositionIndependent)
3558ARMTargetLowering::LowerGlobalTLSAddressDarwin(
SDValue Op,
3561 "This function expects a Darwin target");
3566 SDValue DescAddr = LowerGlobalAddressDarwin(
Op, DAG);
3572 MVT::i32,
DL, Chain, DescAddr,
3596 Chain, FuncTLVGet, DAG.
getRegister(ARM::R0, MVT::i32),
3602ARMTargetLowering::LowerGlobalTLSAddressWindows(
SDValue Op,
3619 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
3646 const auto *GA = cast<GlobalAddressSDNode>(
Op);
3663 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3685 Args.push_back(Entry);
3693 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
3694 return CallResult.first;
3716 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3724 PtrVT, dl, Chain,
Offset,
3726 Chain =
Offset.getValue(1);
3732 PtrVT, dl, Chain,
Offset,
3742 PtrVT, dl, Chain,
Offset,
3758 return LowerGlobalTLSAddressDarwin(
Op, DAG);
3761 return LowerGlobalTLSAddressWindows(
Op, DAG);
3770 return LowerToTLSGeneralDynamicModel(GA, DAG);
3773 return LowerToTLSExecModels(GA, DAG, model);
3782 while (!Worklist.
empty()) {
3784 if (isa<ConstantExpr>(U)) {
3789 auto *
I = dyn_cast<Instruction>(U);
3790 if (!
I ||
I->getParent()->getParent() !=
F)
3818 auto *GVar = dyn_cast<GlobalVariable>(GV);
3819 if (!GVar || !GVar->hasInitializer() ||
3820 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3821 !GVar->hasLocalLinkage())
3826 auto *
Init = GVar->getInitializer();
3828 Init->needsDynamicRelocation())
3837 auto *CDAInit = dyn_cast<ConstantDataArray>(
Init);
3840 unsigned RequiredPadding = 4 - (
Size % 4);
3841 bool PaddingPossible =
3842 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3847 unsigned PaddedSize =
Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3871 if (RequiredPadding != 4) {
3876 while (RequiredPadding--)
3888 ++NumConstpoolPromoted;
3893 if (
const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
3894 if (!(GV = GA->getAliaseeObject()))
3896 if (
const auto *V = dyn_cast<GlobalVariable>(GV))
3897 return V->isConstant();
3898 return isa<Function>(GV);
3906 return LowerGlobalAddressWindows(
Op, DAG);
3908 return LowerGlobalAddressELF(
Op, DAG);
3910 return LowerGlobalAddressDarwin(
Op, DAG);
3918 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
3922 if (GV->
isDSOLocal() && !Subtarget->genExecuteOnly())
3935 }
else if (Subtarget->
isROPI() && IsRO) {
3940 }
else if (Subtarget->
isRWPI() && !IsRO) {
3965 if (Subtarget->
useMovt() || Subtarget->genExecuteOnly()) {
3984 "ROPI/RWPI not currently supported for Darwin");
3987 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4010 "Windows on ARM expects to use movw/movt");
4012 "ROPI/RWPI not currently supported for Windows");
4015 const GlobalValue *GV = cast<GlobalAddressSDNode>(
Op)->getGlobal();
4019 else if (!
TM.shouldAssumeDSOLocal(GV))
4043 DAG.
getVTList(MVT::i32, MVT::Other),
Op.getOperand(0),
4044 Op.getOperand(1), Val);
4061SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
4064 Op.getConstantOperandVal(
Op.getOperand(0).getValueType() == MVT::Other);
4068 case Intrinsic::arm_gnu_eabi_mcount: {
4077 assert(Mask &&
"Missing call preserved mask for calling convention");
4082 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
4086 if (Subtarget->isThumb())
4089 ARM::tBL_PUSHLR, dl, ResultTys,
4090 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
4091 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
4095 {ReturnAddress, Callee, RegisterMask, Chain}),
4104 unsigned IntNo =
Op.getConstantOperandVal(0);
4108 case Intrinsic::thread_pointer: {
4112 case Intrinsic::arm_cls: {
4113 const SDValue &Operand =
Op.getOperand(1);
4114 const EVT VTy =
Op.getValueType();
4125 case Intrinsic::arm_cls64: {
4128 const SDValue &Operand =
Op.getOperand(1);
4129 const EVT VTy =
Op.getValueType();
4152 case Intrinsic::eh_sjlj_lsda: {
4159 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
4169 if (IsPositionIndependent) {
4175 case Intrinsic::arm_neon_vabs:
4178 case Intrinsic::arm_neon_vmulls:
4179 case Intrinsic::arm_neon_vmullu: {
4180 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
4183 Op.getOperand(1),
Op.getOperand(2));
4185 case Intrinsic::arm_neon_vminnm:
4186 case Intrinsic::arm_neon_vmaxnm: {
4187 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
4190 Op.getOperand(1),
Op.getOperand(2));
4192 case Intrinsic::arm_neon_vminu:
4193 case Intrinsic::arm_neon_vmaxu: {
4194 if (
Op.getValueType().isFloatingPoint())
4196 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
4199 Op.getOperand(1),
Op.getOperand(2));
4201 case Intrinsic::arm_neon_vmins:
4202 case Intrinsic::arm_neon_vmaxs: {
4204 if (!
Op.getValueType().isFloatingPoint()) {
4205 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4208 Op.getOperand(1),
Op.getOperand(2));
4210 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
4213 Op.getOperand(1),
Op.getOperand(2));
4215 case Intrinsic::arm_neon_vtbl1:
4217 Op.getOperand(1),
Op.getOperand(2));
4218 case Intrinsic::arm_neon_vtbl2:
4220 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4221 case Intrinsic::arm_mve_pred_i2v:
4222 case Intrinsic::arm_mve_pred_v2i:
4225 case Intrinsic::arm_mve_vreinterpretq:
4228 case Intrinsic::arm_mve_lsll:
4230 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4231 case Intrinsic::arm_mve_asrl:
4233 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
4244 if (!Subtarget->hasDataBarrier()) {
4248 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
4249 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
4260 }
else if (Subtarget->preferISHSTBarriers() &&
4269 DAG.
getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4277 (!Subtarget->
isThumb1Only() && Subtarget->hasV5TEOps())))
4279 return Op.getOperand(0);
4282 unsigned isRead =
~Op.getConstantOperandVal(2) & 1;
4284 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4286 return Op.getOperand(0);
4288 unsigned isData =
Op.getConstantOperandVal(4);
4289 if (Subtarget->isThumb()) {
4291 isRead = ~isRead & 1;
4292 isData = ~isData & 1;
4309 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
4310 return DAG.
getStore(
Op.getOperand(0), dl, FR,
Op.getOperand(1),
4318 const SDLoc &dl)
const {
4324 RC = &ARM::tGPRRegClass;
4326 RC = &ARM::GPRRegClass;
4340 MVT::i32, dl, Root, FIN,
4361 const Value *OrigArg,
4362 unsigned InRegsParamRecordIdx,
4363 int ArgOffset,
unsigned ArgSize)
const {
4378 unsigned RBegin, REnd;
4388 ArgOffset = -4 * (ARM::R4 - RBegin);
4398 for (
unsigned Reg = RBegin, i = 0;
Reg < REnd; ++
Reg, ++i) {
4407 if (!MemOps.
empty())
4416 unsigned TotalArgRegsSaveSize,
4417 bool ForceMutable)
const {
4428 CCInfo.
getStackSize(), std::max(4U, TotalArgRegsSaveSize));
4432bool ARMTargetLowering::splitValueIntoRegisterParts(
4434 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4436 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4448SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4450 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
4451 if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
4464SDValue ARMTargetLowering::LowerFormalArguments(
4482 unsigned CurArgIdx = 0;
4494 unsigned ArgRegBegin = ARM::R4;
4495 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4502 if (!
Flags.isByVal())
4506 unsigned RBegin, REnd;
4508 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4514 int lastInsIndex = -1;
4518 ArgRegBegin = std::min(ArgRegBegin, (
unsigned)
GPRArgRegs[RegIdx]);
4521 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4525 for (
unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4527 if (Ins[VA.
getValNo()].isOrigArg()) {
4528 std::advance(CurOrigArg,
4529 Ins[VA.
getValNo()].getOrigArgIndex() - CurArgIdx);
4540 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4547 MVT::f64, dl, Chain, FIN,
4550 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4558 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4562 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4563 RC = &ARM::HPRRegClass;
4564 else if (RegVT == MVT::f32)
4565 RC = &ARM::SPRRegClass;
4566 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4567 RegVT == MVT::v4bf16)
4568 RC = &ARM::DPRRegClass;
4569 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4570 RegVT == MVT::v8bf16)
4571 RC = &ARM::QPRRegClass;
4572 else if (RegVT == MVT::i32)
4574 : &ARM::GPRRegClass;
4621 assert(VA.
getValVT() != MVT::i64 &&
"i64 should already be lowered");
4627 if (index != lastInsIndex)
4635 if (
Flags.isByVal()) {
4636 assert(Ins[index].isOrigArg() &&
4637 "Byval arguments cannot be implicit");
4641 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4656 lastInsIndex = index;
4663 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.
getStackSize(),
4664 TotalArgRegsSaveSize);
4668 "secure entry function must not be variadic", dl.
getDebugLoc());
4679 StackArgSize =
alignTo(StackArgSize,
DL.getStackAlignment());
4688 "secure entry function requires arguments on stack", dl.
getDebugLoc());
4698 return CFP->getValueAPF().isPosZero();
4702 SDValue WrapperOp =
Op.getOperand(1).getOperand(0);
4704 if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
4705 return CFP->getValueAPF().isPosZero();
4708 Op->getValueType(0) == MVT::f64) {
4723 const SDLoc &dl)
const {
4725 unsigned C = RHSC->getZExtValue();
4791 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
4793 unsigned Mask =
LHS.getConstantOperandVal(1);
4794 auto *RHSC = cast<ConstantSDNode>(
RHS.getNode());
4795 uint64_t RHSV = RHSC->getZExtValue();
4796 if (
isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4798 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4813 isa<ConstantSDNode>(RHS) &&
RHS->getAsZExtVal() == 0x80000000U &&
4815 LHS.getConstantOperandVal(1) < 31) {
4816 unsigned ShiftAmt =
LHS.getConstantOperandVal(1) + 1;
4856 return DAG.
getNode(CompareType, dl, MVT::Glue, LHS, RHS);
4862 bool Signaling)
const {
4863 assert(Subtarget->hasFP64() ||
RHS.getValueType() != MVT::f64);
4867 dl, MVT::Glue, LHS, RHS);
4870 dl, MVT::Glue, LHS);
4878 unsigned Opc =
Cmp.getOpcode();
4881 return DAG.
getNode(Opc,
DL, MVT::Glue,
Cmp.getOperand(0),
Cmp.getOperand(1));
4885 Opc =
Cmp.getOpcode();
4899std::pair<SDValue, SDValue>
4902 assert(
Op.getValueType() == MVT::i32 &&
"Unsupported value type");
4914 switch (
Op.getOpcode()) {
4927 DAG.
getVTList(
Op.getValueType(), MVT::i32), LHS, RHS)
4966 return std::make_pair(
Value, OverflowCmp);
4977 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Op, DAG, ARMcc);
4983 EVT VT =
Op.getValueType();
4986 ARMcc, CCR, OverflowCmp);
5026 EVT VT =
Op.getValueType();
5030 switch (
Op.getOpcode()) {
5055 EVT VT =
Op.getValueType();
5056 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->
isThumb1Only())
5066 switch (
Op->getOpcode()) {
5082 switch (
Op->getOpcode()) {
5101 DAG.
getNode(NewOpcode, dl, MVT::i32,
5112 unsigned Opc =
Cond.getOpcode();
5114 if (
Cond.getResNo() == 1 &&
5122 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5124 EVT VT =
Op.getValueType();
5126 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
5137 dyn_cast<ConstantSDNode>(
Cond.getOperand(0));
5139 dyn_cast<ConstantSDNode>(
Cond.getOperand(1));
5141 if (CMOVTrue && CMOVFalse) {
5147 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
5149 False = SelectFalse;
5150 }
else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
5156 EVT VT =
Op.getValueType();
5161 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
5177 bool &swpCmpOps,
bool &swpVselOps) {
5205 swpCmpOps = !swpCmpOps;
5206 swpVselOps = !swpVselOps;
5229 if (!Subtarget->hasFP64() && VT == MVT::f64) {
5231 DAG.
getVTList(MVT::i32, MVT::i32), FalseVal);
5233 DAG.
getVTList(MVT::i32, MVT::i32), TrueVal);
5243 ARMcc, CCR, duplicateCmp(Cmp, DAG));
5270 ((K ==
LHS && K == TrueVal) || (K ==
RHS && K == FalseVal))) ||
5272 ((K ==
RHS && K == TrueVal) || (K ==
LHS && K == FalseVal)));
5293 EVT VT =
Op.getValueType();
5300 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5315 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5323 if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
5326 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
5327 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
5328 int64_t PosVal = std::max(Val1, Val2);
5329 int64_t NegVal = std::min(Val1, Val2);
5375 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5376 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5381 if (*K != KTmp || V != VTmp)
5392bool ARMTargetLowering::isUnsupportedFloatingType(
EVT VT)
const {
5396 return !Subtarget->hasFP64();
5398 return !Subtarget->hasFullFP16();
5403 EVT VT =
Op.getValueType();
5407 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->
isThumb2())
5419 if (VT == MVT::i32 &&
5439 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
5440 LHS.getValueType() == MVT::i32 &&
RHS.getValueType() == MVT::i32) {
5443 unsigned Opcode = 0;
5445 if (TVal == ~FVal) {
5447 }
else if (TVal == ~FVal + 1) {
5449 }
else if (TVal + 1 == FVal) {
5451 }
else if (TVal == FVal + 1) {
5483 return DAG.
getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5487 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5489 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5493 if (!
RHS.getNode()) {
5499 if (
LHS.getValueType() == MVT::i32) {
5511 TrueVal.getValueType() == MVT::f32 ||
5512 TrueVal.getValueType() == MVT::f64)) {
5527 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5539 (
TrueVal.getValueType() == MVT::f16 ||
5540 TrueVal.getValueType() == MVT::f32 ||
5541 TrueVal.getValueType() == MVT::f64)) {
5542 bool swpCmpOps =
false;
5543 bool swpVselOps =
false;
5558 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5562 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5563 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
5573 if (!
N->hasOneUse())
5576 if (!
N->getNumValues())
5578 EVT VT =
Op.getValueType();
5579 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5596 return DAG.
getLoad(MVT::i32,
SDLoc(
Op), Ld->getChain(), Ld->getBasePtr(),
5597 Ld->getPointerInfo(), Ld->getAlign(),
5598 Ld->getMemOperand()->getFlags());
5616 DAG.
getLoad(MVT::i32, dl, Ld->getChain(),
Ptr, Ld->getPointerInfo(),
5617 Ld->getAlign(), Ld->getMemOperand()->getFlags());
5619 EVT PtrType =
Ptr.getValueType();
5622 RetVal2 = DAG.
getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5623 Ld->getPointerInfo().getWithOffset(4),
5625 Ld->getMemOperand()->getFlags());
5643 bool LHSSeenZero =
false;
5645 bool RHSSeenZero =
false;
5647 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5658 if (
LHS.getValueType() == MVT::f32) {
5666 Chain, Dest, ARMcc, CCR, Cmp);
5678 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5693 unsigned Opc =
Cond.getOpcode();
5696 if (
Cond.getResNo() == 1 &&
5706 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
Cond, DAG, ARMcc);
5730 if (isUnsupportedFloatingType(
LHS.getValueType())) {
5732 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS);
5736 if (!
RHS.getNode()) {
5744 unsigned Opc =
LHS.getOpcode();
5758 std::tie(
Value, OverflowCmp) = getARMXALUOOp(
LHS.getValue(0), DAG, ARMcc);
5773 if (
LHS.getValueType() == MVT::i32) {
5778 Chain, Dest, ARMcc, CCR, Cmp);
5784 if (
SDValue Result = OptimizeVFPBrcond(
Op, DAG))
5795 SDValue Ops[] = { Chain, Dest, ARMcc, CCR,
Cmp };
5817 if (Subtarget->
isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5823 Addr,
Op.getOperand(2), JTI);
5829 Chain =
Addr.getValue(1);
5836 Chain =
Addr.getValue(1);
5842 EVT VT =
Op.getValueType();
5845 if (
Op.getValueType().getVectorElementType() == MVT::i32) {
5846 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5854 const EVT OpTy =
Op.getOperand(0).getValueType();
5855 if (OpTy == MVT::v4f32)
5857 else if (OpTy == MVT::v4f16 && HasFullFP16)
5859 else if (OpTy == MVT::v8f16 && HasFullFP16)
5864 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5867 Op = DAG.
getNode(
Op.getOpcode(), dl, NewTy,
Op.getOperand(0));
5872 EVT VT =
Op.getValueType();
5876 bool IsStrict =
Op->isStrictFPOpcode();
5877 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
5879 if (isUnsupportedFloatingType(SrcVal.
getValueType())) {
5889 MakeLibCallOptions CallOptions;
5892 std::tie(Result, Chain) =
makeLibCall(DAG, LC,
Op.getValueType(), SrcVal,
5893 CallOptions, Loc, Chain);
5903 Loc,
Op.getValueType(), SrcVal);
5912 EVT VT =
Op.getValueType();
5913 EVT ToVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
5914 EVT FromVT =
Op.getOperand(0).getValueType();
5916 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
5918 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
5919 Subtarget->hasFP64())
5921 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
5922 Subtarget->hasFullFP16())
5924 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
5925 Subtarget->hasMVEFloatOps())
5927 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
5928 Subtarget->hasMVEFloatOps())
5931 if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
5948 EVT VT =
Op.getValueType();
5951 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5957 assert((
Op.getOperand(0).getValueType() == MVT::v4i16 ||
5958 Op.getOperand(0).getValueType() == MVT::v8i16) &&
5959 "Invalid type for custom lowering!");
5964 if (VT == MVT::v4f32)
5965 DestVecType = MVT::v4i32;
5966 else if (VT == MVT::v4f16 && HasFullFP16)
5967 DestVecType = MVT::v4i16;
5968 else if (VT == MVT::v8f16 && HasFullFP16)
5969 DestVecType = MVT::v8i16;
5975 switch (
Op.getOpcode()) {
5987 Op = DAG.
getNode(CastOpc, dl, DestVecType,
Op.getOperand(0));
5992 EVT VT =
Op.getValueType();
5995 if (isUnsupportedFloatingType(VT)) {
6003 MakeLibCallOptions CallOptions;
6005 CallOptions,
SDLoc(
Op)).first;
6016 EVT VT =
Op.getValueType();
6020 bool UseNEON = !InGPR && Subtarget->hasNEON();
6027 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
6034 if (SrcVT == MVT::f32) {
6040 }
else if (VT == MVT::f32)
6056 if (VT == MVT::f32) {
6068 if (SrcVT == MVT::f64)
6077 if (VT == MVT::f32) {
6101 EVT VT =
Op.getValueType();
6103 unsigned Depth =
Op.getConstantOperandVal(0);
6105 SDValue FrameAddr = LowerFRAMEADDR(
Op, DAG);
6124 EVT VT =
Op.getValueType();
6126 unsigned Depth =
Op.getConstantOperandVal(0);
6140 .
Case(
"sp", ARM::SP)
6155 assert(
N->getValueType(0) == MVT::i64
6156 &&
"ExpandREAD_REGISTER called for non-i64 type result.");
6159 DAG.
getVTList(MVT::i32, MVT::i32, MVT::Other),
6165 Results.push_back(Read.getOperand(0));
6199 const APInt &APIntIndex =
Index->getAPIntValue();
6201 NewIndex *= APIntIndex;
6231 EVT SrcVT =
Op.getValueType();
6232 EVT DstVT =
N->getValueType(0);
6234 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
6235 (DstVT == MVT::f16 || DstVT == MVT::bf16))
6239 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
6240 (SrcVT == MVT::f16 || SrcVT == MVT::bf16))
6245 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
6249 if (SrcVT == MVT::i64 && TLI.
isTypeLegal(DstVT)) {
6261 if (DstVT == MVT::i64 && TLI.
isTypeLegal(SrcVT)) {
6298 EVT VT =
Op.getValueType();
6317 SDValue LoBigShift = DAG.
getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6325 ? DAG.
getNode(Opc, dl, VT, ShOpHi,
6342 EVT VT =
Op.getValueType();
6385 DAG.
getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6437 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6465 Chain, DAG.
getConstant(Intrinsic::arm_set_fpscr,
DL, MVT::i32), FPSCR};
6495 EVT VT =
N->getValueType(0);
6496 if (VT.
isVector() && ST->hasNEON()) {
6505 if (ElemTy == MVT::i8) {
6513 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6528 if (ElemTy == MVT::i64) {
6541 if (!ST->hasV6T2Ops())
6550 EVT VT =
N->getValueType(0);
6553 assert(ST->hasNEON() &&
"Custom ctpop lowering requires NEON.");
6554 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
6555 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
6556 "Unexpected type for custom ctpop lowering");
6564 unsigned EltSize = 8;
6587 Op =
Op.getOperand(0);
6589 APInt SplatBits, SplatUndef;
6590 unsigned SplatBitSize;
6593 !BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6595 SplatBitSize > ElementBits)
6606 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6610 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6621 assert(VT.
isVector() &&
"vector shift count is not a vector type");
6626 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6627 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6636 EVT VT =
N->getValueType(0);
6658 "unexpected vector shift opcode");
6660 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
6661 unsigned VShiftOpc =
6663 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
6669 EVT ShiftVT =
N->getOperand(1).getValueType();
6672 unsigned VShiftOpc =
6674 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0), NegatedCount);
6679 EVT VT =
N->getValueType(0);
6688 "Unknown shift to lower!");
6690 unsigned ShOpc =
N->getOpcode();
6691 if (ST->hasMVEIntegerOps()) {
6721 DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6735 if (ST->isThumb1Only())
6740 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(0), dl, MVT::i32, MVT::i32);
6756 bool Invert =
false;
6763 EVT VT =
Op.getValueType();
6771 assert(ST->hasMVEIntegerOps() &&
6772 "No hardware support for integer vector comparison!");
6774 if (
Op.getValueType().getVectorElementType() != MVT::i1)
6799 Merged = DAG.
getNOT(dl, Merged, CmpVT);
6809 switch (SetCCOpcode) {
6813 if (ST->hasMVEFloatOps()) {
6816 Invert =
true; [[fallthrough]];
6821 case ISD::SETLT: Swap =
true; [[fallthrough]];
6825 case ISD::SETLE: Swap =
true; [[fallthrough]];
6841 Result = DAG.
getNOT(dl, Result, VT);
6844 case ISD::SETUO: Invert =
true; [[fallthrough]];
6853 Result = DAG.
getNOT(dl, Result, VT);
6859 switch (SetCCOpcode) {
6862 if (ST->hasMVEIntegerOps()) {
6865 Invert =
true; [[fallthrough]];
6868 case ISD::SETLT: Swap =
true; [[fallthrough]];
6870 case ISD::SETLE: Swap =
true; [[fallthrough]];
6879 if (ST->hasNEON() && Opc ==
ARMCC::EQ) {
6895 Result = DAG.
getNOT(dl, Result, VT);
6929 Result = DAG.
getNOT(dl, Result, VT);
6941 assert(
LHS.getSimpleValueType().isInteger() &&
"SETCCCARRY is integer only.");
6971 unsigned OpCmode, Imm;
6982 switch (SplatBitSize) {
6987 assert((SplatBits & ~0xff) == 0 &&
"one byte splat value is too big");
6990 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
6995 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
6996 if ((SplatBits & ~0xff) == 0) {
7002 if ((SplatBits & ~0xff00) == 0) {
7005 Imm = SplatBits >> 8;
7015 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
7016 if ((SplatBits & ~0xff) == 0) {
7022 if ((SplatBits & ~0xff00) == 0) {
7025 Imm = SplatBits >> 8;
7028 if ((SplatBits & ~0xff0000) == 0) {
7031 Imm = SplatBits >> 16;
7034 if ((SplatBits & ~0xff000000) == 0) {
7037 Imm = SplatBits >> 24;
7044 if ((SplatBits & ~0xffff) == 0 &&
7045 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
7048 Imm = SplatBits >> 8;
7056 if ((SplatBits & ~0xffffff) == 0 &&
7057 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
7060 Imm = SplatBits >> 16;
7076 unsigned ImmMask = 1;
7078 for (
int ByteNum = 0; ByteNum < 8; ++ByteNum) {
7079 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
7081 }
else if ((SplatBits & BitMask) != 0) {
7091 unsigned Mask = (1 << BytesPerElem) - 1;
7092 unsigned NumElems = 8 / BytesPerElem;
7093 unsigned NewImm = 0;
7094 for (
unsigned ElemNum = 0; ElemNum < NumElems; ++ElemNum) {
7095 unsigned Elem = ((Imm >> ElemNum * BytesPerElem) & Mask);
7096 NewImm |= Elem << (NumElems - ElemNum - 1) * BytesPerElem;
7103 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
7117 EVT VT =
Op.getValueType();
7118 bool IsDouble = (VT == MVT::f64);
7124 if (
ST->genExecuteOnly()) {
7126 assert((!
ST->isThumb1Only() ||
ST->hasV8MBaselineOps()) &&
7127 "Unexpected architecture");
7150 if (!
ST->hasVFP3Base())
7155 if (IsDouble && !Subtarget->hasFP64())
7162 if (IsDouble || !
ST->useNEONForSinglePrecisionFP()) {
7180 if (!
ST->hasNEON() || (!IsDouble && !
ST->useNEONForSinglePrecisionFP()))
7189 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
7243 unsigned ExpectedElt = Imm;
7244 for (
unsigned i = 1; i < NumElts; ++i) {
7248 if (ExpectedElt == NumElts)
7251 if (M[i] < 0)
continue;
7252 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7260 bool &ReverseVEXT,
unsigned &Imm) {
7262 ReverseVEXT =
false;
7273 unsigned ExpectedElt = Imm;
7274 for (
unsigned i = 1; i < NumElts; ++i) {
7278 if (ExpectedElt == NumElts * 2) {
7283 if (M[i] < 0)
continue;
7284 if (ExpectedElt !=
static_cast<unsigned>(M[i]))
7299 return VT == MVT::v8i8 && M.size() == 8;
7304 if (Mask.size() == Elements * 2)
7305 return Index / Elements;
7306 return Mask[
Index] == 0 ? 0 : 1;
7336 if (M.size() != NumElts && M.size() != NumElts*2)
7344 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7346 for (
unsigned j = 0; j < NumElts; j += 2) {
7347 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7348 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + NumElts + WhichResult))
7353 if (M.size() == NumElts*2)
7368 if (M.size() != NumElts && M.size() != NumElts*2)
7371 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7373 for (
unsigned j = 0; j < NumElts; j += 2) {
7374 if ((M[i+j] >= 0 && (
unsigned) M[i+j] != j + WhichResult) ||
7375 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] != j + WhichResult))
7380 if (M.size() == NumElts*2)
7400 if (M.size() != NumElts && M.size() != NumElts*2)
7403 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7405 for (
unsigned j = 0; j < NumElts; ++j) {
7406 if (M[i+j] >= 0 && (
unsigned) M[i+j] != 2 * j + WhichResult)
7411 if (M.size() == NumElts*2)
7430 if (M.size() != NumElts && M.size() != NumElts*2)
7433 unsigned Half = NumElts / 2;
7434 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7436 for (
unsigned j = 0; j < NumElts; j += Half) {
7437 unsigned Idx = WhichResult;
7438 for (
unsigned k = 0; k < Half; ++k) {
7439 int MIdx = M[i + j + k];
7440 if (MIdx >= 0 && (
unsigned) MIdx !=
Idx)
7447 if (M.size() == NumElts*2)
7471 if (M.size() != NumElts && M.size() != NumElts*2)
7474 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7476 unsigned Idx = WhichResult * NumElts / 2;
7477 for (
unsigned j = 0; j < NumElts; j += 2) {
7478 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7479 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx + NumElts))
7485 if (M.size() == NumElts*2)
7504 if (M.size() != NumElts && M.size() != NumElts*2)
7507 for (
unsigned i = 0; i < M.size(); i += NumElts) {
7509 unsigned Idx = WhichResult * NumElts / 2;
7510 for (
unsigned j = 0; j < NumElts; j += 2) {
7511 if ((M[i+j] >= 0 && (
unsigned) M[i+j] !=
Idx) ||
7512 (M[i+j+1] >= 0 && (
unsigned) M[i+j+1] !=
Idx))
7518 if (M.size() == NumElts*2)
7531 unsigned &WhichResult,
7534 if (
isVTRNMask(ShuffleMask, VT, WhichResult))
7536 if (
isVUZPMask(ShuffleMask, VT, WhichResult))
7538 if (
isVZIPMask(ShuffleMask, VT, WhichResult))
7556 if (NumElts != M.size())
7560 for (
unsigned i = 0; i != NumElts; ++i)
7561 if (M[i] >= 0 && M[i] != (
int) (NumElts - 1 - i))
7570 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7578 int Ofs = Top ? 1 : 0;
7579 int Upper = SingleSource ? 0 : NumElts;
7580 for (
int i = 0, e = NumElts / 2; i != e; ++i) {
7581 if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
7583 if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs +
Upper)
7592 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7601 unsigned Offset = Top ? 0 : 1;
7602 unsigned N = SingleSource ? 0 : NumElts;
7603 for (
unsigned i = 0; i < NumElts; i += 2) {
7604 if (M[i] >= 0 && M[i] != (
int)i)
7606 if (M[i + 1] >= 0 && M[i + 1] != (
int)(
N + i +
Offset))
7615 if (NumElts != M.size())
7623 unsigned Off0 = rev ? NumElts / 2 : 0;
7624 unsigned Off1 = rev ? 0 : NumElts / 2;
7625 for (
unsigned i = 0; i < NumElts; i += 2) {
7626 if (M[i] >= 0 && M[i] != (
int)(Off0 + i / 2))
7628 if (M[i + 1] >= 0 && M[i + 1] != (
int)(Off1 + i / 2))
7644 if (!ST->hasMVEFloatOps())
7649 if (VT != MVT::v8f16)
7670 for (
unsigned i = 1; i < 4; i++) {
7697 if (!ST->hasMVEFloatOps())
7702 if (VT != MVT::v4f32)
7718 for (
unsigned i = 1; i < 4; i++) {
7739 if (!isa<ConstantSDNode>(
N))
7741 Val =
N->getAsZExtVal();
7743 if (ST->isThumb1Only()) {
7744 if (Val <= 255 || ~Val <= 255)
7756 EVT VT =
Op.getValueType();
7758 assert(ST->hasMVEIntegerOps() &&
"LowerBUILD_VECTOR_i1 called without MVE!");
7762 unsigned BitsPerBool;
7766 }
else if (NumElts == 4) {
7769 }
else if (NumElts == 8) {
7772 }
else if (NumElts == 16) {
7781 if (!isa<ConstantSDNode>(FirstOp) &&
7783 return U.get().isUndef() || U.get() == FirstOp;
7791 unsigned Bits32 = 0;
7792 for (
unsigned i = 0; i < NumElts; ++i) {
7794 if (!isa<ConstantSDNode>(V) && !V.isUndef())
7796 bool BitSet = V.isUndef() ?
false : V->getAsZExtVal();
7798 Bits32 |= BoolMask << (i * BitsPerBool);
7804 for (
unsigned i = 0; i < NumElts; ++i) {
7806 if (isa<ConstantSDNode>(V) || V.isUndef())
7817 if (!ST->hasMVEIntegerOps())
7821 EVT VT =
Op.getValueType();
7831 if (
N != 1 &&
N != 2 &&
N != 4 &&
N != 8)
7835 for (
unsigned I = 2;
I < NumElts;
I++) {
7851 switch (
N->getOpcode()) {
7860 return N->getOperand(1).getNode() ==
Op;
7862 switch (
N->getConstantOperandVal(0)) {
7863 case Intrinsic::arm_mve_add_predicated:
7864 case Intrinsic::arm_mve_mul_predicated:
7865 case Intrinsic::arm_mve_qadd_predicated:
7866 case Intrinsic::arm_mve_vhadd:
7867 case Intrinsic::arm_mve_hadd_predicated:
7868 case Intrinsic::arm_mve_vqdmulh:
7869 case Intrinsic::arm_mve_qdmulh_predicated:
7870 case Intrinsic::arm_mve_vqrdmulh:
7871 case Intrinsic::arm_mve_qrdmulh_predicated:
7872 case Intrinsic::arm_mve_vqdmull:
7873 case Intrinsic::arm_mve_vqdmull_predicated:
7875 case Intrinsic::arm_mve_sub_predicated:
7876 case Intrinsic::arm_mve_qsub_predicated:
7877 case Intrinsic::arm_mve_vhsub:
7878 case Intrinsic::arm_mve_hsub_predicated:
7879 return N->getOperand(2).getNode() ==
Op;
7894 EVT VT =
Op.getValueType();
7902 APInt SplatBits, SplatUndef;
7903 unsigned SplatBitSize;
7905 if (BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7912 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
7914 [BVN](
const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
7915 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7916 : SplatBitSize == 16 ? MVT::v8i16
7923 if ((
ST->hasNEON() && SplatBitSize <= 64) ||
7924 (
ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7929 SplatBitSize, DAG, dl, VmovVT, VT,
VMOVModImm);
7937 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7939 NegatedImm, SplatUndef.
getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7947 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7957 if (
ST->hasMVEIntegerOps() &&
7958 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
7959 EVT DupVT = SplatBitSize == 32 ? MVT::v4i32
7960 : SplatBitSize == 16 ? MVT::v8i16
7977 bool isOnlyLowElement =
true;
7978 bool usesOnlyOneValue =
true;
7979 bool hasDominantValue =
false;
7986 for (
unsigned i = 0; i < NumElts; ++i) {
7991 isOnlyLowElement =
false;
7992 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
7995 ValueCounts.
insert(std::make_pair(V, 0));
7996 unsigned &Count = ValueCounts[
V];
7999 if (++Count > (NumElts / 2)) {
8000 hasDominantValue =
true;
8004 if (ValueCounts.
size() != 1)
8005 usesOnlyOneValue =
false;
8006 if (!
Value.getNode() && !ValueCounts.
empty())
8009 if (ValueCounts.
empty())
8021 if (hasDominantValue && EltSize <= 32) {
8032 (constIndex = dyn_cast<ConstantSDNode>(
Value->getOperand(1)))) {
8037 if (VT !=
Value->getOperand(0).getValueType()) {
8050 if (!usesOnlyOneValue) {
8053 for (
unsigned I = 0;
I < NumElts; ++
I) {
8068 assert(FVT == MVT::f32 || FVT == MVT::f16);
8069 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
8070 for (
unsigned i = 0; i < NumElts; ++i)
8075 Val = LowerBUILD_VECTOR(Val, DAG, ST);
8079 if (usesOnlyOneValue) {
8106 if (
ST->hasNEON() && VT.
is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
8126 if (EltSize >= 32) {
8132 for (
unsigned i = 0; i < NumElts; ++i)
8146 for (
unsigned i = 0 ; i < NumElts; ++i) {
8165 EVT VT =
Op.getValueType();
8168 struct ShuffleSourceInfo {
8170 unsigned MinElt = std::numeric_limits<unsigned>::max();
8171 unsigned MaxElt = 0;
8181 int WindowScale = 1;
8183 ShuffleSourceInfo(
SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
8191 for (
unsigned i = 0; i < NumElts; ++i) {
8199 }
else if (!isa<ConstantSDNode>(
V.getOperand(1))) {
8206 SDValue SourceVec =
V.getOperand(0);
8208 if (Source == Sources.
end())
8212 unsigned EltNo =
V.getConstantOperandVal(1);
8219 if (Sources.
size() > 2)
8225 for (
auto &Source : Sources) {
8226 EVT SrcEltTy =
Source.Vec.getValueType().getVectorElementType();
8227 if (SrcEltTy.
bitsLT(SmallestEltTy))
8228 SmallestEltTy = SrcEltTy;
8230 unsigned ResMultiplier =
8238 for (
auto &Src : Sources) {
8239 EVT SrcVT = Src.ShuffleVec.getValueType();
8243 if (SrcVTSize == VTSize)
8252 if (SrcVTSize < VTSize) {
8253 if (2 * SrcVTSize != VTSize)
8259 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8263 if (SrcVTSize != 2 * VTSize)
8266 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
8271 if (Src.MinElt >= NumSrcElts) {
8276 Src.WindowBase = -NumSrcElts;
8277 }
else if (Src.MaxElt < NumSrcElts) {
8294 Src.WindowBase = -Src.MinElt;
8301 for (
auto &Src : Sources) {
8302 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8303 if (SrcEltTy == SmallestEltTy)
8308 Src.WindowBase *= Src.WindowScale;
8314 assert(Src.ShuffleVec.getValueType() == ShuffleVT););
8321 if (Entry.isUndef())
8324 auto Src =
llvm::find(Sources, Entry.getOperand(0));
8325 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
8330 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
8333 int LanesDefined = BitsDefined / BitsPerShuffleLane;
8337 int *LaneMask = &
Mask[i * ResMultiplier];
8339 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
8340 ExtractBase += NumElts * (Src - Sources.begin());
8341 for (
int j = 0;
j < LanesDefined; ++
j)
8342 LaneMask[j] = ExtractBase + j;
8348 assert(Sources.size() <= 2 &&
"Too many sources!");
8351 for (
unsigned i = 0; i < Sources.size(); ++i)
8380 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8400 unsigned PFIndexes[4];
8401 for (
unsigned i = 0; i != 4; ++i) {
8405 PFIndexes[i] = M[i];
8409 unsigned PFTableIndex =
8410 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8412 unsigned Cost = (PFEntry >> 30);
8418 bool ReverseVEXT, isV_UNDEF;
8419 unsigned Imm, WhichResult;
8422 if (EltSize >= 32 ||
8429 else if (Subtarget->hasNEON() &&
8434 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8437 else if (Subtarget->hasMVEIntegerOps() &&
8441 else if (Subtarget->hasMVEIntegerOps() &&
8455 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8456 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8457 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8460 if (LHSID == (1*9+2)*9+3)
return LHS;
8461 assert(LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!");
8518 for (
int I : ShuffleMask)
8521 if (V2.getNode()->isUndef())
8531 EVT VT =
Op.getValueType();
8533 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8534 "Expect an v8i16/v16i8 type");
8540 std::vector<int> NewMask;
8544 NewMask.push_back(i);
8584 if (VT != MVT::v16i1)
8600 EVT VT =
Op.getValueType();
8604 assert(ST->hasMVEIntegerOps() &&
8605 "No support for vector shuffle of boolean predicates");
8631 "Expected identical vector type in expanded i1 shuffle!");
8635 PredAsVector2, ShuffleMask);
8640 if (VT == MVT::v2i1) {
8657 EVT VT =
Op.getValueType();
8661 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
8662 "Unexpected vector type");
8664 int QuarterSize = NumElts / 4;
8673 for (
int i = 0; i <
Length; i++) {
8674 if (ShuffleMask[Start + i] >= 0) {
8675 if (ShuffleMask[Start + i] %
Length != i)
8677 MovIdx = ShuffleMask[Start + i] /
Length;
8685 for (
int i = 1; i <
Length; i++) {
8686 if (ShuffleMask[Start + i] >= 0 &&
8687 (ShuffleMask[Start + i] /
Length != MovIdx ||
8688 ShuffleMask[Start + i] %
Length != i))
8694 for (
int Part = 0; Part < 4; ++Part) {
8696 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8700 Input =
Op->getOperand(1);
8710 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8715 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8717 for (
int Part = 0; Part < 4; ++Part)
8718 for (
int i = 0; i < QuarterSize; i++)
8720 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8722 VT, dl,
Op->getOperand(0),
Op->getOperand(1), NewShuffleMask);
8725 for (
int Part = 0; Part < 4; ++Part)
8741 EVT VT =
Op.getValueType();
8753 for (
int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8757 if (Mask[i] != i + BaseOffset) {
8758 if (OffElement == -1)
8764 return NonUndef > 2 && OffElement != -1;
8768 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8770 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8781 ShuffleMask[OffElement] < (
int)NumElts ? V1 : V2,
8792 EVT VT =
Op.getValueType();
8796 if (ST->hasMVEIntegerOps() && EltSize == 1)
8807 if (EltSize <= 32) {
8811 if (Lane == -1) Lane = 0;
8822 bool IsScalarToVector =
true;
8825 IsScalarToVector =
false;
8828 if (IsScalarToVector)
8835 bool ReverseVEXT =
false;
8837 if (ST->hasNEON() &&
isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8861 unsigned WhichResult = 0;
8862 bool isV_UNDEF =
false;
8863 if (ST->hasNEON()) {
8865 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8872 if (ST->hasMVEIntegerOps()) {
8907 }) &&
"Unexpected shuffle index into UNDEF operand!");
8910 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8913 assert((WhichResult == 0) &&
8914 "In-place shuffle of concat can only have one result!");
8923 if (ST->hasMVEIntegerOps() && EltSize <= 32) {
8927 for (
bool Top : {
false,
true}) {
8928 for (
bool SingleSource : {
false,
true}) {
8929 if (
isTruncMask(ShuffleMask, VT, Top, SingleSource)) {
8934 SingleSource ? V1 : V2);
8950 unsigned PFIndexes[4];
8951 for (
unsigned i = 0; i != 4; ++i) {
8952 if (ShuffleMask[i] < 0)
8955 PFIndexes[i] = ShuffleMask[i];
8959 unsigned PFTableIndex =
8960 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8962 unsigned Cost = (PFEntry >> 30);
8968 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8969 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8979 if (EltSize >= 32) {
8987 for (
unsigned i = 0; i < NumElts; ++i) {
8988 if (ShuffleMask[i] < 0)
8992 ShuffleMask[i] < (
int)NumElts ? V1 : V2,
9000 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&
9004 if (ST->hasNEON() && VT == MVT::v8i8)
9008 if (ST->hasMVEIntegerOps())
9017 EVT VecVT =
Op.getOperand(0).getValueType();
9020 assert(ST->hasMVEIntegerOps() &&
9021 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9025 unsigned Lane =
Op.getConstantOperandVal(2);
9026 unsigned LaneWidth =
9028 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
9040 if (!isa<ConstantSDNode>(Lane))
9046 if (Subtarget->hasMVEIntegerOps() &&
9047 Op.getValueType().getScalarSizeInBits() == 1)
9071 IVecIn, IElt, Lane);
9080 EVT VecVT =
Op.getOperand(0).getValueType();
9083 assert(ST->hasMVEIntegerOps() &&
9084 "LowerINSERT_VECTOR_ELT_i1 called without MVE!");
9088 unsigned Lane =
Op.getConstantOperandVal(1);
9089 unsigned LaneWidth =
9100 if (!isa<ConstantSDNode>(Lane))
9120 assert(
Op.getValueType().getScalarSizeInBits() == 1 &&
9121 "Unexpected custom CONCAT_VECTORS lowering");
9123 "Unexpected custom CONCAT_VECTORS lowering");
9124 assert(ST->hasMVEIntegerOps() &&
9125 "CONCAT_VECTORS lowering only supported for MVE");
9129 EVT Op2VT = V2.getValueType();
9130 assert(Op1VT == Op2VT &&
"Operand types don't match!");
9131 assert((Op1VT == MVT::v2i1 || Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) &&
9132 "Unexpected i1 concat operations!");
9145 if (Op1VT == MVT::v4i1 || Op1VT == MVT::v8i1) {
9159 auto ExtractInto = [&DAG, &dl](
SDValue NewV,
SDValue ConVec,
unsigned &j) {
9160 EVT NewVT = NewV.getValueType();
9161 EVT ConcatVT = ConVec.getValueType();
9162 unsigned ExtScale = 1;
9163 if (NewVT == MVT::v2f64) {
9177 ConVec = ExtractInto(NewV1, ConVec, j);
9178 ConVec = ExtractInto(NewV2, ConVec, j);
9188 while (ConcatOps.
size() > 1) {
9189 for (
unsigned I = 0, E = ConcatOps.
size();
I != E;
I += 2) {
9192 ConcatOps[
I / 2] = ConcatPair(V1, V2);
9196 return ConcatOps[0];
9201 EVT VT =
Op->getValueType(0);
9208 "unexpected CONCAT_VECTORS");
9229 EVT VT =
Op.getValueType();
9232 unsigned Index = V2->getAsZExtVal();
9235 "Unexpected custom EXTRACT_SUBVECTOR lowering");
9236 assert(ST->hasMVEIntegerOps() &&
9237 "EXTRACT_SUBVECTOR lowering only supported for MVE");
9247 EVT SubVT = MVT::v4i32;
9249 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j += 2) {
9264 for (
unsigned i =
Index, j = 0; i < (
Index + NumElts); i++, j++) {
9280 assert(ST->hasMVEIntegerOps() &&
"Expected MVE!");
9281 EVT VT =
N->getValueType(0);
9282 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&
9283 "Expected a vector i1 type!");
9285 EVT FromVT =
Op.getValueType();
9296 if (!Subtarget->hasMVEIntegerOps())
9299 EVT ToVT =
N->getValueType(0);
9342 if (ToVT != MVT::v8i16 && ToVT != MVT::v16i8)
9344 EVT FromVT =
N->getOperand(0).getValueType();
9345 if (FromVT != MVT::v8i32 && FromVT != MVT::v16i16)
9356 if (!Subtarget->hasMVEIntegerOps())
9361 EVT ToVT =
N->getValueType(0);
9362 if (ToVT != MVT::v16i32 && ToVT != MVT::v8i32 && ToVT != MVT::v16i16)
9365 EVT FromVT =
Op.getValueType();
9366 if (FromVT != MVT::v8i16 && FromVT != MVT::v16i8)
9377 SDValue Ext1 = Ext.getValue(1);
9380 Ext = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext);
9381 Ext1 = DAG.
getNode(
N->getOpcode(),
DL, MVT::v8i32, Ext1);
9393 EVT VT =
N->getValueType(0);
9395 SDNode *BVN =
N->getOperand(0).getNode();
9400 unsigned HiElt = 1 - LoElt;
9405 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
9421 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
9422 SDNode *Elt =
N->getOperand(i).getNode();
9425 unsigned HalfSize = EltSize / 2;
9427 if (!
isIntN(HalfSize,
C->getSExtValue()))
9430 if (!
isUIntN(HalfSize,
C->getZExtValue()))
9469 switch (OrigSimpleTy) {
9485 unsigned ExtOpcode) {
9508 if (ExtendedTy == LD->getMemoryVT())
9509 return DAG.
getLoad(LD->getMemoryVT(),
SDLoc(LD), LD->getChain(),
9510 LD->getBasePtr(), LD->getPointerInfo(), LD->getAlign(),
9511 LD->getMemOperand()->getFlags());
9517 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
9518 LD->getMemoryVT(), LD->getAlign(),
9519 LD->getMemOperand()->getFlags());
9532 N->getOperand(0)->getValueType(0),
9538 "Expected extending load");
9544 DAG.
getNode(Opcode,
SDLoc(newLoad), LD->getValueType(0), newLoad);
9553 SDNode *BVN =
N->getOperand(0).getNode();
9555 BVN->
getValueType(0) == MVT::v4i32 &&
"expected v4i32 BUILD_VECTOR");
9563 EVT VT =
N->getValueType(0);
9569 for (
unsigned i = 0; i != NumElts; ++i) {
9570 const APInt &CInt =
N->getConstantOperandAPInt(i);
9579 unsigned Opcode =
N->getOpcode();
9581 SDNode *N0 =
N->getOperand(0).getNode();
9582 SDNode *N1 =
N->getOperand(1).getNode();
9590 unsigned Opcode =
N->getOpcode();
9592 SDNode *N0 =
N->getOperand(0).getNode();
9593 SDNode *N1 =
N->getOperand(1).getNode();
9603 EVT VT =
Op.getValueType();
9605 "unexpected type for custom-lowering ISD::MUL");
9606 SDNode *N0 =
Op.getOperand(0).getNode();
9607 SDNode *N1 =
Op.getOperand(1).getNode();
9608 unsigned NewOpc = 0;
9612 if (isN0SExt && isN1SExt)
9617 if (isN0ZExt && isN1ZExt)
9619 else if (isN1SExt || isN1ZExt) {
9636 if (VT == MVT::v2i64)
9653 "unexpected types for extended operands to VMULL");
9654 return DAG.
getNode(NewOpc,
DL, VT, Op0, Op1);
9689 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9723 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9726 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9747 EVT VT =
Op.getValueType();
9748 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9749 "unexpected type for custom-lowering ISD::SDIV");
9756 if (VT == MVT::v8i8) {
9784 EVT VT =
Op.getValueType();
9785 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
9786 "unexpected type for custom-lowering ISD::UDIV");
9793 if (VT == MVT::v8i8) {
9832 DAG.
getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9835 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9839 DAG.
getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9860 EVT VT =
N->getValueType(0);
9874 Op.getOperand(1), Carry);
9888 Op.getOperand(1), Carry);
9921 bool ShouldUseSRet = Subtarget->
isAPCS_ABI();
9923 if (ShouldUseSRet) {
9933 Entry.IsSExt =
false;
9934 Entry.IsZExt =
false;
9935 Entry.IsSRet =
true;
9936 Args.push_back(Entry);
9943 Entry.IsSExt =
false;
9944 Entry.IsZExt =
false;
9945 Args.push_back(Entry);
9948 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
9958 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
9961 return CallResult.first;
9980 EVT VT =
Op.getValueType();
9981 assert((VT == MVT::i32 || VT == MVT::i64) &&
9982 "unexpected type for custom lowering DIV");
9988 const char *
Name =
nullptr;
9990 Name = (VT == MVT::i32) ?
"__rt_sdiv" :
"__rt_sdiv64";
9992 Name = (VT == MVT::i32) ?
"__rt_udiv" :
"__rt_udiv64";
9998 for (
auto AI : {1, 0}) {
10000 Arg.Node =
Op.getOperand(AI);
10001 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.
getContext());
10002 Args.push_back(Arg);
10005 CallLoweringInfo CLI(DAG);
10009 ES, std::move(Args));
10019ARMTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
10027 const bool MinSize =
ST.hasMinSize();
10028 const bool HasDivide =
ST.isThumb() ?
ST.hasDivideInThumbMode()
10029 :
ST.hasDivideInARMMode();
10033 if (
N->getOperand(0).getValueType().isVector())
10038 if (!(MinSize && HasDivide))
10051 if (Divisor.
sgt(128))
10059 assert(
Op.getValueType() == MVT::i32 &&
10060 "unexpected type for custom lowering DIV");
10066 return LowerWindowsDIVLibCall(
Op, DAG,
Signed, DBZCHK);
10072 if (
N->getValueType(0) == MVT::i32)
10080void ARMTargetLowering::ExpandDIV_Windows(
10086 assert(
Op.getValueType() == MVT::i64 &&
10087 "unexpected type for custom lowering DIV");
10104 EVT MemVT = LD->getMemoryVT();
10105 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10106 MemVT == MVT::v16i1) &&
10107 "Expected a predicate type!");
10108 assert(MemVT ==
Op.getValueType());
10110 "Expected a non-extending load");
10111 assert(LD->isUnindexed() &&
"Expected a unindexed load");
10125 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
10127 LD->getMemOperand());
10134 if (MemVT != MVT::v16i1)
10143 EVT MemVT =
LD->getMemoryVT();
10144 assert(
LD->isUnindexed() &&
"Loads should be unindexed at this point.");
10146 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10152 {LD->getChain(), LD->getBasePtr()}, MemVT,
LD->getMemOperand());
10162 EVT MemVT = ST->getMemoryVT();
10163 assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10164 MemVT == MVT::v16i1) &&
10165 "Expected a predicate type!");
10166 assert(MemVT == ST->getValue().getValueType());
10167 assert(!ST->isTruncatingStore() &&
"Expected a non-extending store");
10168 assert(ST->isUnindexed() &&
"Expected a unindexed store");
10173 SDValue Build = ST->getValue();
10174 if (MemVT != MVT::v16i1) {
10193 ST->getChain(), dl, GRP, ST->getBasePtr(),
10195 ST->getMemOperand());
10201 EVT MemVT = ST->getMemoryVT();
10202 assert(ST->isUnindexed() &&
"Stores should be unindexed at this point.");
10204 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
10220 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
10221 MemVT, ST->getMemOperand());
10222 }
else if (Subtarget->hasMVEIntegerOps() &&
10223 ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
10224 MemVT == MVT::v16i1))) {
10239 MVT VT =
Op.getSimpleValueType();
10241 SDValue PassThru =
N->getPassThru();
10252 VT, dl,
N->getChain(),
N->getBasePtr(),
N->getOffset(), Mask, ZeroVec,
10253 N->getMemoryVT(),
N->getMemOperand(),
N->getAddressingMode(),
10254 N->getExtensionType(),
N->isExpandingLoad());
10259 if (!PassThru.
isUndef() && !PassThruIsCastZero)
10266 if (!ST->hasMVEIntegerOps())
10270 unsigned BaseOpcode = 0;
10271 switch (
Op->getOpcode()) {
10287 unsigned NumActiveLanes = NumElts;
10289 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10290 NumActiveLanes == 2) &&
10291 "Only expected a power 2 vector size");
10295 while (NumActiveLanes > 4) {
10298 Op0 = DAG.
getNode(BaseOpcode, dl, VT, Op0, Rev);
10299 NumActiveLanes /= 2;
10303 if (NumActiveLanes == 4) {
10313 SDValue Res0 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10314 SDValue Res1 = DAG.
getNode(BaseOpcode, dl, EltVT, Ext2, Ext3,
Op->getFlags());
10315 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res0, Res1,
Op->getFlags());
10321 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Ext0, Ext1,
Op->getFlags());
10325 if (EltVT !=
Op->getValueType(0))
10332 if (!ST->hasMVEFloatOps())
10339 if (!ST->hasNEON())
10347 unsigned PairwiseIntrinsic = 0;
10348 switch (
Op->getOpcode()) {
10352 PairwiseIntrinsic = Intrinsic::arm_neon_vpminu;
10355 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxu;
10358 PairwiseIntrinsic = Intrinsic::arm_neon_vpmins;
10361 PairwiseIntrinsic = Intrinsic::arm_neon_vpmaxs;
10367 unsigned NumActiveLanes = NumElts;
10369 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||
10370 NumActiveLanes == 2) &&
10371 "Only expected a power 2 vector size");
10377 VT =
Lo.getValueType();
10379 NumActiveLanes /= 2;
10383 while (NumActiveLanes > 1) {
10385 NumActiveLanes /= 2;
10392 if (EltVT !=
Op.getValueType()) {
10393 unsigned Extend = 0;
10394 switch (
Op->getOpcode()) {
10406 Res = DAG.
getNode(Extend, dl,
Op.getValueType(), Res);
10428 SDValue Ops[] = {
N->getOperand(0),
10438 DAG.
getVTList(MVT::i32, MVT::Other), Ops);
10445 SDLoc dl(V.getNode());
10446 auto [VLo, VHi] = DAG.
SplitScalar(V, dl, MVT::i32, MVT::i32);
10454 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
10456 DAG.
getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
10462 assert(
N->getValueType(0) == MVT::i64 &&
10463 "AtomicCmpSwap on types less than 64 should be legal");
10464 SDValue Ops[] = {
N->getOperand(1),
10469 ARM::CMP_SWAP_64,
SDLoc(
N),
10470 DAG.
getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops);
10489 EVT VT =
Op.getValueType();
10498 if (isUnsupportedFloatingType(
LHS.getValueType())) {
10500 DAG,
LHS.getValueType(), LHS, RHS,
CC, dl, LHS, RHS, Chain, IsSignaling);
10501 if (!
RHS.getNode()) {
10522 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10523 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
10525 ARMcc = DAG.
getConstant(CondCode2, dl, MVT::i32);
10526 Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
10527 Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG);
10543 switch (
Op.getOpcode()) {
10575 case ISD::BITCAST:
return ExpandBITCAST(
Op.getNode(), DAG, Subtarget);
10579 case ISD::SREM:
return LowerREM(
Op.getNode(), DAG);
10580 case ISD::UREM:
return LowerREM(
Op.getNode(), DAG);
10602 return LowerSET_FPMODE(
Op, DAG);
10604 return LowerRESET_FPMODE(
Op, DAG);
10608 return LowerDIV_Windows(
Op, DAG,
true);
10612 return LowerDIV_Windows(
Op, DAG,
false);
10619 return LowerSignedALUO(
Op, DAG);
10622 return LowerUnsignedALUO(
Op, DAG);
10656 return LowerDYNAMIC_STACKALLOC(
Op, DAG);
10665 return LowerSPONENTRY(
Op, DAG);
10672 unsigned IntNo =
N->getConstantOperandVal(0);
10674 if (IntNo == Intrinsic::arm_smlald)
10676 else if (IntNo == Intrinsic::arm_smlaldx)
10678 else if (IntNo == Intrinsic::arm_smlsld)
10680 else if (IntNo == Intrinsic::arm_smlsldx)
10687 std::tie(
Lo,
Hi) = DAG.
SplitScalar(
N->getOperand(3), dl, MVT::i32, MVT::i32);
10691 N->getOperand(1),
N->getOperand(2),
10703 switch (
N->getOpcode()) {
10710 Res = ExpandBITCAST(
N, DAG, Subtarget);
10719 Res = LowerREM(
N, DAG);
10723 Res = LowerDivRem(
SDValue(
N, 0), DAG);
10777 "ROPI/RWPI not currently supported with SjLj");
10786 bool isThumb = Subtarget->isThumb();
10787 bool isThumb2 = Subtarget->
isThumb2();
10790 unsigned PCAdj = (
isThumb || isThumb2) ? 4 : 8;
10796 : &ARM::GPRRegClass;
10814 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10820 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10826 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10844 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10849 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10854 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
10859 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
10865 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
10880 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
10886 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
10910 : &ARM::GPRnopcRegClass;
10915 unsigned MaxCSNum = 0;
10923 if (!II.isEHLabel())
10927 if (!MF->hasCallSiteLandingPad(
Sym))
continue;
10930 for (
unsigned Idx : CallSiteIdxs) {
10931 CallSiteNumToLPad[
Idx].push_back(&BB);
10932 MaxCSNum = std::max(MaxCSNum,
Idx);
10939 std::vector<MachineBasicBlock*> LPadList;
10941 LPadList.reserve(CallSiteNumToLPad.
size());
10942 for (
unsigned I = 1;
I <= MaxCSNum; ++
I) {
10945 LPadList.push_back(
MBB);
10950 assert(!LPadList.empty() &&
10951 "No landing pad destinations for the dispatch jump table!");
10965 unsigned trap_opcode;
10966 if (Subtarget->isThumb())
10967 trap_opcode = ARM::tTRAP;
10969 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
10978 MF->insert(MF->end(), DispatchBB);
10979 MF->insert(MF->end(), DispContBB);
10980 MF->insert(MF->end(), TrapBB);
10984 SetupEntryBlockForSjLj(
MI,
MBB, DispatchBB, FI);
10991 MIB =
BuildMI(DispatchBB, dl,
TII->get(ARM::Int_eh_sjlj_dispatchsetup));
11003 unsigned NumLPads = LPadList.size();
11005 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11006 BuildMI(DispatchBB, dl,
TII->get(ARM::t2LDRi12), NewVReg1)
11012 if (NumLPads < 256) {
11013 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPri))
11015 .
addImm(LPadList.size())
11018 Register VReg1 =
MRI->createVirtualRegister(TRC);
11019 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVi16), VReg1)
11020 .
addImm(NumLPads & 0xFFFF)
11023 unsigned VReg2 = VReg1;
11024 if ((NumLPads & 0xFFFF0000) != 0) {
11025 VReg2 =
MRI->createVirtualRegister(TRC);
11026 BuildMI(DispatchBB, dl,
TII->get(ARM::t2MOVTi16), VReg2)
11032 BuildMI(DispatchBB, dl,
TII->get(ARM::t2CMPrr))
11038 BuildMI(DispatchBB, dl,
TII->get(ARM::t2Bcc))
11043 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11044 BuildMI(DispContBB, dl,
TII->get(ARM::t2LEApcrelJT), NewVReg3)
11048 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11049 BuildMI(DispContBB, dl,
TII->get(ARM::t2ADDrs), NewVReg4)
11056 BuildMI(DispContBB, dl,
TII->get(ARM::t2BR_JT))
11060 }
else if (Subtarget->isThumb()) {
11061 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11062 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRspi), NewVReg1)
11068 if (NumLPads < 256) {
11069 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPi8))
11079 Align Alignment = MF->getDataLayout().getPrefTypeAlign(
Int32Ty);
11082 Register VReg1 =
MRI->createVirtualRegister(TRC);
11083 BuildMI(DispatchBB, dl,
TII->get(ARM::tLDRpci))
11087 BuildMI(DispatchBB, dl,
TII->get(ARM::tCMPr))
11093 BuildMI(DispatchBB, dl,
TII->get(ARM::tBcc))
11098 Register NewVReg2 =
MRI->createVirtualRegister(TRC);
11099 BuildMI(DispContBB, dl,
TII->get(ARM::tLSLri), NewVReg2)
11105 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11106 BuildMI(DispContBB, dl,
TII->get(ARM::tLEApcrelJT), NewVReg3)
11110 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11111 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg4)
11121 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11122 BuildMI(DispContBB, dl,
TII->get(ARM::tLDRi), NewVReg5)
11128 unsigned NewVReg6 = NewVReg5;
11129 if (IsPositionIndependent) {
11130 NewVReg6 =
MRI->createVirtualRegister(TRC);
11131 BuildMI(DispContBB, dl,
TII->get(ARM::tADDrr), NewVReg6)
11138 BuildMI(DispContBB, dl,
TII->get(ARM::tBR_JTr))
11142 Register NewVReg1 =
MRI->createVirtualRegister(TRC);
11143 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRi12), NewVReg1)
11149 if (NumLPads < 256) {
11150 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPri))
11154 }
else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
11155 Register VReg1 =
MRI->createVirtualRegister(TRC);
11156 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVi16), VReg1)
11157 .
addImm(NumLPads & 0xFFFF)
11160 unsigned VReg2 = VReg1;
11161 if ((NumLPads & 0xFFFF0000) != 0) {
11162 VReg2 =
MRI->createVirtualRegister(TRC);
11163 BuildMI(DispatchBB, dl,
TII->get(ARM::MOVTi16), VReg2)
11169 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11179 Align Alignment = MF->getDataLayout().getPrefTypeAlign(
Int32Ty);
11182 Register VReg1 =
MRI->createVirtualRegister(TRC);
11183 BuildMI(DispatchBB, dl,
TII->get(ARM::LDRcp))
11188 BuildMI(DispatchBB, dl,
TII->get(ARM::CMPrr))
11199 Register NewVReg3 =
MRI->createVirtualRegister(TRC);
11200 BuildMI(DispContBB, dl,
TII->get(ARM::MOVsi), NewVReg3)
11205 Register NewVReg4 =
MRI->createVirtualRegister(TRC);
11206 BuildMI(DispContBB, dl,
TII->get(ARM::LEApcrelJT), NewVReg4)
11213 Register NewVReg5 =
MRI->createVirtualRegister(TRC);
11214 BuildMI(DispContBB, dl,
TII->get(ARM::LDRrs), NewVReg5)
11221 if (IsPositionIndependent) {
11222 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTadd))
11227 BuildMI(DispContBB, dl,
TII->get(ARM::BR_JTr))
11236 if (SeenMBBs.
insert(CurMBB).second)
11248 while (!Successors.empty()) {
11257 BB->normalizeSuccProbs();
11264 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
11265 if (!II->isCall())
continue;
11269 OI = II->operands_begin(), OE = II->operands_end();
11271 if (!OI->isReg())
continue;
11272 DefRegs[OI->getReg()] =
true;
11277 for (
unsigned i = 0; SavedRegs[i] != 0; ++i) {
11278 unsigned Reg = SavedRegs[i];
11280 !ARM::tGPRRegClass.contains(Reg) &&
11281 !ARM::hGPRRegClass.contains(Reg))
11283 if (Subtarget->
isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
11285 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
11298 MBBLPad->setIsEHPad(
false);
11301 MI.eraseFromParent();
11314static unsigned getLdOpcode(
unsigned LdSize,
bool IsThumb1,
bool IsThumb2) {
11316 return LdSize == 16 ? ARM::VLD1q32wb_fixed
11317 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
11319 return LdSize == 4 ? ARM::tLDRi
11320 : LdSize == 2 ? ARM::tLDRHi
11321 : LdSize == 1 ? ARM::tLDRBi : 0;
11323 return LdSize == 4 ? ARM::t2LDR_POST
11324 : LdSize == 2 ? ARM::t2LDRH_POST
11325 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
11326 return LdSize == 4 ? ARM::LDR_POST_IMM
11327 : LdSize == 2 ? ARM::LDRH_POST
11328 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
11333static unsigned getStOpcode(
unsigned StSize,
bool IsThumb1,
bool IsThumb2) {
11335 return StSize == 16 ? ARM::VST1q32wb_fixed
11336 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
11338 return StSize == 4 ? ARM::tSTRi
11339 : StSize == 2 ? ARM::tSTRHi
11340 : StSize == 1 ? ARM::tSTRBi : 0;
11342 return StSize == 4 ? ARM::t2STR_POST
11343 : StSize == 2 ? ARM::t2STRH_POST
11344 : StSize == 1 ? ARM::t2STRB_POST : 0;
11345 return StSize == 4 ? ARM::STR_POST_IMM
11346 : StSize == 2 ? ARM::STRH_POST
11347 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
11354 unsigned LdSize,
unsigned Data,
unsigned AddrIn,
11355 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11356 unsigned LdOpc =
getLdOpcode(LdSize, IsThumb1, IsThumb2);
11357 assert(LdOpc != 0 &&
"Should have a load opcode");
11364 }
else if (IsThumb1) {
11370 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11375 }
else if (IsThumb2) {
11395 unsigned StSize,
unsigned Data,
unsigned AddrIn,
11396 unsigned AddrOut,
bool IsThumb1,
bool IsThumb2) {
11397 unsigned StOpc =
getStOpcode(StSize, IsThumb1, IsThumb2);
11398 assert(StOpc != 0 &&
"Should have a store opcode");
11400 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11405 }
else if (IsThumb1) {
11412 BuildMI(*BB, Pos, dl,
TII->get(ARM::tADDi8), AddrOut)
11417 }
else if (IsThumb2) {
11418 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11424 BuildMI(*BB, Pos, dl,
TII->get(StOpc), AddrOut)
11445 unsigned SizeVal =
MI.getOperand(2).getImm();
11446 unsigned Alignment =
MI.getOperand(3).getImm();
11451 unsigned UnitSize = 0;
11456 bool IsThumb2 = Subtarget->
isThumb2();
11457 bool IsThumb = Subtarget->isThumb();
11459 if (Alignment & 1) {
11461 }
else if (Alignment & 2) {
11466 Subtarget->hasNEON()) {
11467 if ((Alignment % 16 == 0) && SizeVal >= 16)
11469 else if ((Alignment % 8 == 0) && SizeVal >= 8)
11478 bool IsNeon = UnitSize >= 8;
11479 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
11481 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
11482 : UnitSize == 8 ? &ARM::DPRRegClass
11485 unsigned BytesLeft = SizeVal % UnitSize;
11486 unsigned LoopSize = SizeVal - BytesLeft;
11488 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
11492 unsigned srcIn = src;
11493 unsigned destIn = dest;
11494 for (
unsigned i = 0; i < LoopSize; i+=UnitSize) {
11495 Register srcOut =
MRI.createVirtualRegister(TRC);
11496 Register destOut =
MRI.createVirtualRegister(TRC);
11497 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11499 IsThumb1, IsThumb2);
11501 IsThumb1, IsThumb2);
11509 for (
unsigned i = 0; i < BytesLeft; i++) {
11510 Register srcOut =
MRI.createVirtualRegister(TRC);
11511 Register destOut =
MRI.createVirtualRegister(TRC);
11512 Register scratch =
MRI.createVirtualRegister(TRC);
11514 IsThumb1, IsThumb2);
11516 IsThumb1, IsThumb2);
11520 MI.eraseFromParent();
11546 MF->
insert(It, loopMBB);
11547 MF->
insert(It, exitMBB);
11550 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
11560 Register varEnd =
MRI.createVirtualRegister(TRC);
11562 BuildMI(BB, dl,
TII->get(IsThumb ? ARM::t2MOVi32imm : ARM::MOVi32imm),
11565 }
else if (Subtarget->genExecuteOnly()) {
11566 assert(IsThumb &&
"Non-thumb expected to have used movt");
11602 Register varLoop =
MRI.createVirtualRegister(TRC);
11603 Register varPhi =
MRI.createVirtualRegister(TRC);
11604 Register srcLoop =
MRI.createVirtualRegister(TRC);
11605 Register srcPhi =
MRI.createVirtualRegister(TRC);
11606 Register destLoop =
MRI.createVirtualRegister(TRC);
11607 Register destPhi =
MRI.createVirtualRegister(TRC);
11615 BuildMI(BB, dl,
TII->get(ARM::PHI), destPhi)
11621 Register scratch =
MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
11623 IsThumb1, IsThumb2);
11625 IsThumb1, IsThumb2);
11629 BuildMI(*BB, BB->
end(), dl,
TII->get(ARM::tSUBi8), varLoop)
11637 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
11646 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
11655 auto StartOfExit = exitMBB->
begin();
11659 unsigned srcIn = srcLoop;
11660 unsigned destIn = destLoop;
11661 for (
unsigned i = 0; i < BytesLeft; i++) {
11662 Register srcOut =
MRI.createVirtualRegister(TRC);
11663 Register destOut =
MRI.createVirtualRegister(TRC);
11664 Register scratch =
MRI.createVirtualRegister(TRC);
11665 emitPostLd(BB, StartOfExit,
TII, dl, 1, scratch, srcIn, srcOut,
11666 IsThumb1, IsThumb2);
11667 emitPostSt(BB, StartOfExit,
TII, dl, 1, scratch, destIn, destOut,
11668 IsThumb1, IsThumb2);
11673 MI.eraseFromParent();
11685 "__chkstk is only supported on Windows");
11686 assert(Subtarget->
isThumb2() &&
"Windows on ARM requires Thumb-2 mode");
11706 switch (
TM.getCodeModel()) {
11748 MI.eraseFromParent();
11772 .
addReg(
MI.getOperand(0).getReg())
11780 MI.eraseFromParent();
11804 if (miI == BB->
end()) {
11806 if (Succ->isLiveIn(ARM::CPSR))
11812 SelectItr->addRegisterKilled(ARM::CPSR,
TRI);
11824 Register AddDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11825 BuildMI(TpEntry, Dl,
TII->get(ARM::t2ADDri), AddDestReg)
11831 Register LsrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11832 BuildMI(TpEntry, Dl,
TII->get(ARM::t2LSRri), LsrDestReg)
11838 Register TotalIterationsReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11839 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11842 BuildMI(TpEntry, Dl,
TII->get(ARM::t2WhileLoopStart))
11843 .
addUse(TotalIterationsReg)
11850 return TotalIterationsReg;
11861 Register TotalIterationsReg,
bool IsMemcpy) {
11868 SrcPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11869 CurrSrcReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11870 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), SrcPhiReg)
11878 Register DestPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11879 Register CurrDestReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11880 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), DestPhiReg)
11887 Register LoopCounterPhiReg =
MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11888 Register RemainingLoopIterationsReg =
11889 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11890 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), LoopCounterPhiReg)
11891 .
addUse(TotalIterationsReg)
11893 .
addUse(RemainingLoopIterationsReg)
11897 Register PredCounterPhiReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11898 Register RemainingElementsReg =
MRI.createVirtualRegister(&ARM::rGPRRegClass);
11899 BuildMI(TpLoopBody, Dl,
TII->get(ARM::PHI), PredCounterPhiReg)
11900 .
addUse(ElementCountReg)
11902 .
addUse(RemainingElementsReg)
11906 Register VccrReg =
MRI.createVirtualRegister(&ARM::VCCRRegClass);
11907 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VCTP8), VccrReg)
11908 .
addUse(PredCounterPhiReg)
11913 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2SUBri), RemainingElementsReg)
11914 .
addUse(PredCounterPhiReg)
11922 SrcValueReg =
MRI.createVirtualRegister(&ARM::MQPRRegClass);
11923 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VLDRBU8_post))
11932 SrcValueReg = OpSrcReg;
11934 BuildMI(TpLoopBody, Dl,
TII->get(ARM::MVE_VSTRBU8_post))
11945 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11946 .
addUse(LoopCounterPhiReg)
11949 BuildMI(TpLoopBody, Dl,
TII->get(ARM::t2LoopEnd))
11950 .
addUse(RemainingLoopIterationsReg)
11963 bool isThumb2 = Subtarget->
isThumb2();
11964 switch (
MI.getOpcode()) {
11971 case ARM::tLDR_postidx: {
11975 .
add(
MI.getOperand(2))
11976 .
add(
MI.getOperand(3))
11977 .
add(
MI.getOperand(4))
11978 .
add(
MI.getOperand(0))
11980 MI.eraseFromParent();
11984 case ARM::MVE_MEMCPYLOOPINST:
11985 case ARM::MVE_MEMSETLOOPINST: {
12015 Register OpDestReg =
MI.getOperand(0).getReg();
12016 Register OpSrcReg =
MI.getOperand(1).getReg();
12017 Register OpSizeReg =
MI.getOperand(2).getReg();
12037 if (TpExit == BB) {
12039 "block containing memcpy/memset Pseudo");
12052 bool IsMemcpy =
MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
12054 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
12069 MI.eraseFromParent();
12079 case ARM::t2STR_preidx:
12080 MI.setDesc(
TII->get(ARM::t2STR_PRE));
12082 case ARM::t2STRB_preidx:
12083 MI.setDesc(
TII->get(ARM::t2STRB_PRE));
12085 case ARM::t2STRH_preidx:
12086 MI.setDesc(
TII->get(ARM::t2STRH_PRE));
12089 case ARM::STRi_preidx:
12090 case ARM::STRBi_preidx: {
12091 unsigned NewOpc =
MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
12092 : ARM::STRB_PRE_IMM;
12094 unsigned Offset =
MI.getOperand(4).getImm();
12102 .
add(
MI.getOperand(0))
12103 .
add(
MI.getOperand(1))
12104 .
add(
MI.getOperand(2))
12106 .
add(
MI.getOperand(5))
12107 .
add(
MI.getOperand(6))
12109 MI.eraseFromParent();
12112 case ARM::STRr_preidx:
12113 case ARM::STRBr_preidx:
12114 case ARM::STRH_preidx: {
12116 switch (
MI.getOpcode()) {
12118 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG;
break;
12119 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG;
break;
12120 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE;
break;
12125 MI.eraseFromParent();
12129 case ARM::tMOVCCr_pseudo: {
12147 F->insert(It, copy0MBB);
12148 F->insert(It, sinkMBB);
12151 unsigned CallFrameSize =
TII->getCallFrameSizeAt(
MI);
12157 if (!
MI.killsRegister(ARM::CPSR,
nullptr) &&
12173 .
addImm(
MI.getOperand(3).getImm())
12174 .
addReg(
MI.getOperand(4).getReg());
12189 .
addReg(
MI.getOperand(1).getReg())
12191 .
addReg(
MI.getOperand(2).getReg())
12194 MI.eraseFromParent();
12199 case ARM::BCCZi64: {
12205 bool RHSisZero =
MI.getOpcode() == ARM::BCCZi64;
12210 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12214 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12220 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12224 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
12234 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
12243 MI.eraseFromParent();
12247 case ARM::Int_eh_sjlj_setjmp:
12248 case ARM::Int_eh_sjlj_setjmp_nofp:
12249 case ARM::tInt_eh_sjlj_setjmp:
12250 case ARM::t2Int_eh_sjlj_setjmp:
12251 case ARM::t2Int_eh_sjlj_setjmp_nofp:
12254 case ARM::Int_eh_sjlj_setup_dispatch:
12255 EmitSjLjDispatchBlock(
MI, BB);
12278 Fn->
insert(BBI, SinkBB);
12280 Register ABSSrcReg =
MI.getOperand(1).getReg();
12281 Register ABSDstReg =
MI.getOperand(0).getReg();
12282 bool ABSSrcKIll =
MI.getOperand(1).isKill();
12283 bool isThumb2 = Subtarget->
isThumb2();
12287 Register NewRsbDstReg =
MRI.createVirtualRegister(
12288 isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
12302 BuildMI(BB, dl,
TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
12309 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).
addMBB(SinkBB)
12316 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
12325 TII->get(ARM::PHI), ABSDstReg)
12330 MI.eraseFromParent();
12335 case ARM::COPY_STRUCT_BYVAL_I32:
12337 return EmitStructByval(
MI, BB);
12338 case ARM::WIN__CHKSTK:
12339 return EmitLowered__chkstk(
MI, BB);
12340 case ARM::WIN__DBZCHK:
12341 return EmitLowered__dbzchk(
MI, BB);
12358 if (!
Node->hasAnyUseOfValue(0)) {
12359 MI.getOperand(0).setIsDead(
true);
12361 if (!
Node->hasAnyUseOfValue(1)) {
12362 MI.getOperand(1).setIsDead(
true);
12366 for (
unsigned I = 0;
I !=
MI.getOperand(4).getImm(); ++
I) {
12367 Register TmpReg =
MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
12368 : &ARM::GPRRegClass);
12375 if (
MI.getOpcode() == ARM::MEMCPY) {
12393 MCID = &
TII->get(NewOpc);
12396 MI.getDesc().getNumOperands() + 5 -
MI.getDesc().getSize()
12397 &&
"converted opcode should be the same except for cc_out"
12398 " (and, on Thumb1, pred)");
12408 MI.addOperand(
MI.getOperand(1));
12409 MI.removeOperand(1);
12413 for (
unsigned i =
MI.getNumOperands(); i--;) {
12415 if (
op.isReg() &&
op.isUse()) {
12418 MI.tieOperands(DefIdx, i);
12432 if (!
MI.hasOptionalDef() || !MCID->
operands()[ccOutIdx].isOptionalDef()) {
12433 assert(!NewOpc &&
"Optional cc_out operand required");
12438 bool definesCPSR =
false;
12439 bool deadCPSR =
false;
12440 for (
unsigned i = MCID->
getNumOperands(), e =
MI.getNumOperands(); i != e;
12444 definesCPSR =
true;
12447 MI.removeOperand(i);
12451 if (!definesCPSR) {
12452 assert(!NewOpc &&
"Optional cc_out operand required");
12455 assert(deadCPSR == !Node->hasAnyUseOfValue(1) &&
"inconsistent dead flag");
12457 assert(!
MI.getOperand(ccOutIdx).getReg() &&
12458 "expect uninitialized optional cc_out operand");
12496 switch (
N->getOpcode()) {
12497 default:
return false;
12499 CC =
N->getOperand(0);
12521 EVT VT =
N->getValueType(0);
12522 CC =
N->getOperand(0);
12523 if (
CC.getValueType() != MVT::i1 ||
CC.getOpcode() !=
ISD::SETCC)
12569 EVT VT =
N->getValueType(0);
12572 bool SwapSelectOps;
12574 NonConstantVal, DAG))
12580 OtherOp, NonConstantVal);
12586 CCOp, TrueVal, FalseVal);
12610 if (
N->getOpcode() ==
ARMISD::VTRN &&
N->getValueType(0) == MVT::v2i32)
12625 if (!
N->getValueType(0).is64BitVector())
12633 EVT VT =
N->getValueType(0);
12672 EVT VT =
N->getValueType(0);
12678 Opcode = Intrinsic::arm_neon_vpaddls;
12680 Opcode = Intrinsic::arm_neon_vpaddlu;
12708 EVT VT =
N->getValueType(0);
12723 unsigned nextIndex = 0;
12791 return DAG.
getNode(ExtOp, dl, VT, tmp);
12822 if (SRA.getOpcode() !=
ISD::SRA) {
12828 if (
auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) {
12829 if (Const->getZExtValue() != 31)
12834 if (SRA.getOperand(0) !=
Mul)
12838 SDLoc dl(AddcNode);
12839 unsigned Opcode = 0;
12845 Op0 =
Mul.getOperand(0);
12846 Op1 =
Mul.getOperand(1);
12849 Op0 =
Mul.getOperand(0);
12850 Op1 =
Mul.getOperand(1).getOperand(0);
12853 Op0 =
Mul.getOperand(0).getOperand(0);
12854 Op1 =
Mul.getOperand(1);
12857 Op0 =
Mul->getOperand(0).getOperand(0);
12858 Op1 =
Mul->getOperand(1).getOperand(0);
12867 SDValue HiMLALResult(SMLAL.getNode(), 1);
12868 SDValue LoMLALResult(SMLAL.getNode(), 0);
12874 SDValue resNode(AddcNode, 0);
12903 "Expect an ADDE or SUBE");
12907 "ADDE node has the wrong inputs");
12926 "Expect ADDC with two result values. First: i32");
12946 bool IsLeftOperandMUL =
false;
12951 IsLeftOperandMUL =
true;
12962 SDValue *LowAddSub =
nullptr;
12965 if ((AddeSubeOp0 != MULOp.
getValue(1)) && (AddeSubeOp1 != MULOp.
getValue(1)))
12968 if (IsLeftOperandMUL)
12969 HiAddSub = &AddeSubeOp1;
12971 HiAddSub = &AddeSubeOp0;
12976 if (AddcSubcOp0 == MULOp.
getValue(0)) {
12977 LoMul = &AddcSubcOp0;
12978 LowAddSub = &AddcSubcOp1;
12980 if (AddcSubcOp1 == MULOp.
getValue(0)) {
12981 LoMul = &AddcSubcOp1;
12982 LowAddSub = &AddcSubcOp0;
12990 if (AddcSubcNode == HiAddSub->getNode() ||
13006 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->
useMulOps() &&
13020 return SDValue(AddeSubeNode, 0);
13031 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13041 return SDValue(AddeSubeNode, 0);
13053 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13062 SDNode *UmlalNode =
nullptr;
13087 DAG.
getVTList(MVT::i32, MVT::i32), Ops);
13101 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
13106 SDNode* AddcNode =
N->getOperand(2).getNode();
13107 SDNode* AddeNode =
N->getOperand(3).getNode();
13115 {N->getOperand(0), N->getOperand(1),
13116 AddcNode->getOperand(0), AddcNode->getOperand(1)});
13140 int32_t imm =
C->getSExtValue();
13141 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
13146 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
N->getOperand(0),
RHS);
13161 int64_t imm =
C->getSExtValue();
13172 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
13173 N->getOperand(0),
RHS,
N->getOperand(2));
13185 if (!Subtarget->hasMVEIntegerOps())
13198 SetCC =
N->getOperand(0);
13201 CC = cast<CondCodeSDNode>(SetCC->
getOperand(2))->get();
13202 TrueVal =
N->getOperand(1);
13203 FalseVal =
N->getOperand(2);
13205 LHS =
N->getOperand(0);
13206 RHS =
N->getOperand(1);
13207 CC = cast<CondCodeSDNode>(
N->getOperand(4))->get();
13208 TrueVal =
N->getOperand(2);
13209 FalseVal =
N->getOperand(3);
13214 unsigned int Opcode = 0;
13243 switch (TrueVal->getOpcode()) {
13262 if (TrueVal !=
LHS || FalseVal !=
RHS)
13265 EVT LeftType =
LHS->getValueType(0);
13266 EVT RightType =
RHS->getValueType(0);
13269 if (LeftType != VectorScalarType || RightType != VectorScalarType)
13273 if (VectorScalarType != MVT::i32)
13281 if (VectorScalarType != MVT::i32)
13294 EVT VT =
N->getValueType(0);
13302 Shft =
N->getOperand(0);
13308 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() !=
ISD::SETLT ||
13309 Cmp.getOperand(0) !=
N->getOperand(1) ||
13310 Cmp.getOperand(1) !=
N->getOperand(2))
13312 Shft =
N->getOperand(1);
13324 ScalarType = MVT::i8;
13327 case (1 << 15) - 1:
13328 ScalarType = MVT::i16;
13331 case (1ULL << 31) - 1:
13332 ScalarType = MVT::i32;
13363 unsigned LegalLanes = 128 / (ShftAmt + 1);
13387 for (
unsigned I = 0;
I < NumParts; ++
I) {
13404 if (!Subtarget->hasMVEIntegerOps())
13419 if (
N->getOperand(0).getOpcode() !=
ISD::XOR)
13429 if (!Const || !Const->isOne())
13447 EVT VT =
N->getValueType(0);
13449 if (!Subtarget->hasMVEIntegerOps() ||
13478 Opc = Intrinsic::arm_mve_vctp64;
13481 Opc = Intrinsic::arm_mve_vctp32;
13484 Opc = Intrinsic::arm_mve_vctp16;
13487 Opc = Intrinsic::arm_mve_vctp8;
13553 EVT VT =
N->getValueType(0);
13559 switch (
Op.getOpcode()) {
13576 !isa<ConstantSDNode>(N0) && N1->
hasOneUse()) {
13584 unsigned N0RedOp = 0;
13591 unsigned N1RedOp = 0;
13605 if (
SDValue R = DistrubuteAddAddVecReduce(N0, N1))
13607 if (
SDValue R = DistrubuteAddAddVecReduce(N1, N0))
13614 auto DistrubuteVecReduceLoad = [&](
SDValue N0,
SDValue N1,
bool IsForward) {
13628 LoadSDNode *Load0 = dyn_cast<LoadSDNode>(N0);
13629 LoadSDNode *Load1 = dyn_cast<LoadSDNode>(N1);
13638 if (!BaseLocDecomp0.getBase() ||
13639 BaseLocDecomp0.getBase() != BaseLocDecomp1.getBase() ||
13640 !BaseLocDecomp0.hasValidOffset() || !BaseLocDecomp1.hasValidOffset())
13642 if (BaseLocDecomp0.getOffset() < BaseLocDecomp1.getOffset())
13644 if (BaseLocDecomp0.getOffset() > BaseLocDecomp1.getOffset())
13654 if (IsBefore < 0) {
13657 }
else if (IsBefore > 0) {
13670 }
else if (IsForward && IsVecReduce(N0) && IsVecReduce(N1) &&
13680 if (!IsVecReduce(N0) || !IsVecReduce(N1))
13690 if (
SDValue R = DistrubuteVecReduceLoad(N0, N1,
true))
13692 if (
SDValue R = DistrubuteVecReduceLoad(N1, N0,
false))
13699 if (!Subtarget->hasMVEIntegerOps())
13705 EVT VT =
N->getValueType(0);
13710 if (VT != MVT::i64)
13721 auto MakeVecReduce = [&](
unsigned Opcode,
unsigned OpcodeA,
SDValue NA,
13739 std::tie(Ops[0], Ops[1]) = DAG.
SplitScalar(NA, dl, MVT::i32, MVT::i32);
13741 unsigned S = VecRed->
getOpcode() == OpcodeA ? 2 : 0;
13790 "Expected shift op");
13808 if (
auto *Const = dyn_cast<ConstantSDNode>(N1->
getOperand(1))) {
13809 if (Const->getAPIntValue().ult(256))
13812 Const->getAPIntValue().sgt(-256))
13828 (
N->getOperand(0).getOpcode() ==
ISD::SHL ||
13829 N->getOperand(0).getOpcode() ==
ISD::SRL) &&
13830 "Expected XOR(SHIFT) pattern");
13833 auto *XorC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13834 auto *ShiftC = dyn_cast<ConstantSDNode>(
N->getOperand(0).getOperand(1));
13835 if (XorC && ShiftC) {
13836 unsigned MaskIdx, MaskLen;
13837 if (XorC->getAPIntValue().isShiftedMask(MaskIdx, MaskLen)) {
13838 unsigned ShiftAmt = ShiftC->getZExtValue();
13839 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
13840 if (
N->getOperand(0).getOpcode() ==
ISD::SHL)
13841 return MaskIdx == ShiftAmt && MaskLen == (
BitWidth - ShiftAmt);
13842 return MaskIdx == 0 && MaskLen == (
BitWidth - ShiftAmt);
13852 N->getOperand(0).getOpcode() ==
ISD::SRL) ||
13854 N->getOperand(0).getOpcode() ==
ISD::SHL)) &&
13855 "Expected shift-shift mask");
13868 return Subtarget->hasMVEIntegerOps() &&
isTypeLegal(VT);
13872 if (!Subtarget->hasNEON()) {
13891 return Subtarget->hasFP64();
13894 return Subtarget->hasMVEFloatOps();
13923 if (ST->isThumb() && ST->isThumb1Only())
13927 for (
auto *U :
N->uses()) {
13928 switch(U->getOpcode()) {
13941 if (isa<ConstantSDNode>(U->getOperand(0)) ||
13942 isa<ConstantSDNode>(U->getOperand(1)))
13946 if (U->getOperand(0).getOpcode() ==
ISD::SHL ||
13947 U->getOperand(1).getOpcode() ==
ISD::SHL)
13957 if (
N->getOperand(0).getOpcode() !=
ISD::SHL)
13962 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13963 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1));
13964 if (!C1ShlC2 || !C2)
13967 APInt C2Int = C2->getAPIntValue();
13968 APInt C1Int = C1ShlC2->getAPIntValue();
13970 if (C2Int.
uge(C2Width))
13976 if ((C1Int & Mask) != C1Int)
13983 auto LargeImm = [](
const APInt &Imm) {
13984 unsigned Zeros = Imm.countl_zero() + Imm.countr_zero();
13985 return Imm.getBitWidth() - Zeros > 8;
13988 if (LargeImm(C1Int) || LargeImm(C2Int))
14000 SHL.dump();
N->dump());
14035 if (CSINC.getOpcode() !=
ARMISD::CSINC || !CSINC.hasOneUse())
14044 CSINC.getOperand(0)),
14045 CSINC.getOperand(1), CSINC.getOperand(2),
14046 CSINC.getOperand(3));
14065 if (!Subtarget->hasMVEIntegerOps() || !
N->getValueType(0).isVector())
14107 if (!Subtarget->hasVMLxForwarding())
14126 EVT VT =
N->getValueType(0);
14137 EVT VT =
N->getValueType(0);
14138 if (VT != MVT::v2i64)
14147 EVT VT = cast<VTSDNode>(
Op->getOperand(1))->getVT();
14149 return Op->getOperand(0);
14163 And =
And->getOperand(0);
14168 Mask = Mask->getOperand(0);
14171 Mask.getValueType() != MVT::v4i32)
14177 return And->getOperand(0);
14182 if (
SDValue Op0 = IsSignExt(N0)) {
14183 if (
SDValue Op1 = IsSignExt(N1)) {
14189 if (
SDValue Op0 = IsZeroExt(N0)) {
14190 if (
SDValue Op1 = IsZeroExt(N1)) {
14205 EVT VT =
N->getValueType(0);
14206 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
14217 if (VT != MVT::i32)
14224 int64_t MulAmt =
C->getSExtValue();
14225 unsigned ShiftAmt = llvm::countr_zero<uint64_t>(MulAmt);
14227 ShiftAmt = ShiftAmt & (32 - 1);
14232 MulAmt >>= ShiftAmt;
14235 if (llvm::has_single_bit<uint32_t>(MulAmt - 1)) {
14243 }
else if (llvm::has_single_bit<uint32_t>(MulAmt + 1)) {
14255 if (llvm::has_single_bit<uint32_t>(MulAmtAbs + 1)) {
14263 }
else if (llvm::has_single_bit<uint32_t>(MulAmtAbs - 1)) {
14293 if (
N->getValueType(0) != MVT::i32)
14302 if (C1 == 255 || C1 == 65535)
14305 SDNode *N0 =
N->getOperand(0).getNode();
14319 if (!C2 || C2 >= 32)
14363 if (Trailing == C2 && C2 + C3 < 32) {
14376 if (Leading == C2 && C2 + C3 < 32) {
14396 EVT VT =
N->getValueType(0);
14400 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)
14403 APInt SplatBits, SplatUndef;
14404 unsigned SplatBitSize;
14406 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14407 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14408 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14409 SplatBitSize == 64) {
14443 if (!Subtarget->hasV6Ops() ||
14444 (Subtarget->isThumb() &&
14445 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
14448 SDValue SRL = OR->getOperand(0);
14449 SDValue SHL = OR->getOperand(1);
14452 SRL = OR->getOperand(1);
14453 SHL = OR->getOperand(0);
14460 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
14464 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
14465 if (SRL.getOperand(0) !=
SDValue(SMULLOHI, 0) ||
14466 SHL.getOperand(0) !=
SDValue(SMULLOHI, 1))
14485 unsigned Opcode = 0;
14486 if (
isS16(OpS16, DAG))
14504 if (Subtarget->
isThumb1Only() || !Subtarget->hasV6T2Ops())
14507 EVT VT =
N->getValueType(0);
14522 if (VT != MVT::i32)
14535 if (Mask == 0xffff)
14542 if ((Val & ~Mask) != Val)
14567 (Mask == ~Mask2)) {
14570 if (Subtarget->hasDSP() &&
14571 (Mask == 0xffff || Mask == 0xffff0000))
14584 (~Mask == Mask2)) {
14587 if (Subtarget->hasDSP() &&
14588 (Mask2 == 0xffff || Mask2 == 0xffff0000))
14661 EVT VT =
N->getValueType(0);
14666 auto IsFreelyInvertable = [&](
SDValue V) {
14673 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
14689 EVT VT =
N->getValueType(0);
14695 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 ||
14696 VT == MVT::v8i1 || VT == MVT::v16i1))
14699 APInt SplatBits, SplatUndef;
14700 unsigned SplatBitSize;
14702 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
14703 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
14704 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
14705 SplatBitSize == 64) {
14741 unsigned SplatBitSize;
14744 APInt SplatBits0, SplatBits1;
14748 if (BVN0 && BVN0->
isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
14749 HasAnyUndefs) && !HasAnyUndefs) {
14750 if (BVN1 && BVN1->
isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
14751 HasAnyUndefs) && !HasAnyUndefs) {
14756 SplatBits0 == ~SplatBits1) {
14786 EVT VT =
N->getValueType(0);
14801 if (Subtarget->hasMVEIntegerOps()) {
14832 ToMask = ~N->getConstantOperandAPInt(2);
14838 isa<ConstantSDNode>(
From->getOperand(1))) {
14839 APInt Shift =
From->getConstantOperandAPInt(1);
14852 unsigned LastActiveBitInA =
A.countr_zero();
14853 unsigned FirstActiveBitInB =
B.getBitWidth() -
B.countl_zero() - 1;
14854 return LastActiveBitInA - 1 == FirstActiveBitInB;
14859 APInt ToMask, FromMask;
14867 APInt NewToMask, NewFromMask;
14869 if (NewFrom !=
From)
14873 if ((NewToMask & ToMask).getBoolValue())
14898 unsigned InvMask =
N->getConstantOperandVal(2);
14900 unsigned Width = llvm::bit_width<unsigned>(~InvMask) - LSB;
14902 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
14903 "undefined behavior");
14904 unsigned Mask = (1u << Width) - 1;
14906 if ((Mask & (~Mask2)) == 0)
14908 N->getOperand(0), N1.
getOperand(0),
N->getOperand(2));
14915 APInt ToMask1, FromMask1;
14918 APInt ToMask2, FromMask2;
14924 APInt NewFromMask = FromMask1 | FromMask2;
14925 APInt NewToMask = ToMask1 | ToMask2;
14927 EVT VT =
N->getValueType(0);
14930 if (NewFromMask[0] == 0)
14942 APInt ToMask1 = ~N->getConstantOperandAPInt(2);
14943 APInt ToMask2 = ~N0.getConstantOperandAPInt(2);
14945 if (!N0.
hasOneUse() || (ToMask1 & ToMask2) != 0 ||
14949 EVT VT =
N->getValueType(0);
14952 N->getOperand(1),
N->getOperand(2));
14966 SDValue CSInc = Cmp->getOperand(0);
15016 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ)
15017 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15020 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
15022 N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
15035 SDValue InDouble =
N->getOperand(0);
15044 !cast<LoadSDNode>(InNode)->isVolatile()) {
15050 SDValue BasePtr = LD->getBasePtr();
15052 DAG.
getLoad(MVT::i32,
DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
15053 LD->getAlign(), LD->getMemOperand()->getFlags());
15059 LD->getPointerInfo().getWithOffset(4),
15061 LD->getMemOperand()->getFlags());
15073 isa<ConstantSDNode>(InDouble.
getOperand(1))) {
15094 if (!Subtarget->
isLittle() && BVSwap)
15104 if (isa<ConstantSDNode>(BV.
getOperand(2))) {
15112 if (!Subtarget->
isLittle() && BVSwap)
15157 if (Copy.getValueType() == MVT::f32 &&
15159 bool HasGlue = Copy->getNumOperands() == 3;
15160 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1),
15161 HasGlue ? Copy->getOperand(2) :
SDValue()};
15162 EVT OutTys[] = {
N->getValueType(0), MVT::Other, MVT::Glue};
15180 if (
LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Op0)) {
15181 if (LN0->hasOneUse() && LN0->isUnindexed() &&
15182 LN0->getMemoryVT() == MVT::i16) {
15185 LN0->getBasePtr(), LN0->getMemOperand());
15203 EVT VT =
N->getValueType(0);
15237 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
15238 for (
unsigned i = 0; i < NumElts; ++i) {
15239 SDNode *Elt =
N->getOperand(i).getNode();
15256 if (
N->getNumOperands() == 2)
15262 EVT VT =
N->getValueType(0);
15268 for (
unsigned i = 0; i < NumElts; ++i) {
15294 EVT VT =
N->getValueType(0);
15302 assert(EltVT == MVT::f32 &&
"Unexpected type!");
15307 Use->getValueType(0).isFloatingPoint())
15315 unsigned NumOfBitCastedElts = 0;
15317 unsigned NumOfRelevantElts = NumElts;
15318 for (
unsigned Idx = 0;
Idx < NumElts; ++
Idx) {
15323 ++NumOfBitCastedElts;
15324 }
else if (Elt.
isUndef() || isa<ConstantSDNode>(Elt))
15327 --NumOfRelevantElts;
15331 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
15349 for (
unsigned Idx = 0 ;
Idx < NumElts; ++
Idx) {
15354 V->getOperand(0).getValueType() == MVT::i32)
15356 V = V.getOperand(0);
15373 EVT VT =
N->getValueType(0);
15380 if (
Op->getOperand(0).getValueType() == VT)
15381 return Op->getOperand(0);
15396 if (
Op.getValueType() == MVT::i32) {
15407 EVT VT =
N->getValueType(0);
15412 if (ST->isLittle())
15422 if (
Op->getOperand(0).getValueType() == VT)
15423 return Op->getOperand(0);
15432 if (!Subtarget->hasMVEIntegerOps())
15435 EVT VT =
N->getValueType(0);
15466 EVT VT =
N->getValueType(0);
15467 SDNode *Elt =
N->getOperand(1).getNode();
15482 Vec, V,
N->getOperand(2));
15492 EVT VT =
N->getValueType(0);
15501 Ext.getOperand(0).getValueType() == MVT::f32)
15502 Ext = Ext.getOperand(0);
15504 !isa<ConstantSDNode>(Ext.getOperand(1)) ||
15505 Ext.getConstantOperandVal(1) % 2 != 0)
15507 if (Ext->use_size() == 1 &&
15512 SDValue Op0 = Ext.getOperand(0);
15515 unsigned Lane = Ext.getConstantOperandVal(1);
15521 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15522 isa<ConstantSDNode>(V->getOperand(1)) &&
15523 V->getConstantOperandVal(1) == Lane + 1 &&
15524 V->getOperand(0).getResNo() == ResNo;
15526 if (OtherIt == Op0->
uses().end())
15531 SDValue OtherExt(*OtherIt, 0);
15544 DCI.
DAG.
getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
15556 EVT VT =
N->getValueType(0);
15562 if (VT == MVT::f16 &&
X.getValueType() == MVT::i32)
15564 if (VT == MVT::i32 &&
X.getValueType() == MVT::f16)
15566 if (VT == MVT::f32 &&
X.getValueType() == MVT::i32)
15569 while (
X.getValueType() != VT &&
X->getOpcode() ==
ISD::BITCAST)
15570 X =
X->getOperand(0);
15571 if (
X.getValueType() == VT)
15577 isa<ConstantSDNode>(
N->getOperand(1)) &&
15579 return Op0.
getOperand(
N->getConstantOperandVal(1));
15584 isa<ConstantSDNode>(
N->getOperand(1)) &&
15589 unsigned Offset =
N->getConstantOperandVal(1);
15601 unsigned Idx =
N->getConstantOperandVal(1);
15615 EVT VT =
N->getValueType(0);
15619 cast<VTSDNode>(
N->getOperand(1))->getVT() ==
15620 Op.getOperand(0).getValueType().getScalarType())
15630 SDValue SubVec =
N->getOperand(1);
15631 uint64_t IdxVal =
N->getConstantOperandVal(2);
15642 if (IdxVal == 0 && Vec.
isUndef())
15648 (IdxVal != 0 && IdxVal != NumSubElts))
15721 EVT VT =
N->getValueType(0);
15732 unsigned HalfElts = NumElts/2;
15734 for (
unsigned n = 0; n < NumElts; ++n) {
15737 if (MaskElt < (
int)HalfElts)
15739 else if (MaskElt >= (
int)NumElts && MaskElt < (
int)(NumElts + HalfElts))
15740 NewElt = HalfElts + MaskElt - NumElts;
15767 bool SimpleConstIncOnly,
15775 bool isLoadOp =
true;
15776 bool isLaneOp =
false;
15779 bool hasAlignment =
true;
15780 unsigned NewOpc = 0;
15781 unsigned NumVecs = 0;
15782 if (
Target.isIntrinsic) {
15783 unsigned IntNo =
N->getConstantOperandVal(1);
15787 case Intrinsic::arm_neon_vld1:
15791 case Intrinsic::arm_neon_vld2:
15795 case Intrinsic::arm_neon_vld3:
15799 case Intrinsic::arm_neon_vld4:
15803 case Intrinsic::arm_neon_vld1x2:
15806 hasAlignment =
false;
15808 case Intrinsic::arm_neon_vld1x3:
15811 hasAlignment =
false;
15813 case Intrinsic::arm_neon_vld1x4:
15816 hasAlignment =
false;
15818 case Intrinsic::arm_neon_vld2dup:
15822 case Intrinsic::arm_neon_vld3dup:
15826 case Intrinsic::arm_neon_vld4dup:
15830 case Intrinsic::arm_neon_vld2lane:
15835 case Intrinsic::arm_neon_vld3lane:
15840 case Intrinsic::arm_neon_vld4lane:
15845 case Intrinsic::arm_neon_vst1:
15850 case Intrinsic::arm_neon_vst2:
15855 case Intrinsic::arm_neon_vst3:
15860 case Intrinsic::arm_neon_vst4:
15865 case Intrinsic::arm_neon_vst2lane:
15871 case Intrinsic::arm_neon_vst3lane:
15877 case Intrinsic::arm_neon_vst4lane:
15883 case Intrinsic::arm_neon_vst1x2:
15887 hasAlignment =
false;
15889 case Intrinsic::arm_neon_vst1x3:
15893 hasAlignment =
false;
15895 case Intrinsic::arm_neon_vst1x4:
15899 hasAlignment =
false;
15904 switch (
N->getOpcode()) {
15940 VecTy =
N->getValueType(0);
15941 }
else if (
Target.isIntrinsic) {
15942 VecTy =
N->getOperand(
Target.AddrOpIdx + 1).getValueType();
15945 "Node has to be a load, a store, or an intrinsic!");
15946 VecTy =
N->getOperand(1).getValueType();
15954 if (isLaneOp || isVLDDUPOp)
15957 if (NumBytes >= 3 * 16 &&
User.ConstInc != NumBytes) {
15963 if (SimpleConstIncOnly &&
User.ConstInc != NumBytes)
15969 EVT AlignedVecTy = VecTy;
15986 if (isa<LSBaseSDNode>(
N)) {
15989 assert(NumVecs == 1 &&
"Unexpected multi-element generic load/store.");
15990 assert(!isLaneOp &&
"Unexpected generic load/store lane.");
16001 Alignment =
Align(1);
16007 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16009 for (n = 0; n < NumResultVecs; ++n)
16010 Tys[n] = AlignedVecTy;
16011 Tys[n++] = MVT::i32;
16012 Tys[n] = MVT::Other;
16027 unsigned LastOperand =
16028 hasAlignment ?
N->getNumOperands() - 1 :
N->getNumOperands();
16029 for (
unsigned i =
Target.AddrOpIdx + 1; i < LastOperand; ++i)
16038 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::STORE) {
16049 for (
unsigned i = 0; i < NumResultVecs; ++i)
16054 if (AlignedVecTy != VecTy &&
N->getOpcode() ==
ISD::LOAD) {
16055 SDValue &LdVal = NewResults[0];
16091 switch (
N->getOpcode()) {
16094 if (isa<ConstantSDNode>(
N->getOperand(1))) {
16095 *
Ptr =
N->getOperand(0);
16096 *CInc =
N->getOperand(1);
16102 if (isa<ConstantSDNode>(
N->getOperand(2))) {
16103 *
Ptr =
N->getOperand(1);
16104 *CInc =
N->getOperand(2);
16139 const unsigned AddrOpIdx = ((isIntrinsic ||
isStore) ? 2 : 1);
16148 UE =
Addr.getNode()->use_end(); UI != UE; ++UI) {
16150 if (UI.getUse().getResNo() !=
Addr.getResNo() ||
16155 unsigned ConstInc =
16173 if (UI.getUse().getResNo() !=
Base.getResNo() ||
User ==
Addr.getNode() ||
16178 unsigned UserOffset =
16181 if (!UserOffset || UserOffset <=
Offset)
16184 unsigned NewConstInc = UserOffset -
Offset;
16194 unsigned NumValidUpd = BaseUpdates.
size();
16195 for (
unsigned I = 0;
I < NumValidUpd;) {
16199 std::swap(BaseUpdates[
I], BaseUpdates[NumValidUpd]);
16207 BaseUpdates.
resize(NumValidUpd);
16212 std::stable_sort(BaseUpdates.
begin(), BaseUpdates.
end(),
16214 return LHS.ConstInc < RHS.ConstInc;
16243 unsigned IntNo =
N->getConstantOperandVal(1);
16244 if (IntNo == Intrinsic::arm_mve_vst2q &&
N->getConstantOperandVal(5) != 1)
16246 if (IntNo == Intrinsic::arm_mve_vst4q &&
N->getConstantOperandVal(7) != 3)
16251 UE =
Addr.getNode()->use_end();
16255 UI.getUse().getResNo() !=
Addr.getResNo())
16271 bool isLoadOp =
true;
16272 unsigned NewOpc = 0;
16273 unsigned NumVecs = 0;
16277 case Intrinsic::arm_mve_vld2q:
16281 case Intrinsic::arm_mve_vld4q:
16285 case Intrinsic::arm_mve_vst2q:
16290 case Intrinsic::arm_mve_vst4q:
16300 VecTy =
N->getValueType(0);
16302 VecTy =
N->getOperand(3).getValueType();
16316 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
16318 for (n = 0; n < NumResultVecs; ++n)
16320 Tys[n++] = MVT::i32;
16321 Tys[n] = MVT::Other;
16330 for (
unsigned i = 3; i <
N->getNumOperands(); ++i)
16338 for (
unsigned i = 0; i < NumResultVecs; ++i)
16357 EVT VT =
N->getValueType(0);
16363 SDNode *VLD =
N->getOperand(0).getNode();
16366 unsigned NumVecs = 0;
16367 unsigned NewOpc = 0;
16369 if (IntNo == Intrinsic::arm_neon_vld2lane) {
16372 }
else if (IntNo == Intrinsic::arm_neon_vld3lane) {
16375 }
else if (IntNo == Intrinsic::arm_neon_vld4lane) {
16388 if (UI.getUse().getResNo() == NumVecs)
16392 VLDLaneNo !=
User->getConstantOperandVal(1))
16399 for (n = 0; n < NumVecs; ++n)
16401 Tys[n] = MVT::Other;
16412 unsigned ResNo = UI.getUse().
getResNo();
16414 if (ResNo == NumVecs)
16422 std::vector<SDValue> VLDDupResults;
16423 for (
unsigned n = 0; n < NumVecs; ++n)
16437 EVT VT =
N->getValueType(0);
16440 if (Subtarget->hasMVEIntegerOps()) {
16444 ExtractVT = MVT::i32;
16446 N->getOperand(0),
N->getOperand(1));
16458 Op =
Op.getOperand(0);
16463 unsigned EltSize =
Op.getScalarValueSizeInBits();
16465 unsigned Imm =
Op.getConstantOperandVal(0);
16481 if (Subtarget->hasMVEIntegerOps()) {
16484 if (
Op.getValueType() == MVT::f32)
16487 else if (
Op.getValueType() == MVT::f16)
16492 if (!Subtarget->hasNEON())
16499 if (LD &&
Op.hasOneUse() && LD->isUnindexed() &&
16500 LD->getMemoryVT() ==
N->getValueType(0).getVectorElementType()) {
16501 SDValue Ops[] = {LD->getOperand(0), LD->getOperand(1),
16506 LD->getMemoryVT(), LD->getMemOperand());
16517 EVT VT =
N->getValueType(0);
16539 assert(StVT != VT &&
"Cannot truncate to the same type");
16549 if (0 != (NumElems * FromEltSz) % ToEltSz)
16552 unsigned SizeRatio = FromEltSz / ToEltSz;
16557 NumElems * SizeRatio);
16563 for (
unsigned i = 0; i < NumElems; ++i)
16577 MVT StoreType = MVT::i8;
16579 if (TLI.
isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
16598 unsigned E = (ToEltSz * NumElems) / StoreType.
getSizeInBits();
16599 for (
unsigned I = 0;
I < E;
I++) {
16630 if (FromEltVT != MVT::f32 || ToEltVT != MVT::f16)
16633 unsigned NumElements = 4;
16650 unsigned Off0 = Rev ? NumElts : 0;
16651 unsigned Off1 = Rev ? 0 : NumElts;
16653 for (
unsigned I = 0;
I < NumElts;
I += 2) {
16654 if (M[
I] >= 0 && M[
I] != (
int)(Off0 +
I / 2))
16656 if (M[
I + 1] >= 0 && M[
I + 1] != (
int)(Off1 +
I / 2))
16663 if (
auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.
getOperand(0)))
16664 if (isVMOVNShuffle(Shuffle,
false) || isVMOVNShuffle(Shuffle,
true))
16684 unsigned NewOffset = i * NumElements * ToEltVT.
getSizeInBits() / 8;
16699 NewToVT, Alignment, MMOFlags, AAInfo);
16732 unsigned NewOffset =
16740 NewToVT, Alignment, MMOFlags, AAInfo);
16762 {Extract.getOperand(0), Extract.getOperand(1)});
16793 if (Subtarget->hasNEON())
16797 if (Subtarget->hasMVEFloatOps())
16801 if (Subtarget->hasMVEIntegerOps()) {
16877 if (!Subtarget->hasNEON())
16881 if (!
Op.getValueType().isVector() || !
Op.getValueType().isSimple() ||
16886 if (!isa<BuildVectorSDNode>(ConstVec))
16889 MVT FloatTy =
Op.getSimpleValueType().getVectorElementType();
16891 MVT IntTy =
N->getSimpleValueType(0).getVectorElementType();
16893 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
16894 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
16905 if (
C == -1 ||
C == 0 ||
C > 32)
16910 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
16911 Intrinsic::arm_neon_vcvtfp2fxu;
16914 DAG.
getConstant(IntrinsicOpcode, dl, MVT::i32),
Op->getOperand(0),
16917 if (IntBits < FloatBits)
16925 if (!Subtarget->hasMVEFloatOps())
16933 EVT VT =
N->getValueType(0);
16938 auto isIdentitySplat = [&](
SDValue Op,
bool NSZ) {
16942 uint64_t ImmVal =
Op.getOperand(0).getConstantOperandVal(0);
16943 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ)))
16945 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ)))
16958 if (!isIdentitySplat(Op1.
getOperand(2), NSZ))
16969 EVT VT =
N->getValueType(0);
16972 if (!
N->getFlags().hasAllowReassociation())
16979 unsigned Opc =
A.getConstantOperandVal(0);
16980 if (Opc != Intrinsic::arm_mve_vcmlaq)
16985 A.getOperand(3),
A.getOperand(4));
17017 if (!Subtarget->hasNEON())
17021 unsigned OpOpcode =
Op.getNode()->getOpcode();
17022 if (!
N->getValueType(0).isVector() || !
N->getValueType(0).isSimple() ||
17026 SDValue ConstVec =
N->getOperand(1);
17027 if (!isa<BuildVectorSDNode>(ConstVec))
17030 MVT FloatTy =
N->getSimpleValueType(0).getVectorElementType();
17032 MVT IntTy =
Op.getOperand(0).getSimpleValueType().getVectorElementType();
17034 unsigned NumLanes =
Op.getValueType().getVectorNumElements();
17035 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
17046 if (
C == -1 ||
C == 0 ||
C > 32)
17052 if (IntBits < FloatBits)
17054 dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
17057 unsigned IntrinsicOpcode =
isSigned ? Intrinsic::arm_neon_vcvtfxs2fp :
17058 Intrinsic::arm_neon_vcvtfxu2fp;
17067 if (!ST->hasMVEIntegerOps())
17071 EVT ResVT =
N->getValueType(0);
17099 EVT AVT =
A.getValueType();
17105 auto ExtendIfNeeded = [&](
SDValue A,
unsigned ExtendCode) {
17106 EVT AVT =
A.getValueType();
17118 if (ExtTypeMatches(
A, ExtTypes))
17119 return ExtendIfNeeded(
A, ExtendCode);
17122 auto IsPredVADDV = [&](
MVT RetTy,
unsigned ExtendCode,
17129 if (Ext->getOpcode() != ExtendCode)
17132 if (ExtTypeMatches(
A, ExtTypes))
17133 return ExtendIfNeeded(
A, ExtendCode);
17146 if (ResVT !=
RetTy)
17149 if (
Mul->getOpcode() == ExtendCode &&
17150 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17152 Mul =
Mul->getOperand(0);
17161 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17162 A = ExtendIfNeeded(
A, ExtendCode);
17163 B = ExtendIfNeeded(
B, ExtendCode);
17181 if (
Mul->getOpcode() == ExtendCode &&
17182 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
17184 Mul =
Mul->getOperand(0);
17193 if (ExtTypeMatches(
A, ExtTypes) && ExtTypeMatches(
B, ExtTypes)) {
17194 A = ExtendIfNeeded(
A, ExtendCode);
17195 B = ExtendIfNeeded(
B, ExtendCode);
17206 EVT VT = Ops[0].getValueType();
17207 if (VT == MVT::v16i8) {
17209 "Unexpected illegal long reduction opcode");
17214 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[0]);
17217 DAG.
getVTList(MVT::v8i16, MVT::v8i16), Ops[1]);
17238 if (IsVMLAV(MVT::i64,
ISD::SIGN_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17241 if (IsVMLAV(MVT::i64,
ISD::ZERO_EXTEND, {MVT::v16i8, MVT::v8i16, MVT::v4i32},
17305 Op =
Op->getOperand(1);
17307 Op->getOperand(0)->getOpcode() ==
ISD::MUL) {
17309 if (
Mul->getOperand(0) ==
Mul->getOperand(1) &&
17326 unsigned VecOp =
N->getOperand(0).getValueType().isVector() ? 0 : 2;
17327 auto *Shuf = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp));
17328 if (!Shuf || !Shuf->getOperand(1).isUndef())
17333 APInt SetElts(Mask.size(), 0);
17334 for (
int E : Mask) {
17335 if (E < 0 || E >= (
int)Mask.size())
17342 if (
N->getNumOperands() != VecOp + 1) {
17343 auto *Shuf2 = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(VecOp + 1));
17344 if (!Shuf2 || !Shuf2->getOperand(1).isUndef() || Shuf2->getMask() != Mask)
17350 if (
Op.getValueType().isVector())
17362 unsigned IsTop =
N->getConstantOperandVal(2);
17369 if (Op0->
isUndef() && !IsTop)
17383 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17385 APInt Op0DemandedElts =
17386 IsTop ? Op1DemandedElts
17401 unsigned IsTop =
N->getConstantOperandVal(2);
17403 unsigned NumElts =
N->getValueType(0).getVectorNumElements();
17404 APInt Op0DemandedElts =
17416 EVT VT =
N->getValueType(0);
17420 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(
LHS);
17421 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(
RHS);
17423 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
17424 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
17428 LHS.getOperand(0),
RHS.getOperand(0));
17442 if (
auto C = dyn_cast<ConstantSDNode>(
N->getOperand(2))) {
17443 int ShiftAmt =
C->getSExtValue();
17444 if (ShiftAmt == 0) {
17450 if (ShiftAmt >= -32 && ShiftAmt < 0) {
17451 unsigned NewOpcode =
17467 unsigned IntNo =
N->getConstantOperandVal(0);
17478 case Intrinsic::arm_neon_vshifts:
17479 case Intrinsic::arm_neon_vshiftu:
17480 case Intrinsic::arm_neon_vrshifts:
17481 case Intrinsic::arm_neon_vrshiftu:
17482 case Intrinsic::arm_neon_vrshiftn:
17483 case Intrinsic::arm_neon_vqshifts:
17484 case Intrinsic::arm_neon_vqshiftu:
17485 case Intrinsic::arm_neon_vqshiftsu:
17486 case Intrinsic::arm_neon_vqshiftns:
17487 case Intrinsic::arm_neon_vqshiftnu:
17488 case Intrinsic::arm_neon_vqshiftnsu:
17489 case Intrinsic::arm_neon_vqrshiftns:
17490 case Intrinsic::arm_neon_vqrshiftnu:
17491 case Intrinsic::arm_neon_vqrshiftnsu: {
17492 EVT VT =
N->getOperand(1).getValueType();
17494 unsigned VShiftOpc = 0;
17497 case Intrinsic::arm_neon_vshifts:
17498 case Intrinsic::arm_neon_vshiftu:
17503 if (
isVShiftRImm(
N->getOperand(2), VT,
false,
true, Cnt)) {
17510 case Intrinsic::arm_neon_vrshifts:
17511 case Intrinsic::arm_neon_vrshiftu:
17516 case Intrinsic::arm_neon_vqshifts:
17517 case Intrinsic::arm_neon_vqshiftu:
17522 case Intrinsic::arm_neon_vqshiftsu:
17527 case Intrinsic::arm_neon_vrshiftn:
17528 case Intrinsic::arm_neon_vqshiftns:
17529 case Intrinsic::arm_neon_vqshiftnu:
17530 case Intrinsic::arm_neon_vqshiftnsu:
17531 case Intrinsic::arm_neon_vqrshiftns:
17532 case Intrinsic::arm_neon_vqrshiftnu:
17533 case Intrinsic::arm_neon_vqrshiftnsu:
17545 case Intrinsic::arm_neon_vshifts:
17546 case Intrinsic::arm_neon_vshiftu:
17549 case Intrinsic::arm_neon_vrshifts:
17552 case Intrinsic::arm_neon_vrshiftu:
17555 case Intrinsic::arm_neon_vrshiftn:
17558 case Intrinsic::arm_neon_vqshifts:
17561 case Intrinsic::arm_neon_vqshiftu:
17564 case Intrinsic::arm_neon_vqshiftsu:
17567 case Intrinsic::arm_neon_vqshiftns:
17570 case Intrinsic::arm_neon_vqshiftnu:
17573 case Intrinsic::arm_neon_vqshiftnsu:
17576 case Intrinsic::arm_neon_vqrshiftns:
17579 case Intrinsic::arm_neon_vqrshiftnu:
17582 case Intrinsic::arm_neon_vqrshiftnsu:
17588 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17589 N->getOperand(1), DAG.
getConstant(Cnt, dl, MVT::i32));
17592 case Intrinsic::arm_neon_vshiftins: {
17593 EVT VT =
N->getOperand(1).getValueType();
17595 unsigned VShiftOpc = 0;
17599 else if (
isVShiftRImm(
N->getOperand(3), VT,
false,
true, Cnt))
17606 return DAG.
getNode(VShiftOpc, dl,
N->getValueType(0),
17607 N->getOperand(1),
N->getOperand(2),
17611 case Intrinsic::arm_neon_vqrshifts:
17612 case Intrinsic::arm_neon_vqrshiftu:
17616 case Intrinsic::arm_mve_vqdmlah:
17617 case Intrinsic::arm_mve_vqdmlash:
17618 case Intrinsic::arm_mve_vqrdmlah:
17619 case Intrinsic::arm_mve_vqrdmlash:
17620 case Intrinsic::arm_mve_vmla_n_predicated:
17621 case Intrinsic::arm_mve_vmlas_n_predicated:
17622 case Intrinsic::arm_mve_vqdmlah_predicated:
17623 case Intrinsic::arm_mve_vqdmlash_predicated:
17624 case Intrinsic::arm_mve_vqrdmlah_predicated:
17625 case Intrinsic::arm_mve_vqrdmlash_predicated: {
17630 unsigned BitWidth =
N->getValueType(0).getScalarSizeInBits();
17637 case Intrinsic::arm_mve_minv:
17638 case Intrinsic::arm_mve_maxv:
17639 case Intrinsic::arm_mve_minav:
17640 case Intrinsic::arm_mve_maxav:
17641 case Intrinsic::arm_mve_minv_predicated:
17642 case Intrinsic::arm_mve_maxv_predicated:
17643 case Intrinsic::arm_mve_minav_predicated:
17644 case Intrinsic::arm_mve_maxav_predicated: {
17647 unsigned BitWidth =
N->getOperand(2)->getValueType(0).getScalarSizeInBits();
17654 case Intrinsic::arm_mve_addv: {
17657 bool Unsigned =
N->getConstantOperandVal(2);
17662 case Intrinsic::arm_mve_addlv:
17663 case Intrinsic::arm_mve_addlv_predicated: {
17666 bool Unsigned =
N->getConstantOperandVal(2);
17667 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
17672 for (
unsigned i = 1, e =
N->getNumOperands(); i < e; i++)
17695 EVT VT =
N->getValueType(0);
17697 if (ST->isThumb1Only() &&
N->getOpcode() ==
ISD::SHL && VT == MVT::i32 &&
17698 N->getOperand(0)->getOpcode() ==
ISD::AND &&
17699 N->getOperand(0)->hasOneUse()) {
17707 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17716 if (AndMask == 255 || AndMask == 65535)
17720 if (MaskedBits > ShiftAmt) {
17735 if (ST->hasMVEIntegerOps())
17740 switch (
N->getOpcode()) {
17753 if (
isVShiftRImm(
N->getOperand(1), VT,
false,
false, Cnt)) {
17754 unsigned VShiftOpc =
17757 return DAG.
getNode(VShiftOpc, dl, VT,
N->getOperand(0),
17773 if (!LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed() ||
17776 EVT FromVT = LD->getValueType(0);
17777 EVT ToVT =
N->getValueType(0);
17784 unsigned NumElements = 0;
17785 if (ToEltVT == MVT::i32 && FromEltVT == MVT::i8)
17787 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
17789 if (NumElements == 0 ||
17799 SDValue BasePtr = LD->getBasePtr();
17800 Align Alignment = LD->getOriginalAlign();
17821 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
17822 Alignment, MMOFlags, AAInfo);
17828 if (FromEltVT == MVT::f16) {
17831 for (
unsigned i = 0; i < Loads.
size(); i++) {
17857 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
17861 EVT VT =
N->getValueType(0);
17865 if (VT == MVT::i32 &&
17866 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
17868 isa<ConstantSDNode>(Lane)) {
17871 switch (
N->getOpcode()) {
17885 if (ST->hasMVEIntegerOps())
17894 if (ST->hasMVEFloatOps())
17905 if ((Subtarget->isThumb() || !Subtarget->hasV6Ops()) &&
17909 EVT VT =
Op.getValueType();
17912 if (VT != MVT::i32 ||
17914 !isa<ConstantSDNode>(
Op.getOperand(1)) ||
17925 APInt MaxC = Max.getConstantOperandAPInt(1);
17928 !(MinC + 1).isPowerOf2())
17946 EVT VT =
N->getValueType(0);
17949 if (VT == MVT::i32)
17952 if (!ST->hasMVEIntegerOps())
17958 if (VT != MVT::v4i32 && VT != MVT::v8i16)
17961 auto IsSignedSaturate = [&](
SDNode *Min,
SDNode *Max) {
17969 if (VT == MVT::v4i32)
17970 SaturateC =
APInt(32, (1 << 15) - 1,
true);
17972 SaturateC =
APInt(16, (1 << 7) - 1,
true);
17979 MaxC != ~SaturateC)
17984 if (IsSignedSaturate(
N, N0.
getNode())) {
17987 if (VT == MVT::v4i32) {
17988 HalfVT = MVT::v8i16;
17989 ExtVT = MVT::v4i16;
17991 HalfVT = MVT::v16i8;
18006 auto IsUnsignedSaturate = [&](
SDNode *Min) {
18012 if (VT == MVT::v4i32)
18013 SaturateC =
APInt(32, (1 << 16) - 1,
true);
18015 SaturateC =
APInt(16, (1 << 8) - 1,
true);
18024 if (IsUnsignedSaturate(
N)) {
18028 if (VT == MVT::v4i32) {
18029 HalfVT = MVT::v8i16;
18030 ExtConst = 0x0000FFFF;
18032 HalfVT = MVT::v16i8;
18054 const APInt *CV = &
C->getAPIntValue();
18072 SDValue Op0 = CMOV->getOperand(0);
18073 SDValue Op1 = CMOV->getOperand(1);
18074 auto CC = CMOV->getConstantOperandAPInt(2).getLimitedValue();
18075 SDValue CmpZ = CMOV->getOperand(4);
18111 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
18118 if ((OrCI & Known.
Zero) != OrCI)
18124 EVT VT =
X.getValueType();
18125 unsigned BitInX = AndC->
logBase2();
18133 for (
unsigned BitInY = 0, NumActiveBits = OrCI.
getActiveBits();
18134 BitInY < NumActiveBits; ++BitInY) {
18135 if (OrCI[BitInY] == 0)
18138 Mask.setBit(BitInY);
18155 switch (
N->getOpcode()) {
18159 if (!isa<ConstantSDNode>(
N.getOperand(1)))
18161 if (!cast<ConstantSDNode>(
N.getOperand(1))->isOne())
18167 auto *Const = dyn_cast<ConstantSDNode>(
N.getOperand(1));
18170 if (Const->isZero())
18172 else if (Const->isOne())
18176 CC = cast<CondCodeSDNode>(
N.getOperand(2))->get();
18180 unsigned IntOp =
N.getConstantOperandVal(1);
18181 if (IntOp != Intrinsic::test_start_loop_iterations &&
18182 IntOp != Intrinsic::loop_decrement_reg)
18208 bool Negate =
false;
18214 Cond =
N->getOperand(1);
18215 Dest =
N->getOperand(2);
18218 CC = cast<CondCodeSDNode>(
N->getOperand(1))->get();
18219 Cond =
N->getOperand(2);
18220 Dest =
N->getOperand(4);
18221 if (
auto *Const = dyn_cast<ConstantSDNode>(
N->getOperand(3))) {
18222 if (!Const->isOne() && !Const->isZero())
18224 Imm = Const->getZExtValue();
18252 assert((IsTrueIfZero(
CC, Imm) || IsFalseIfZero(
CC, Imm)) &&
18253 "unsupported condition");
18258 unsigned IntOp =
Int->getConstantOperandVal(1);
18260 &&
"expected single br user");
18261 SDNode *Br = *
N->use_begin();
18271 if (IntOp == Intrinsic::test_start_loop_iterations) {
18275 if (IsTrueIfZero(
CC, Imm)) {
18276 SDValue Ops[] = {Chain, Setup, Dest};
18281 UpdateUncondBr(Br, Dest, DAG);
18283 SDValue Ops[] = {Chain, Setup, OtherTarget};
18296 DAG.
getVTList(MVT::i32, MVT::Other), Args);
18304 if (
Target == OtherTarget)
18305 UpdateUncondBr(Br, Dest, DAG);
18324 EVT VT =
N->getValueType(0);
18343 LHS->getOperand(0)->getOperand(3),
LHS->getOperand(0)->getOperand(4));
18357 EVT VT =
N->getValueType(0);
18361 SDValue FalseVal =
N->getOperand(0);
18362 SDValue TrueVal =
N->getOperand(1);
18367 if (!Subtarget->
isThumb1Only() && Subtarget->hasV6T2Ops()) {
18393 N->getOperand(3), Cmp);
18398 N->getOperand(3), NewCmp);
18407 LHS->getOperand(2),
LHS->getOperand(3),
18408 LHS->getOperand(4));
18418 if (
N->getConstantOperandVal(2) ==
ARMCC::EQ ||
18422 if (
N->getConstantOperandVal(2) ==
ARMCC::NE)
18424 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), MVT::i32,
N->getOperand(0),
18427 N->getOperand(3),
C);
18434 if (!Subtarget->
isThumb1Only() && Subtarget->hasV5TOps()) {
18470 N->getOperand(3), CPSRGlue.
getValue(1));
18485 N->getOperand(3), CPSRGlue.
getValue(1));
18503 const APInt *TrueConst;
18506 FalseVal.getOperand(0) ==
LHS && FalseVal.getOperand(1) ==
RHS) ||
18510 unsigned ShiftAmount = TrueConst->
logBase2();
18525 if (Known.
Zero == 0xfffffffe)
18528 else if (Known.
Zero == 0xffffff00)
18531 else if (Known.
Zero == 0xffff0000)
18544 EVT DstVT =
N->getValueType(0);
18547 if (ST->hasMVEIntegerOps() && Src.getOpcode() ==
ARMISD::VDUP) {
18548 EVT SrcVT = Src.getValueType();
18556 Src = Src.getOperand(0);
18560 EVT SrcVT = Src.getValueType();
18580 EVT VT =
N->getValueType(0);
18588 if (
N->getNumOperands() == 2 &&
18592 N->getOperand(0).getOperand(1),
18593 N->getOperand(1).getOperand(0),
18594 N->getOperand(1).getOperand(1));
18597 if (
N->getNumOperands() == 2 &&
18600 auto *S0 = cast<ShuffleVectorSDNode>(
N->getOperand(0).getNode());
18601 auto *
S1 = cast<ShuffleVectorSDNode>(
N->getOperand(1).getNode());
18603 if (S0->getOperand(0) ==
S1->getOperand(0) &&
18604 S0->getOperand(1) ==
S1->getOperand(1)) {
18607 Mask.append(
S1->getMask().begin(),
S1->getMask().end());
18627 return Op.getOpcode() == ISD::BUILD_VECTOR ||
18628 Op.getOpcode() == ISD::VECTOR_SHUFFLE ||
18629 (Op.getOpcode() == ISD::BITCAST &&
18630 Op.getOperand(0).getOpcode() == ISD::BUILD_VECTOR);
18633 for (
unsigned Op = 0;
Op <
N->getNumOperands();
Op++) {
18635 for (
unsigned i = 0; i < O.getValueType().getVectorNumElements(); i++) {
18652 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18653 int NumIns =
N->getNumOperands();
18654 assert((NumIns == 2 || NumIns == 4) &&
18655 "Expected 2 or 4 inputs to an MVETrunc");
18657 if (
N->getNumOperands() == 4)
18661 for (
int I = 0;
I < NumIns;
I++) {
18663 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18683 if (!LD || !LD->isSimple() || !N0.
hasOneUse() || LD->isIndexed())
18686 EVT FromVT = LD->getMemoryVT();
18687 EVT ToVT =
N->getValueType(0);
18694 unsigned NumElements = 0;
18695 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
18697 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
18699 assert(NumElements != 0);
18705 LD->getExtensionType() != NewExtType)
18712 SDValue BasePtr = LD->getBasePtr();
18713 Align Alignment = LD->getOriginalAlign();
18732 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
18733 Alignment, MMOFlags, AAInfo);
18749 EVT VT =
N->getValueType(0);
18751 assert(
N->getNumValues() == 2 &&
"Expected MVEEXT with 2 elements");
18752 assert((VT == MVT::v4i32 || VT == MVT::v8i16) &&
"Unexpected MVEEXT type");
18754 EVT ExtVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18756 auto Extend = [&](
SDValue V) {
18766 SDValue Ext = Extend(
N->getOperand(0));
18771 if (
auto *SVN = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(0))) {
18774 assert(Mask.size() == SVN->getValueType(0).getVectorNumElements());
18779 auto CheckInregMask = [&](
int Start,
int Offset) {
18787 if (CheckInregMask(0, 0))
18789 else if (CheckInregMask(0, 1))
18790 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18791 else if (CheckInregMask(0, Mask.size()))
18793 else if (CheckInregMask(0, Mask.size() + 1))
18794 V0 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18799 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op1));
18803 V1 = Extend(DAG.
getNode(Rev,
DL, SVN->getValueType(0), Op0));
18810 if (
N->getOperand(0)->getOpcode() ==
ISD::LOAD)
18820 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
18821 int NumOuts =
N->getNumValues();
18822 assert((NumOuts == 2 || NumOuts == 4) &&
18823 "Expected 2 or 4 outputs to an MVEEXT");
18824 EVT LoadVT =
N->getOperand(0).getValueType().getHalfNumVectorElementsVT(
18826 if (
N->getNumOperands() == 4)
18832 StackPtr, MPI,
Align(4));
18835 for (
int I = 0;
I < NumOuts;
I++) {
18837 ISD::ADD,
DL, StackPtr.getValueType(), StackPtr,
18838 DAG.
getConstant(
I * 16 / NumOuts,
DL, StackPtr.getValueType()));
18843 VT, Chain,
Ptr, MPI, LoadVT,
Align(4));
18852 switch (
N->getOpcode()) {
18970 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
18977 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
18988 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
18996 unsigned LowWidth =
N->getOperand(0).getValueType().getSizeInBits();
18998 unsigned HighWidth =
N->getOperand(1).getValueType().getSizeInBits();
19006 unsigned HighWidth =
N->getOperand(0).getValueType().getSizeInBits();
19008 unsigned LowWidth =
N->getOperand(1).getValueType().getSizeInBits();
19016 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19027 unsigned BitWidth =
N->getValueType(0).getSizeInBits();
19036 switch (
N->getConstantOperandVal(1)) {
19037 case Intrinsic::arm_neon_vld1:
19038 case Intrinsic::arm_neon_vld1x2:
19039 case Intrinsic::arm_neon_vld1x3:
19040 case Intrinsic::arm_neon_vld1x4:
19041 case Intrinsic::arm_neon_vld2:
19042 case Intrinsic::arm_neon_vld3:
19043 case Intrinsic::arm_neon_vld4:
19044 case Intrinsic::arm_neon_vld2lane:
19045 case Intrinsic::arm_neon_vld3lane:
19046 case Intrinsic::arm_neon_vld4lane:
19047 case Intrinsic::arm_neon_vld2dup:
19048 case Intrinsic::arm_neon_vld3dup:
19049 case Intrinsic::arm_neon_vld4dup:
19050 case Intrinsic::arm_neon_vst1:
19051 case Intrinsic::arm_neon_vst1x2:
19052 case Intrinsic::arm_neon_vst1x3:
19053 case Intrinsic::arm_neon_vst1x4:
19054 case Intrinsic::arm_neon_vst2:
19055 case Intrinsic::arm_neon_vst3:
19056 case Intrinsic::arm_neon_vst4:
19057 case Intrinsic::arm_neon_vst2lane:
19058 case Intrinsic::arm_neon_vst3lane:
19059 case Intrinsic::arm_neon_vst4lane:
19061 case Intrinsic::arm_mve_vld2q:
19062 case Intrinsic::arm_mve_vld4q:
19063 case Intrinsic::arm_mve_vst2q:
19064 case Intrinsic::arm_mve_vst4q:
19081 unsigned *
Fast)
const {
19090 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
19092 if (AllowsUnaligned) {
19094 *
Fast = Subtarget->hasV7Ops();
19099 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
19103 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->
isLittle())) {
19110 if (!Subtarget->hasMVEIntegerOps())
19114 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 ||
19115 Ty == MVT::v2i1)) {
19123 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
19139 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
19140 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
19141 Ty == MVT::v2f64) {
19154 if ((
Op.isMemcpy() ||
Op.isZeroMemset()) && Subtarget->hasNEON() &&
19155 !FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat)) {
19157 if (
Op.size() >= 16 &&
19163 }
else if (
Op.size() >= 8 &&
19184 return (SrcBits == 64 && DestBits == 32);
19193 return (SrcBits == 64 && DestBits == 32);
19229 return Subtarget->hasFullFP16();
19239 return Ext->getType()->getScalarSizeInBits() ==
19240 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
19245 !areExtDoubled(cast<Instruction>(Ext1)) ||
19246 !areExtDoubled(cast<Instruction>(Ext2)))
19257 if (!
I->getType()->isVectorTy())
19260 if (Subtarget->hasNEON()) {
19261 switch (
I->getOpcode()) {
19262 case Instruction::Sub:
19263 case Instruction::Add: {
19275 if (!Subtarget->hasMVEIntegerOps())
19279 if (!
I->hasOneUse())
19281 auto *Sub = cast<Instruction>(*
I->users().begin());
19282 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) ==
I;
19292 switch (
I->getOpcode()) {
19293 case Instruction::Add:
19294 case Instruction::Mul:
19295 case Instruction::FAdd:
19296 case Instruction::ICmp:
19297 case Instruction::FCmp:
19299 case Instruction::FMul:
19300 return !IsFMSMul(
I);
19301 case Instruction::Sub:
19302 case Instruction::FSub:
19303 case Instruction::Shl:
19304 case Instruction::LShr:
19305 case Instruction::AShr:
19306 return Operand == 1;
19307 case Instruction::Call:
19308 if (
auto *II = dyn_cast<IntrinsicInst>(
I)) {
19309 switch (II->getIntrinsicID()) {
19310 case Intrinsic::fma:
19312 case Intrinsic::sadd_sat:
19313 case Intrinsic::uadd_sat:
19314 case Intrinsic::arm_mve_add_predicated:
19315 case Intrinsic::arm_mve_mul_predicated:
19316 case Intrinsic::arm_mve_qadd_predicated:
19317 case Intrinsic::arm_mve_vhadd:
19318 case Intrinsic::arm_mve_hadd_predicated:
19319 case Intrinsic::arm_mve_vqdmull:
19320 case Intrinsic::arm_mve_vqdmull_predicated:
19321 case Intrinsic::arm_mve_vqdmulh:
19322 case Intrinsic::arm_mve_qdmulh_predicated:
19323 case Intrinsic::arm_mve_vqrdmulh:
19324 case Intrinsic::arm_mve_qrdmulh_predicated:
19325 case Intrinsic::arm_mve_fma_predicated:
19327 case Intrinsic::ssub_sat:
19328 case Intrinsic::usub_sat:
19329 case Intrinsic::arm_mve_sub_predicated:
19330 case Intrinsic::arm_mve_qsub_predicated:
19331 case Intrinsic::arm_mve_hsub_predicated:
19332 case Intrinsic::arm_mve_vhsub:
19333 return Operand == 1;
19344 for (
auto OpIdx :
enumerate(
I->operands())) {
19345 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
19347 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
19351 if (Shuffle->
getOpcode() == Instruction::BitCast)
19352 Shuffle = dyn_cast<Instruction>(Shuffle->
getOperand(0));
19359 if (!IsSinker(
I, OpIdx.index()))
19364 for (
Use &U :
Op->uses()) {
19366 if (!IsSinker(
Insn, U.getOperandNo()))
19379 if (!Subtarget->hasMVEIntegerOps())
19397 if (
auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal.
getOperand(0))) {
19398 if (Ld->isExpandingLoad())
19402 if (Subtarget->hasMVEIntegerOps())
19447bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(
const MachineFunction &MF,
19455 return Subtarget->hasMVEFloatOps();
19473 unsigned Scale = 1;
19490 if ((V & (Scale - 1)) != 0)
19492 return isUInt<5>(V / Scale);
19499 if (VT.
isVector() && Subtarget->hasNEON())
19502 !Subtarget->hasMVEFloatOps())
19505 bool IsNeg =
false;
19511 unsigned NumBytes = std::max((
unsigned)VT.
getSizeInBits() / 8, 1U);
19514 if (VT.
isVector() && Subtarget->hasMVEIntegerOps()) {
19518 return isShiftedUInt<7,2>(V);
19521 return isShiftedUInt<7,1>(V);
19523 return isUInt<7>(V);
19530 if (VT.
isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
19531 return isShiftedUInt<8, 1>(V);
19534 return isShiftedUInt<8, 2>(V);
19536 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
19539 return isUInt<8>(V);
19540 return isUInt<12>(V);
19566 default:
return false;
19571 return isUInt<12>(V);
19574 return isUInt<8>(V);
19579 return isShiftedUInt<8, 2>(V);
19585 int Scale = AM.
Scale;
19590 default:
return false;
19598 Scale = Scale & ~1;
19599 return Scale == 2 || Scale == 4 || Scale == 8;
19616 if (Scale & 1)
return false;
19623 const int Scale = AM.
Scale;
19633 return (Scale == 1) || (!AM.
HasBaseReg && Scale == 2);
19649 switch (AM.
Scale) {
19666 int Scale = AM.
Scale;
19668 default:
return false;
19672 if (Scale < 0) Scale = -Scale;
19680 if (Scale == 1 || (AM.
HasBaseReg && Scale == -1))
19693 if (Scale & 1)
return false;
19706 if (!Subtarget->isThumb())
19713 return Imm >= 0 && Imm <= 255;
19722 int64_t AbsImm = std::abs(Imm);
19723 if (!Subtarget->isThumb())
19728 return AbsImm >= 0 && AbsImm <= 255;
19763 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
19767 int RHSC = (int)
RHS->getZExtValue();
19768 if (RHSC < 0 && RHSC > -256) {
19778 }
else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
19781 int RHSC = (int)
RHS->getZExtValue();
19782 if (RHSC < 0 && RHSC > -0x1000) {
19824 int RHSC = (int)
RHS->getZExtValue();
19825 if (RHSC < 0 && RHSC > -0x100) {
19830 }
else if (RHSC > 0 && RHSC < 0x100) {
19841 bool isSEXTLoad,
bool IsMasked,
bool isLE,
19846 if (!isa<ConstantSDNode>(
Ptr->getOperand(1)))
19852 bool CanChangeType = isLE && !IsMasked;
19855 int RHSC = (int)
RHS->getZExtValue();
19857 auto IsInRange = [&](
int RHSC,
int Limit,
int Scale) {
19858 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
19863 }
else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
19874 if (VT == MVT::v4i16) {
19875 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
19877 }
else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
19878 if (IsInRange(RHSC, 0x80, 1))
19880 }
else if (Alignment >= 4 &&
19881 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
19882 IsInRange(RHSC, 0x80, 4))
19884 else if (Alignment >= 2 &&
19885 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
19886 IsInRange(RHSC, 0x80, 2))
19888 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
19907 bool isSEXTLoad =
false;
19908 bool IsMasked =
false;
19910 Ptr = LD->getBasePtr();
19911 VT = LD->getMemoryVT();
19912 Alignment = LD->getAlign();
19914 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19915 Ptr = ST->getBasePtr();
19916 VT = ST->getMemoryVT();
19917 Alignment = ST->getAlign();
19919 Ptr = LD->getBasePtr();
19920 VT = LD->getMemoryVT();
19921 Alignment = LD->getAlign();
19925 Ptr = ST->getBasePtr();
19926 VT = ST->getMemoryVT();
19927 Alignment = ST->getAlign();
19933 bool isLegal =
false;
19935 isLegal = Subtarget->hasMVEIntegerOps() &&
19937 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
19965 bool isSEXTLoad =
false, isNonExt;
19966 bool IsMasked =
false;
19968 VT = LD->getMemoryVT();
19969 Ptr = LD->getBasePtr();
19970 Alignment = LD->getAlign();
19973 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
19974 VT = ST->getMemoryVT();
19975 Ptr = ST->getBasePtr();
19976 Alignment = ST->getAlign();
19977 isNonExt = !ST->isTruncatingStore();
19979 VT = LD->getMemoryVT();
19980 Ptr = LD->getBasePtr();
19981 Alignment = LD->getAlign();
19986 VT = ST->getMemoryVT();
19987 Ptr = ST->getBasePtr();
19988 Alignment = ST->getAlign();
19989 isNonExt = !ST->isTruncatingStore();
19997 assert(
Op->getValueType(0) == MVT::i32 &&
"Non-i32 post-inc op?!");
19998 if (
Op->getOpcode() !=
ISD::ADD || !isNonExt)
20000 auto *
RHS = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
20001 if (!
RHS ||
RHS->getZExtValue() != 4)
20003 if (Alignment <
Align(4))
20007 Base =
Op->getOperand(0);
20013 bool isLegal =
false;
20015 isLegal = Subtarget->hasMVEIntegerOps() &&
20048 const APInt &DemandedElts,
20050 unsigned Depth)
const {
20053 switch (
Op.getOpcode()) {
20060 if (
Op.getResNo() == 0) {
20086 case Intrinsic::arm_ldaex:
20087 case Intrinsic::arm_ldrex: {
20088 EVT VT = cast<MemIntrinsicSDNode>(
Op)->getMemoryVT();
20102 const APInt &Mask =
Op.getConstantOperandAPInt(2);
20103 Known.
Zero &= Mask;
20109 const SDValue &SrcSV =
Op.getOperand(0);
20115 "VGETLANE index out of bounds");
20120 EVT VT =
Op.getValueType();
20127 Known = Known.
sext(DstSz);
20129 Known = Known.
zext(DstSz);
20137 Known = KnownOp.
zext(32);
20152 true,
false,
false, KnownOp1,
20178 EVT VT =
Op.getValueType();
20184 assert(VT == MVT::i32 &&
"Unexpected integer type");
20191 unsigned Mask =
C->getZExtValue();
20194 unsigned ShrunkMask = Mask & Demanded;
20195 unsigned ExpandedMask = Mask | ~Demanded;
20199 if (ShrunkMask == 0)
20205 if (ExpandedMask == ~0U)
20208 auto IsLegalMask = [ShrunkMask, ExpandedMask](
unsigned Mask) ->
bool {
20209 return (ShrunkMask & Mask) == ShrunkMask && (~ExpandedMask & Mask) == 0;
20211 auto UseMask = [Mask,
Op, VT, &TLO](
unsigned NewMask) ->
bool {
20212 if (NewMask == Mask)
20221 if (IsLegalMask(0xFF))
20222 return UseMask(0xFF);
20225 if (IsLegalMask(0xFFFF))
20226 return UseMask(0xFFFF);
20230 if (ShrunkMask < 256)
20231 return UseMask(ShrunkMask);
20235 if ((
int)ExpandedMask <= -2 && (
int)ExpandedMask >= -256)
20236 return UseMask(ExpandedMask);
20251 unsigned Depth)
const {
20252 unsigned Opc =
Op.getOpcode();
20260 if (
Op.getResNo() == 0 && !
Op->hasAnyUseOfValue(1) &&
20261 isa<ConstantSDNode>(
Op->getOperand(2))) {
20262 unsigned ShAmt =
Op->getConstantOperandVal(2);
20274 unsigned ModImm =
Op.getConstantOperandVal(1);
20275 unsigned EltBits = 0;
20277 if ((OriginalDemandedBits & Mask) == 0)
20283 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO,
Depth);
20292 if (!Subtarget->hasV6Ops())
20298 SplitString(AsmStr, AsmPieces,
";\n");
20300 switch (AsmPieces.
size()) {
20301 default:
return false;
20303 AsmStr = AsmPieces[0];
20305 SplitString(AsmStr, AsmPieces,
" \t,");
20308 if (AsmPieces.
size() == 3 &&
20309 AsmPieces[0] ==
"rev" && AsmPieces[1] ==
"$0" && AsmPieces[2] ==
"$1" &&
20310 IA->getConstraintString().compare(0, 4,
"=l,l") == 0) {
20333 if (ConstraintVT.
isVector() && Subtarget->hasNEON() &&
20345 unsigned S = Constraint.
size();
20347 switch (Constraint[0]) {
20359 }
else if (S == 2) {
20360 switch (Constraint[0]) {
20377 Value *CallOperandVal =
info.CallOperandVal;
20380 if (!CallOperandVal)
20384 switch (*constraint) {
20390 if (Subtarget->isThumb())
20404using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
20408 switch (Constraint.
size()) {
20411 switch (Constraint[0]) {
20413 if (Subtarget->isThumb())
20414 return RCPair(0U, &ARM::tGPRRegClass);
20415 return RCPair(0U, &ARM::GPRRegClass);
20417 if (Subtarget->isThumb())
20418 return RCPair(0U, &ARM::hGPRRegClass);
20422 return RCPair(0U, &ARM::tGPRRegClass);
20423 return RCPair(0U, &ARM::GPRRegClass);
20425 if (VT == MVT::Other)
20427 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20428 return RCPair(0U, &ARM::SPRRegClass);
20430 return RCPair(0U, &ARM::DPRRegClass);
20432 return RCPair(0U, &ARM::QPRRegClass);
20435 if (VT == MVT::Other)
20437 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16)
20438 return RCPair(0U, &ARM::SPR_8RegClass);
20440 return RCPair(0U, &ARM::DPR_8RegClass);
20442 return RCPair(0U, &ARM::QPR_8RegClass);
20445 if (VT == MVT::Other)
20447 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16)
20448 return RCPair(0U, &ARM::SPRRegClass);
20450 return RCPair(0U, &ARM::DPR_VFP2RegClass);
20452 return RCPair(0U, &ARM::QPR_VFP2RegClass);
20458 if (Constraint[0] ==
'T') {
20459 switch (Constraint[1]) {
20463 return RCPair(0U, &ARM::tGPREvenRegClass);
20465 return RCPair(0U, &ARM::tGPROddRegClass);
20474 if (
StringRef(
"{cc}").equals_insensitive(Constraint))
20475 return std::make_pair(
unsigned(ARM::CPSR), &ARM::CCRRegClass);
20484 std::vector<SDValue> &Ops,
20489 if (Constraint.
size() != 1)
20492 char ConstraintLetter = Constraint[0];
20493 switch (ConstraintLetter) {
20496 case 'I':
case 'J':
case 'K':
case 'L':
20497 case 'M':
case 'N':
case 'O':
20502 int64_t CVal64 =
C->getSExtValue();
20503 int CVal = (int) CVal64;
20506 if (CVal != CVal64)
20509 switch (ConstraintLetter) {
20513 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
20514 if (CVal >= 0 && CVal <= 65535)
20521 if (CVal >= 0 && CVal <= 255)
20523 }
else if (Subtarget->
isThumb2()) {
20542 if (CVal >= -255 && CVal <= -1)
20548 if (CVal >= -4095 && CVal <= 4095)
20561 }
else if (Subtarget->
isThumb2()) {
20584 if (CVal >= -7 && CVal < 7)
20586 }
else if (Subtarget->
isThumb2()) {
20609 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
20615 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
20623 if (CVal >= 0 && CVal <= 31)
20632 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
20641 if (Result.getNode()) {
20642 Ops.push_back(Result);
20652 "Unhandled Opcode in getDivRemLibcall");
20658 case MVT::i8: LC =
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
20659 case MVT::i16: LC =
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
20660 case MVT::i32: LC =
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
20661 case MVT::i64: LC =
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
20670 "Unhandled Opcode in getDivRemArgList");
20675 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
20676 EVT ArgVT =
N->getOperand(i).getValueType();
20678 Entry.Node =
N->getOperand(i);
20682 Args.push_back(Entry);
20693 "Register-based DivRem lowering only");
20694 unsigned Opcode =
Op->getOpcode();
20696 "Invalid opcode for Div/Rem lowering");
20698 EVT VT =
Op->getValueType(0);
20701 if (VT == MVT::i64 && isa<ConstantSDNode>(
Op.getOperand(1))) {
20720 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
20721 : Subtarget->hasDivideInARMMode();
20722 if (hasDivide &&
Op->getValueType(0).isSimple() &&
20723 Op->getSimpleValueType(0) == MVT::i32) {
20725 const SDValue Dividend =
Op->getOperand(0);
20726 const SDValue Divisor =
Op->getOperand(1);
20727 SDValue Div = DAG.
getNode(DivOpcode, dl, VT, Dividend, Divisor);
20731 SDValue Values[2] = {Div, Rem};
20763 EVT VT =
N->getValueType(0);
20765 if (VT == MVT::i64 && isa<ConstantSDNode>(
N->getOperand(1))) {
20769 Result[0], Result[1]);
20773 std::vector<Type*> RetTyParams;
20774 Type *RetTyElement;
20784 RetTyParams.push_back(RetTyElement);
20785 RetTyParams.push_back(RetTyElement);
20802 CallLoweringInfo CLI(DAG);
20806 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
20809 SDNode *ResNode = CallResult.first.getNode();
20824 "no-stack-arg-probe")) {
20826 cast<ConstantSDNode>(
Op.getOperand(2))->getMaybeAlignValue();
20835 SDValue Ops[2] = { SP, Chain };
20852 SDValue Ops[2] = { NewSP, Chain };
20857 bool IsStrict =
Op->isStrictFPOpcode();
20858 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20859 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20861 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&
20862 "Unexpected type for custom-lowering FP_EXTEND");
20865 "With both FP DP and 16, any FP conversion is legal!");
20867 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&
20868 "With FP16, 16 to 32 conversion is legal!");
20871 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
20876 Loc,
Op.getValueType(), SrcVal);
20889 MakeLibCallOptions CallOptions;
20891 for (
unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
20892 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
20893 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
20894 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
20898 {DstVT, MVT::Other}, {Chain, SrcVal});
20905 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20906 "Unexpected type for custom-lowering FP_EXTEND");
20907 std::tie(SrcVal, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20912 return IsStrict ? DAG.
getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
20916 bool IsStrict =
Op->isStrictFPOpcode();
20918 SDValue SrcVal =
Op.getOperand(IsStrict ? 1 : 0);
20920 EVT DstVT =
Op.getValueType();
20921 const unsigned DstSz =
Op.getValueType().getSizeInBits();
20924 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&
20925 "Unexpected type for custom-lowering FP_ROUND");
20928 "With both FP DP and 16, any FP conversion is legal!");
20933 if (SrcSz == 32 && Subtarget->hasFP16())
20938 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
20939 "Unexpected type for custom-lowering FP_ROUND");
20940 MakeLibCallOptions CallOptions;
20943 std::tie(Result, Chain) =
makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
20955 if (v == 0xffffffff)
20967 bool ForCodeSize)
const {
20970 if (VT == MVT::f16 && Subtarget->hasFullFP16())
20972 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
20975 if (VT == MVT::f32)
20977 if (VT == MVT::f64 && Subtarget->hasFP64())
20988 unsigned Intrinsic)
const {
20989 switch (Intrinsic) {
20990 case Intrinsic::arm_neon_vld1:
20991 case Intrinsic::arm_neon_vld2:
20992 case Intrinsic::arm_neon_vld3:
20993 case Intrinsic::arm_neon_vld4:
20994 case Intrinsic::arm_neon_vld2lane:
20995 case Intrinsic::arm_neon_vld3lane:
20996 case Intrinsic::arm_neon_vld4lane:
20997 case Intrinsic::arm_neon_vld2dup:
20998 case Intrinsic::arm_neon_vld3dup:
20999 case Intrinsic::arm_neon_vld4dup: {
21002 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21003 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21005 Info.ptrVal =
I.getArgOperand(0);
21007 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21008 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21013 case Intrinsic::arm_neon_vld1x2:
21014 case Intrinsic::arm_neon_vld1x3:
21015 case Intrinsic::arm_neon_vld1x4: {
21018 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21019 uint64_t NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
21021 Info.ptrVal =
I.getArgOperand(
I.arg_size() - 1);
21023 Info.align.reset();
21028 case Intrinsic::arm_neon_vst1:
21029 case Intrinsic::arm_neon_vst2:
21030 case Intrinsic::arm_neon_vst3:
21031 case Intrinsic::arm_neon_vst4:
21032 case Intrinsic::arm_neon_vst2lane:
21033 case Intrinsic::arm_neon_vst3lane:
21034 case Intrinsic::arm_neon_vst4lane: {
21037 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21038 unsigned NumElts = 0;
21039 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21040 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21043 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21046 Info.ptrVal =
I.getArgOperand(0);
21048 Value *AlignArg =
I.getArgOperand(
I.arg_size() - 1);
21049 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
21054 case Intrinsic::arm_neon_vst1x2:
21055 case Intrinsic::arm_neon_vst1x3:
21056 case Intrinsic::arm_neon_vst1x4: {
21059 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21060 unsigned NumElts = 0;
21061 for (
unsigned ArgI = 1, ArgE =
I.arg_size(); ArgI < ArgE; ++ArgI) {
21062 Type *ArgTy =
I.getArgOperand(ArgI)->getType();
21065 NumElts +=
DL.getTypeSizeInBits(ArgTy) / 64;
21068 Info.ptrVal =
I.getArgOperand(0);
21070 Info.align.reset();
21075 case Intrinsic::arm_mve_vld2q:
21076 case Intrinsic::arm_mve_vld4q: {
21079 Type *VecTy = cast<StructType>(
I.getType())->getElementType(1);
21080 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
21082 Info.ptrVal =
I.getArgOperand(0);
21089 case Intrinsic::arm_mve_vst2q:
21090 case Intrinsic::arm_mve_vst4q: {
21093 Type *VecTy =
I.getArgOperand(1)->getType();
21094 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
21096 Info.ptrVal =
I.getArgOperand(0);
21103 case Intrinsic::arm_mve_vldr_gather_base:
21104 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
21106 Info.ptrVal =
nullptr;
21112 case Intrinsic::arm_mve_vldr_gather_base_wb:
21113 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
21115 Info.ptrVal =
nullptr;
21121 case Intrinsic::arm_mve_vldr_gather_offset:
21122 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
21124 Info.ptrVal =
nullptr;
21126 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
21133 case Intrinsic::arm_mve_vstr_scatter_base:
21134 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
21136 Info.ptrVal =
nullptr;
21142 case Intrinsic::arm_mve_vstr_scatter_base_wb:
21143 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
21145 Info.ptrVal =
nullptr;
21151 case Intrinsic::arm_mve_vstr_scatter_offset:
21152 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
21154 Info.ptrVal =
nullptr;
21156 unsigned MemSize = cast<ConstantInt>(
I.getArgOperand(3))->getZExtValue();
21163 case Intrinsic::arm_ldaex:
21164 case Intrinsic::arm_ldrex: {
21165 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21166 Type *ValTy =
I.getParamElementType(0);
21169 Info.ptrVal =
I.getArgOperand(0);
21171 Info.align =
DL.getABITypeAlign(ValTy);
21175 case Intrinsic::arm_stlex:
21176 case Intrinsic::arm_strex: {
21177 auto &
DL =
I.getCalledFunction()->getParent()->getDataLayout();
21178 Type *ValTy =
I.getParamElementType(1);
21181 Info.ptrVal =
I.getArgOperand(1);
21183 Info.align =
DL.getABITypeAlign(ValTy);
21187 case Intrinsic::arm_stlexd:
21188 case Intrinsic::arm_strexd:
21190 Info.memVT = MVT::i64;
21191 Info.ptrVal =
I.getArgOperand(2);
21197 case Intrinsic::arm_ldaexd:
21198 case Intrinsic::arm_ldrexd:
21200 Info.memVT = MVT::i64;
21201 Info.ptrVal =
I.getArgOperand(0);
21221 if (Bits == 0 || Bits > 32)
21227 unsigned Index)
const {
21239 if (!Subtarget->hasDataBarrier()) {
21243 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
21280 if (Subtarget->preferISHSTBarriers())
21313 bool has64BitAtomicStore;
21315 has64BitAtomicStore =
false;
21316 else if (Subtarget->isThumb())
21317 has64BitAtomicStore = Subtarget->hasV7Ops();
21319 has64BitAtomicStore = Subtarget->hasV6Ops();
21321 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
21335 bool has64BitAtomicLoad;
21337 has64BitAtomicLoad =
false;
21338 else if (Subtarget->isThumb())
21339 has64BitAtomicLoad = Subtarget->hasV7Ops();
21341 has64BitAtomicLoad = Subtarget->hasV6Ops();
21358 hasAtomicRMW = Subtarget->hasV8MBaselineOps();
21359 else if (Subtarget->isThumb())
21360 hasAtomicRMW = Subtarget->hasV7Ops();
21362 hasAtomicRMW = Subtarget->hasV6Ops();
21363 if (
Size <= (Subtarget->
isMClass() ? 32U : 64U) && hasAtomicRMW) {
21386 bool HasAtomicCmpXchg;
21388 HasAtomicCmpXchg = Subtarget->hasV8MBaselineOps();
21389 else if (Subtarget->isThumb())
21390 HasAtomicCmpXchg = Subtarget->hasV7Ops();
21392 HasAtomicCmpXchg = Subtarget->hasV6Ops();
21394 HasAtomicCmpXchg &&
Size <= (Subtarget->
isMClass() ? 32U : 64U))
21401 return InsertFencesForAtomic;
21414 M.getOrInsertGlobal(
"__security_cookie",
21422 F->addParamAttr(0, Attribute::AttrKind::InReg);
21428 return M.getGlobalVariable(
"__security_cookie");
21435 return M.getFunction(
"__security_check_cookie");
21440 unsigned &
Cost)
const {
21442 if (!Subtarget->hasNEON())
21454 if (!isa<ConstantInt>(
Idx))
21469 return Subtarget->hasV6T2Ops();
21473 return Subtarget->hasV6T2Ops();
21478 if (!Subtarget->hasV7Ops())
21484 if (!Mask || Mask->getValue().getBitWidth() > 32u)
21486 auto MaskVal =
unsigned(Mask->getValue().getZExtValue());
21511 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
21523 Lo, Builder.
CreateShl(
Hi, ConstantInt::get(ValueTy, 32)),
"val64");
21526 Type *Tys[] = {
Addr->getType() };
21527 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
21532 0,
Attribute::get(M->getContext(), Attribute::ElementType, ValueTy));
21538 if (!Subtarget->hasV7Ops())
21555 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
21566 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
21567 Type *Tys[] = {
Addr->getType() };
21589 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
21596 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
21599 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
21607 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
21615 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
21618 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
21623 if (Subtarget->hasNEON() && VecSize == 64)
21625 return VecSize % 128 == 0;
21629 if (Subtarget->hasNEON())
21631 if (Subtarget->hasMVEIntegerOps())
21651 "Invalid interleave factor");
21652 assert(!Shuffles.
empty() &&
"Empty shufflevector input");
21654 "Unmatched number of shufflevectors and indices");
21656 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21657 Type *EltTy = VecTy->getElementType();
21680 if (NumLoads > 1) {
21684 VecTy->getNumElements() / NumLoads);
21689 auto createLoadIntrinsic = [&](
Value *BaseAddr) {
21690 if (Subtarget->hasNEON()) {
21692 Type *Tys[] = {VecTy, PtrTy};
21693 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
21694 Intrinsic::arm_neon_vld3,
21695 Intrinsic::arm_neon_vld4};
21703 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21705 assert((Factor == 2 || Factor == 4) &&
21706 "expected interleave factor of 2 or 4 for MVE");
21708 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
21710 Type *Tys[] = {VecTy, PtrTy};
21716 return Builder.
CreateCall(VldnFunc, Ops,
"vldN");
21725 for (
unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
21730 VecTy->getNumElements() * Factor);
21732 CallInst *VldN = createLoadIntrinsic(BaseAddr);
21736 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21738 unsigned Index = Indices[i];
21748 SubVecs[SV].push_back(SubVec);
21757 auto &SubVec = SubVecs[SVI];
21760 SVI->replaceAllUsesWith(WideVec);
21794 unsigned Factor)
const {
21796 "Invalid interleave factor");
21798 auto *VecTy = cast<FixedVectorType>(SVI->
getType());
21799 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
21801 unsigned LaneLen = VecTy->getNumElements() / Factor;
21802 Type *EltTy = VecTy->getElementType();
21805 const DataLayout &
DL = SI->getModule()->getDataLayout();
21806 Align Alignment = SI->getAlign();
21823 Type *IntTy =
DL.getIntPtrType(EltTy);
21835 Value *BaseAddr = SI->getPointerOperand();
21837 if (NumStores > 1) {
21840 LaneLen /= NumStores;
21848 auto createStoreIntrinsic = [&](
Value *BaseAddr,
21850 if (Subtarget->hasNEON()) {
21851 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
21852 Intrinsic::arm_neon_vst3,
21853 Intrinsic::arm_neon_vst4};
21854 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21855 Type *Tys[] = {PtrTy, SubVecTy};
21858 SI->getModule(), StoreInts[Factor - 2], Tys);
21866 assert((Factor == 2 || Factor == 4) &&
21867 "expected interleave factor of 2 or 4 for MVE");
21869 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
21870 Type *PtrTy = Builder.
getPtrTy(SI->getPointerAddressSpace());
21871 Type *Tys[] = {PtrTy, SubVecTy};
21878 for (
unsigned F = 0;
F < Factor;
F++) {
21886 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
21889 if (StoreCount > 0)
21891 BaseAddr, LaneLen * Factor);
21896 for (
unsigned i = 0; i < Factor; i++) {
21897 unsigned IdxI = StoreCount * LaneLen * Factor + i;
21898 if (Mask[IdxI] >= 0) {
21902 unsigned StartMask = 0;
21903 for (
unsigned j = 1; j < LaneLen; j++) {
21904 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
21905 if (Mask[IdxJ * Factor + IdxI] >= 0) {
21906 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
21921 createStoreIntrinsic(BaseAddr, Shuffles);
21936 if (
auto *ST = dyn_cast<StructType>(Ty)) {
21937 for (
unsigned i = 0; i < ST->getNumElements(); ++i) {
21941 Members += SubMembers;
21943 }
else if (
auto *AT = dyn_cast<ArrayType>(Ty)) {
21947 Members += SubMembers * AT->getNumElements();
21958 }
else if (
auto *VT = dyn_cast<VectorType>(Ty)) {
21965 return VT->getPrimitiveSizeInBits().getFixedValue() == 64;
21967 return VT->getPrimitiveSizeInBits().getFixedValue() == 128;
21969 switch (VT->getPrimitiveSizeInBits().getFixedValue()) {
21982 return (Members > 0 && Members <= 4);
21988 const Align ABITypeAlign =
DL.getABITypeAlign(ArgTy);
21990 return ABITypeAlign;
21994 return std::min(ABITypeAlign,
DL.getStackAlignment());
22003 if (getEffectiveCallingConv(CallConv, isVarArg) !=
22013 return IsHA || IsIntArray;
22017 const Constant *PersonalityFn)
const {
22024 const Constant *PersonalityFn)
const {
22036void ARMTargetLowering::insertCopiesSplitCSR(
22040 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
22050 RC = &ARM::GPRRegClass;
22051 else if (ARM::DPRRegClass.
contains(*
I))
22052 RC = &ARM::DPRRegClass;
22062 assert(Entry->getParent()->getFunction().hasFnAttribute(
22063 Attribute::NoUnwind) &&
22064 "Function should be nounwind in insertCopiesSplitCSR!");
22065 Entry->addLiveIn(*
I);
22070 for (
auto *Exit : Exits)
22072 TII->get(TargetOpcode::COPY), *
I)
22083 return Subtarget->hasMVEIntegerOps();
22088 auto *VTy = dyn_cast<FixedVectorType>(Ty);
22093 unsigned NumElements = VTy->getNumElements();
22100 if (ScalarTy->isHalfTy() || ScalarTy->isFloatTy())
22101 return Subtarget->hasMVEFloatOps();
22106 return Subtarget->hasMVEIntegerOps() &&
22107 (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
22108 ScalarTy->isIntegerTy(32));
22120 assert(TyWidth >= 128 &&
"Width of vector type must be at least 128 bits");
22122 if (TyWidth > 128) {
22127 ArrayRef<int> UpperSplitMask(&SplitSeqVec[Stride], Stride);
22129 auto *LowerSplitA =
B.CreateShuffleVector(InputA, LowerSplitMask);
22130 auto *LowerSplitB =
B.CreateShuffleVector(InputB, LowerSplitMask);
22131 auto *UpperSplitA =
B.CreateShuffleVector(InputA, UpperSplitMask);
22132 auto *UpperSplitB =
B.CreateShuffleVector(InputB, UpperSplitMask);
22133 Value *LowerSplitAcc =
nullptr;
22134 Value *UpperSplitAcc =
nullptr;
22137 LowerSplitAcc =
B.CreateShuffleVector(
Accumulator, LowerSplitMask);
22138 UpperSplitAcc =
B.CreateShuffleVector(
Accumulator, UpperSplitMask);
22142 B, OperationType, Rotation, LowerSplitA, LowerSplitB, LowerSplitAcc);
22144 B, OperationType, Rotation, UpperSplitA, UpperSplitB, UpperSplitAcc);
22147 return B.CreateShuffleVector(LowerSplitInt, UpperSplitInt, JoinMask);
22154 ConstRotation = ConstantInt::get(IntTy, (
int)Rotation);
22157 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmlaq, Ty,
22159 return B.CreateIntrinsic(Intrinsic::arm_mve_vcmulq, Ty,
22160 {ConstRotation, InputB, InputA});
22165 auto *ConstHalving = ConstantInt::get(IntTy, 1);
22168 ConstRotation = ConstantInt::get(IntTy, 0);
22170 ConstRotation = ConstantInt::get(IntTy, 1);
22172 if (!ConstRotation)
22175 return B.CreateIntrinsic(Intrinsic::arm_mve_vcaddq, Ty,
22176 {ConstHalving, ConstRotation, InputA, InputB});
unsigned const MachineRegisterInfo * MRI
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG, bool isSigned)
static bool isZeroExtended(SDValue N, SelectionDAG &DAG)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static const MCPhysReg GPRArgRegs[]
static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1, SDValue V2, unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static bool isSignExtended(SDValue N, SelectionDAG &DAG)
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
static bool isStore(int Opcode)
static bool isThumb(const MCSubtargetInfo &STI)
static SDValue LowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const TargetInstrInfo *TII)
MatchingStackOffset - Return true if the given stack call argument is already available in the same p...
static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG)
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total value size to 64 bits.
static cl::opt< unsigned > ConstpoolPromotionMaxSize("arm-promote-constant-max-size", cl::Hidden, cl::desc("Maximum size of constant to promote into a constant pool"), cl::init(64))
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isVTBLMask(ArrayRef< int > M, EVT VT)
static SDValue PerformSUBCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
static cl::opt< bool > EnableConstpoolPromotion("arm-promote-constant", cl::Hidden, cl::desc("Enable / disable promotion of unnamed_addr constants into " "constant pools"), cl::init(false))
static SDValue PerformFAddVSelectCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG)
static SDValue PerformVDUPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
static SDValue PerformExtractEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static const APInt * isPowerOf2Constant(SDValue V)
static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool isValidMVECond(unsigned CC, bool IsFloat)
static SDValue PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC)
IntCCToARMCC - Convert a DAG integer condition code to an ARM CC.
static SDValue PerformSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformSTORECombine - Target-specific dag combine xforms for ISD::STORE.
static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry, SelectionDAG &DAG)
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isGTorGE(ISD::CondCode CC)
static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a vldN-lane (N > 1) intrinsic,...
static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask)
static bool isReverseMask(ArrayRef< int > M, EVT VT)
static bool isVZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of "vector_shuffle v,...
static SDValue PerformSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVECTOR_REG_CASTCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG)
static bool findPointerConstIncrement(SDNode *N, SDValue *Ptr, SDValue *CInc)
static bool isVTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool CanInvertMVEVCMP(SDValue N)
static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG)
static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformShiftCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
PerformShiftCombine - Checks for immediate versions of vector shifts and lowers them.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode, ARMCC::CondCodes &CondCode2)
FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static EVT getVectorTyFromPredicateVector(EVT VT)
static SDValue PerformFADDVCMLACombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
static bool isSRL16(const SDValue &Op)
static SDValue PerformVMOVrhCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC)
static unsigned getPointerConstIncrement(unsigned Opcode, SDValue Ptr, SDValue Inc, const SelectionDAG &DAG)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static Register genTPEntry(MachineBasicBlock *TpEntry, MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpExit, Register OpSizeReg, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI)
Adds logic in loop entry MBB to calculate loop iteration count and adds t2WhileLoopSetup and t2WhileL...
static bool isLTorLE(ISD::CondCode CC)
static SDValue PerformVCMPCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl, SelectionDAG &DAG)
static SDValue PerformBITCASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG)
static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static SDValue PerformCMPZCombine(SDNode *N, SelectionDAG &DAG)
static bool hasNormalLoadOperand(SDNode *N)
hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node are normal,...
static SDValue PerformInsertEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
PerformInsertEltCombine - Target-specific dag combine xforms for ISD::INSERT_VECTOR_ELT.
static SDValue PerformVDUPLANECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVDUPLANECombine - Target-specific dag combine xforms for ARMISD::VDUPLANE.
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static cl::opt< unsigned > ConstpoolPromotionMaxTotal("arm-promote-constant-max-total", cl::Hidden, cl::desc("Maximum size of ALL constants to promote into a constant pool"), cl::init(128))
static SDValue LowerTruncatei1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static RTLIB::Libcall getDivRemLibcall(const SDNode *N, MVT::SimpleValueType SVT)
static SDValue PerformABSCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG &DAG)
SkipLoadExtensionForVMULL - return a load of the original vector size that does not do any sign/zero ...
static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static const MCPhysReg GPRArgRegs[]
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, SelectionDAG &DAG)
static bool isVZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformORCombineToSMULWBT(SDNode *OR, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isVTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of "vector_shuffle v,...
static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue FindBFIToCombineWith(SDNode *N)
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, SelectionDAG &DAG)
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode, bool &swpCmpOps, bool &swpVselOps)
static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isS16(const SDValue &Op, SelectionDAG &DAG)
static bool isSRA16(const SDValue &Op)
static SDValue AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue LowerInterruptReturn(SmallVectorImpl< SDValue > &RetOps, const SDLoc &DL, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl, SelectionDAG &DAG)
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG, SDValue &RetVal1, SDValue &RetVal2)
static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isSHL16(const SDValue &Op)
static bool isVEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseVEXT, unsigned &Imm)
static SDValue PerformMVEVLDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isTruncMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2)
Return the load opcode for a given load size.
static bool isLegalT2AddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
static bool isLegalMVEShuffleOp(unsigned PFEntry)
static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformShuffleVMOVNCombine(ShuffleVectorSDNode *N, SelectionDAG &DAG)
static bool isVUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG)
PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for ISD::VECTOR_SHUFFLE.
static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG)
SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND, ANY_EXTEND,...
static bool isVMOVNTruncMask(ArrayRef< int > M, EVT ToVT, bool rev)
static SDValue PerformVQMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
static SDValue LowerVecReduceMinMax(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformAddcSubcCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static TargetLowering::ArgListTy getDivRemArgList(const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget)
static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static ARMCC::CondCodes getVCMPCondCode(SDValue N)
static cl::opt< bool > ARMInterworking("arm-interworking", cl::Hidden, cl::desc("Enable / disable ARM interworking (for debugging only)"), cl::init(true))
static void ReplaceREADCYCLECOUNTER(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformORCombineToBFI(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes, SDValue &CC, bool &Invert, SDValue &OtherOp, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformVSetCCToVCTPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isZeroVector(SDValue N)
static SDValue PerformAddeSubeCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void ReplaceCMP_SWAP_64Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS, const SDValue TrueVal, const SDValue FalseVal, const ISD::CondCode CC, const SDValue K)
static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG)
static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned StSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment store operation with given size.
static bool isVMOVNMask(ArrayRef< int > M, EVT VT, bool Top, bool SingleSource)
static SDValue CombineBaseUpdate(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
CombineBaseUpdate - Target-specific DAG combine function for VLDDUP, NEON load/store intrinsics,...
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSubCSINCCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVRRDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMOVRRDCombine - Target-specific dag combine xforms for ARMISD::VMOVRRD.
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformCSETCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformVMOVNCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue PerformInsertSubvectorCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerVectorExtend(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain)
static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue PerformVMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformVMULCombine Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the special multi...
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
static SDValue PerformBFICombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformORCombine - Target-specific dag combine xforms for ISD::OR.
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
static SDValue PerformTruncatingStoreCombine(StoreSDNode *St, SelectionDAG &DAG)
static unsigned SelectPairHalf(unsigned Elements, ArrayRef< int > Mask, unsigned Index)
static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos, const TargetInstrInfo *TII, const DebugLoc &dl, unsigned LdSize, unsigned Data, unsigned AddrIn, unsigned AddrOut, bool IsThumb1, bool IsThumb2)
Emit a post-increment load operation with given size.
static SDValue TryDistrubutionADDVecReduce(SDNode *N, SelectionDAG &DAG)
static bool isValidBaseUpdate(SDNode *N, SDNode *User)
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG, const ARMSubtarget *ST, const SDLoc &dl)
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op)
static SDValue PerformMinMaxToSatCombine(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformXORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, bool isSEXTLoad, bool IsMasked, bool isLE, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
std::pair< unsigned, const TargetRegisterClass * > RCPair
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,...
static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
cl::opt< unsigned > MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden, cl::desc("Maximum interleave factor for MVE VLDn to generate."), cl::init(2))
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef, unsigned SplatBitSize, SelectionDAG &DAG, const SDLoc &dl, EVT &VT, EVT VectorVT, VMOVModImmType type)
isVMOVModifiedImm - Check if the specified splat value corresponds to a valid vector constant for a N...
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC, SelectionDAG &DAG)
BC is a bitcast that is about to be turned into a VMOVDRR.
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI, const GlobalValue *GV, SelectionDAG &DAG, EVT PtrVT, const SDLoc &dl)
static unsigned isNEONTwoResultShuffleMask(ArrayRef< int > ShuffleMask, EVT VT, unsigned &WhichResult, bool &isV_UNDEF)
Check if ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN), and return the corresponding AR...
static bool BitsProperlyConcatenate(const APInt &A, const APInt &B)
static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, bool isSEXTLoad, SDValue &Base, SDValue &Offset, bool &isInc, SelectionDAG &DAG)
static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG)
static bool TryCombineBaseUpdate(struct BaseUpdateTarget &Target, struct BaseUpdateUser &User, bool SimpleConstIncOnly, TargetLowering::DAGCombinerInfo &DCI)
static bool allUsersAreInFunction(const Value *V, const Function *F)
Return true if all users of V are within function F, looking through ConstantExprs.
static bool isSingletonVEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG)
PerformVMOVDRRCombine - Target-specific dag combine xforms for ARMISD::VMOVDRR.
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V, SDValue &SatK)
static bool isLegalAddressImmediate(int64_t V, EVT VT, const ARMSubtarget *Subtarget)
isLegalAddressImmediate - Return true if the integer value can be used as the offset of the target ad...
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static bool isLegalT1AddressImmediate(int64_t V, EVT VT)
static SDValue CombineANDShift(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG)
static SDValue PerformSHLSimplify(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformADDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDECombine - Target-specific dag combine transform from ARMISD::ADDC, ARMISD::ADDE,...
static SDValue PerformReduceShuffleCombine(SDNode *N, SelectionDAG &DAG)
static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerTruncate(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformHWLoopCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *ST)
static SDValue PerformSplittingMVETruncToNarrowingStores(StoreSDNode *St, SelectionDAG &DAG)
static bool isVUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of "vector_shuffle v,...
static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base, uint64_t &Members)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG)
static SDValue PerformVDIVCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
PerformVDIVCombine - VCVT (fixed-point to floating-point, Advanced SIMD) can replace combinations of ...
static SDValue PerformANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformADDVecReduce(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG)
static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm, bool &Negate)
static bool canChangeToInt(SDValue Op, bool &SeenZero, const ARMSubtarget *Subtarget)
canChangeToInt - Given the fp compare operand, return true if it is suitable to morph to an integer c...
static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2)
Return the store opcode for a given store size.
static bool IsVUZPShuffleNode(SDNode *N)
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget, MachineInstr &MI, const SDNode *Node)
Attaches vregs to MEMCPY that it will use as scratch registers when it is expanded into LDM/STM.
static bool isFloatingPointZero(SDValue Op)
isFloatingPointZero - Return true if this is +0.0.
static SDValue findMUL_LOHI(SDValue V)
static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformORCombine_i1(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformSplittingMVEEXTToWideningLoad(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG)
static void genTPLoopBody(MachineBasicBlock *TpLoopBody, MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit, const TargetInstrInfo *TII, DebugLoc Dl, MachineRegisterInfo &MRI, Register OpSrcReg, Register OpDestReg, Register ElementCountReg, Register TotalIterationsReg, bool IsMemcpy)
Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and t2DoLoopEnd.
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
PerformMinMaxCombine - Target-specific DAG combining for creating truncating saturates.
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static Function * getFunction(Constant *C)
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
loop Loop Strength Reduction
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static X86::CondCode getSwappedCondition(X86::CondCode CC)
Assuming the flags are set by MI(a,b), return the condition code if we modify the instructions such t...
static constexpr int Concat[]
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
virtual const ARMBaseRegisterInfo & getRegisterInfo() const =0
const uint32_t * getSjLjDispatchPreservedMask(const MachineFunction &MF) const
const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const override
Code Generation virtual methods...
Register getFrameRegister(const MachineFunction &MF) const override
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
const uint32_t * getTLSCallPreservedMask(const MachineFunction &MF) const
const uint32_t * getThisReturnPreservedMask(const MachineFunction &MF, CallingConv::ID) const
getThisReturnPreservedMask - Returns a call preserved mask specific to the case that 'returned' is on...
static ARMConstantPoolConstant * Create(const Constant *C, unsigned ID)
static ARMConstantPoolMBB * Create(LLVMContext &C, const MachineBasicBlock *mbb, unsigned ID, unsigned char PCAdj)
static ARMConstantPoolSymbol * Create(LLVMContext &C, StringRef s, unsigned ID, unsigned char PCAdj)
ARMConstantPoolValue - ARM specific constantpool value.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
int getVarArgsFrameIndex() const
int getPromotedConstpoolIncrease() const
SmallPtrSet< const GlobalVariable *, 2 > & getGlobalsPromotedToConstantPool()
void setArgumentStackToRestore(unsigned v)
bool branchTargetEnforcement() const
unsigned createPICLabelUId()
void setPromotedConstpoolIncrease(int Sz)
bool isThumb1OnlyFunction() const
void setArgRegsSaveSize(unsigned s)
bool shouldSignReturnAddress() const
bool isCmseNSEntryFunction() const
void setReturnRegsCount(unsigned s)
void setVarArgsFrameIndex(int Index)
unsigned getArgRegsSaveSize() const
void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV)
Indicate to the backend that GV has had its storage changed to inside a constant pool.
void setIsSplitCSR(bool s)
void setArgumentStackSize(unsigned size)
unsigned getArgumentStackSize() const
bool isTargetMachO() const
bool isTargetAEABI() const
bool supportsTailCall() const
const Triple & getTargetTriple() const
const ARMBaseInstrInfo * getInstrInfo() const override
bool isThumb1Only() const
bool hasFPARMv8Base() const
bool isTargetWindows() const
bool isGVIndirectSymbol(const GlobalValue *GV) const
True if the GV will be accessed via an indirect symbol.
const ARMTargetLowering * getTargetLowering() const override
bool isTargetDarwin() const
const ARMBaseRegisterInfo * getRegisterInfo() const override
bool isTargetAndroid() const
bool isTargetCOFF() const
bool isTargetGNUAEABI() const
bool isTargetWatchOS() const
bool useNEONForSinglePrecisionFP() const
const InstrItineraryData * getInstrItineraryData() const override
getInstrItins - Return the instruction itineraries based on subtarget selection.
bool isTargetWatchABI() const
bool hasAnyDataBarrier() const
bool isTargetDriverKit() const
bool allowsUnalignedMem() const
bool isTargetMuslAEABI() const
bool isTargetLinux() const
unsigned getPrefLoopLogAlignment() const
bool isTargetHardFloat() const
Align getDualLoadStoreAlignment() const
bool isReadOnly(const GlobalValue *GV) const
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const override
Return the correct alignment for the current calling convention.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
const ARMSubtarget * getSubtarget() const
bool isLegalT2ScaledAddressingMode(const AddrMode &AM, EVT VT) const
bool isLegalT1ScaledAddressingMode(const AddrMode &AM, EVT VT) const
Returns true if the addressing mode representing by AM is legal for the Thumb1 target,...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPreIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mod...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
ReplaceNodeResults - Replace the results of node with an illegal result type with new values built ou...
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool isLegalAddImmediate(int64_t Imm) const override
isLegalAddImmediate - Return true if the specified immediate is legal add immediate,...
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
SDValue PerformMVETruncCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const override
isFPImmLegal - Returns true if the target can instruction select the specified FP immediate natively.
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
SDValue PerformIntrinsicCombine(SDNode *N, DAGCombinerInfo &DCI) const
PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const
PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
SDValue PerformMVEExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
getSetCCResultType - Return the value type to use for ISD::SETCC.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
createFastISel - This method returns a target specific FastISel object, or null if the target does no...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
isShuffleMaskLegal - Targets can use this to indicate that they only support some VECTOR_SHUFFLE oper...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const override
getRegClassFor - Return the register class that should be used for the specified value type.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vstN intrinsic.
ARMTargetLowering(const TargetMachine &TM, const ARMSubtarget &STI)
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const
PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Type * shouldConvertSplatType(ShuffleVectorInst *SVI) const override
Given a shuffle vector SVI representing a vector splat, return a new scalar type of size equal to SVI...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
Instruction * makeDMB(IRBuilderBase &Builder, ARM_MB::MemBOpt Domain) const
bool isLegalICmpImmediate(int64_t Imm) const override
isLegalICmpImmediate - Return true if the specified immediate is legal icmp immediate,...
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
allowsMisalignedMemoryAccesses - Returns true if the target allows unaligned memory accesses of the s...
bool isLegalInterleavedAccessType(unsigned Factor, FixedVectorType *VecTy, Align Alignment, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
bool isVectorLoadExtDesirable(SDValue ExtVal) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx, unsigned &Cost) const override
Return true if the target can combine store(extractelement VectorTy, Idx).
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vldN intrinsic.
bool useSoftFloat() const override
bool alignLoopsWithOptSize() const override
Should loops be aligned even when the function is marked OptSize (but not MinSize).
SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
LowerAsmOperandForConstraint - Lower the specified operand into the Ops vector.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
Returns true if an argument of type Ty needs to be passed in a contiguous block of registers in calli...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
getPostIndexedAddressParts - returns true by value, base pointer and offset pointer and addressing mo...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
bool isFloatingPointOperation() const
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
The address of a basic block.
static BranchProbability getZero()
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
CCState - This class holds information needed while lowering arguments and return values.
void getInRegsParamInfo(unsigned InRegsParamRecordIndex, unsigned &BeginReg, unsigned &EndReg) const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
void rewindByValRegsInfo()
unsigned getInRegsParamsProcessed() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void addInRegsParamInfo(unsigned RegBegin, unsigned RegEnd)
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getInRegsParamsCount() const
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
unsigned getValNo() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getCalledOperand() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
const APFloat & getValueAPF() const
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
Align getStackAlignment() const
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPreferredAlign(const GlobalVariable *GV) const
Returns the preferred alignment of the specified global.
StringRef getPrivateGlobalPrefix() const
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
const GlobalValue * getGlobal() const
bool hasExternalWeakLinkage() const
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
bool isStrongDefinitionForLinker() const
Returns true if this global's definition will be the one chosen by the linker.
@ InternalLinkage
Rename collisions when linking (static functions).
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
Common base class shared among various IRBuilders.
Value * CreateZExtOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
BasicBlock * GetInsertBlock() const
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTruncOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::optional< unsigned > getOperandCycle(unsigned ItinClassIndx, unsigned OperandIdx) const
Return the cycle for the given class and operand.
bool isEmpty() const
Returns true if there are no itineraries.
bool hasAtomicStore() const LLVM_READONLY
Return true if this atomic instruction stores to memory.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
const BasicBlock * getParent() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
static bool LowerToByteSwap(CallInst *CI)
Try to replace a call instruction with a call to a bswap intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Describe properties that are true of each instruction in the target description file.
unsigned getSchedClass() const
Return the scheduling class for this instruction.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool canFallThrough()
Return true if the block can implicitly transfer control to the block after it by falling off the end...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
pred_iterator pred_begin()
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void moveAfter(MachineBasicBlock *NewBefore)
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void computeMaxCallFrameSize(MachineFunction &MF, std::vector< MachineBasicBlock::iterator > *FrameSDOps=nullptr)
Computes the maximum size of a callframe.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool hasVAStart() const
Returns true if the function calls the llvm.va_start intrinsic.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
int getFunctionContextIndex() const
Return the index for the function context object.
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & reset(Property P)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i....
Representation of each machine instruction.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
const MachineOperand & getOperand(unsigned i) const
unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
void setIsDef(bool Val=true)
Change a def to a use, or a use to a def.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< use_iterator > uses()
size_t use_size() const
Return the number of uses of this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
void setFlags(SDNodeFlags NewFlags)
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
void addCallSiteInfo(const SDNode *Node, CallSiteInfo &&CallInfo)
Set CallSiteInfo to be associated with Node.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
StringRef - Represent a constant reference to a string, i.e.
const unsigned char * bytes_end() const
constexpr size_t size() const
size - Get the string size.
const unsigned char * bytes_begin() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC)
Override the default CondCode to be used to test the result of the comparison libcall against zero.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
void setMinStackArgumentAlignment(Align Alignment)
Set the minimum stack alignment of an argument.
const TargetMachine & getTargetMachine() const
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
void setIndexedMaskedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked load does or does not work with the specified type and ind...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
virtual unsigned getMaxSupportedInterleaveFactor() const
Get the maximum supported factor for interleaved memory accesses.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setIndexedMaskedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed masked store does or does not work with the specified type and in...
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
virtual std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const
Return the largest legal super-reg register class of the register class for the specified type and it...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
bool isPositionIndependent() const
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
bool isOSVersionLT(unsigned Major, unsigned Minor=0, unsigned Micro=0) const
Helper function for doing comparisons against version numbers included in the target triple.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isPointerTy() const
True if this is an instance of PointerType.
Type * getArrayElementType() const
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static CondCodes getOppositeCondition(CondCodes CC)
@ SECREL
Thread Pointer Offset.
@ SBREL
Section Relative (Windows TLS)
@ GOTTPOFF
Global Offset Table, PC Relative.
@ TPOFF
Global Offset Table, Thread Pointer Offset.
TOF
Target Operand Flag enum.
@ MO_NONLAZY
MO_NONLAZY - This is an independent flag, on a symbol operand "FOO" it represents a symbol which,...
@ MO_SBREL
MO_SBREL - On a symbol operand, this represents a static base relative relocation.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_GOT
MO_GOT - On a symbol operand, this represents a GOT relative relocation.
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
unsigned getAM2Offset(unsigned AM2Opc)
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned createVMOVModImm(unsigned OpCmode, unsigned Val)
int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
int getFP32FP16Imm(const APInt &Imm)
If this is a FP16Imm encoded as a fp32 value, return the 8-bit encoding for it.
AddrOpc getAM2Op(unsigned AM2Opc)
bool isBitFieldInvertedMask(unsigned v)
const unsigned FPStatusBits
const unsigned FPReservedBits
const unsigned RoundingBitsPos
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Swift
Calling convention for Swift.
@ ARM_APCS
ARM Procedure Calling Standard (obsolete, but still used on some targets).
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ ARM_AAPCS
ARM Architecture Procedure Calling Standard calling convention (aka EABI).
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ ARM_AAPCS_VFP
Same as ARM_AAPCS, but uses hard floating point ABI.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns true if Val1 has a lower Constant Materialization Cost than Val2.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
ComplexDeinterleavingOperation
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
ComplexDeinterleavingRotation
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
bool isVREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isVREVMask - Check if a vector shuffle corresponds to a VREV instruction with the specified blocksize...
unsigned gettBLXrOpcode(const MachineFunction &MF)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
unsigned convertAddSubFlagsOpcode(unsigned OldOpc)
Map pseudo instructions that imply an 'S' bit onto real opcodes.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
static const unsigned PerfectShuffleTable[6561+1]
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Load/store instruction that can be merged with a base address update.
SDNode * N
Instruction that updates a pointer.
unsigned ConstInc
Pointer increment value if it is a constant, or 0 otherwise.
SDValue Inc
Pointer increment operand.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
bool isUnknown() const
Returns true if we don't know any bits.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This contains information for each constraint that we are lowering.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setInRegister(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)