32 #include "llvm/IR/IntrinsicsARM.h"
41 #define DEBUG_TYPE "arm-isel"
45 cl::desc(
"Disable isel of shifter-op"),
70 StringRef getPassName()
const override {
return "ARM Instruction Selection"; }
72 void PreprocessISelDAG()
override;
77 return CurDAG->getTargetConstant(
Imm, dl,
MVT::i32);
84 bool ComplexPatternFuncMutatesDAG()
const override {
return true; }
86 bool hasNoVMLxHazardUse(
SDNode *
N)
const;
91 bool CheckProfitability =
true);
93 SDValue &
B,
bool CheckProfitability =
true);
97 return SelectRegShifterOperand(
N, A,
B,
C,
false);
101 return SelectImmShifterOperand(
N, A,
B,
false);
106 return SelectImmShifterOperand(
N, A,
B,
false);
152 template <
unsigned Shift>
157 template <
unsigned Shift>
163 template <
unsigned Shift>
167 template <
unsigned Shift>
173 template<
int Min,
int Max>
176 inline bool is_so_imm(
unsigned Imm)
const {
180 inline bool is_so_imm_not(
unsigned Imm)
const {
184 inline bool is_t2_so_imm(
unsigned Imm)
const {
188 inline bool is_t2_so_imm_not(
unsigned Imm)
const {
193 #include "ARMGenDAGISel.inc"
199 bool tryARMIndexedLoad(
SDNode *
N);
200 bool tryT1IndexedLoad(
SDNode *
N);
201 bool tryT2IndexedLoad(
SDNode *
N);
202 bool tryMVEIndexedLoad(
SDNode *
N);
205 bool transformFixedFloatingPointConversion(
SDNode *
N,
SDNode *FMul,
213 void SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
221 void SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
228 void SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
229 unsigned NumVecs,
const uint16_t *DOpcodes,
233 template <
typename SDValueVector>
234 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
236 template <
typename SDValueVector>
237 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
240 template <
typename SDValueVector>
241 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc);
242 template <
typename SDValueVector>
243 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
EVT InactiveTy);
246 void SelectMVE_WB(
SDNode *
N,
const uint16_t *Opcodes,
bool Predicated);
250 bool HasSaturationOperand);
254 uint16_t OpcodeWithNoCarry,
bool Add,
bool Predicated);
258 void SelectMVE_VSHLC(
SDNode *
N,
bool Predicated);
268 void SelectBaseMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
270 size_t Stride,
size_t TySize);
274 void SelectMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
const uint16_t *OpcodesS,
278 void SelectMVE_VRMLLDAVH(
SDNode *
N,
bool Predicated,
const uint16_t *OpcodesS,
286 void SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
287 const uint16_t *
const *Opcodes,
bool HasWriteback);
292 bool Wrapping,
bool Predicated);
306 void SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
bool isUpdating,
307 unsigned NumVecs,
const uint16_t *DOpcodes,
308 const uint16_t *QOpcodes0 =
nullptr,
309 const uint16_t *QOpcodes1 =
nullptr);
312 bool tryV6T2BitfieldExtractOp(
SDNode *
N,
bool isSigned);
314 bool tryInsertVectorElt(
SDNode *
N);
319 bool tryReadRegister(
SDNode *
N);
320 bool tryWriteRegister(
SDNode *
N);
324 void SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI);
326 void SelectCMP_SWAP(
SDNode *
N);
330 bool SelectInlineAsmMemoryOperand(
const SDValue &
Op,
unsigned ConstraintID,
331 std::vector<SDValue> &OutOps)
override;
353 bool canExtractShiftFromMul(
const SDValue &
N,
unsigned MaxShift,
354 unsigned &PowerOfTwo,
SDValue &NewMulConst)
const;
366 Imm = cast<ConstantSDNode>(
N)->getZExtValue();
382 return N->getOpcode() == Opc &&
391 int RangeMin,
int RangeMax,
392 int &ScaledConstant) {
393 assert(Scale > 0 &&
"Invalid scale!");
400 ScaledConstant = (
int)
C->getZExtValue();
401 if ((ScaledConstant % Scale) != 0)
404 ScaledConstant /= Scale;
405 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
408 void ARMDAGToDAGISel::PreprocessISelDAG() {
409 if (!Subtarget->hasV6T2Ops())
412 bool isThumb2 = Subtarget->isThumb();
429 unsigned And_imm = 0;
439 if (TZ != 1 && TZ != 2)
451 if (And_imm & (And_imm + 1))
456 unsigned Srl_imm = 0;
467 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
470 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
471 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
478 CurDAG->getConstant(Srl_imm + TZ,
SDLoc(Srl),
485 CurDAG->UpdateNodeOperands(&
N, N0, N1);
492 bool ARMDAGToDAGISel::hasNoVMLxHazardUse(
SDNode *
N)
const {
496 if (!Subtarget->hasVMLxHazards())
505 if (
Use->isMachineOpcode()) {
507 CurDAG->getSubtarget().getInstrInfo());
528 return TII->isFpMLxInstruction(Opcode);
534 bool ARMDAGToDAGISel::isShifterOpProfitable(
const SDValue &
Shift,
539 if (
Shift.hasOneUse())
543 (ShAmt == 2 || (Subtarget->
isSwift() && ShAmt == 1));
546 bool ARMDAGToDAGISel::canExtractShiftFromMul(
const SDValue &
N,
548 unsigned &PowerOfTwo,
555 if (!
N.hasOneUse())
return false;
558 if (!MulConst)
return false;
561 if (!MulConst->
hasOneUse())
return false;
563 if (MulConstVal == 0)
return false;
566 PowerOfTwo = MaxShift;
567 while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
569 if (PowerOfTwo == 0)
return false;
573 unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
574 NewMulConst = CurDAG->getConstant(NewMulConstVal,
SDLoc(
N),
MVT::i32);
577 return NewCost < OldCost;
581 CurDAG->RepositionNode(
N.getNode()->getIterator(),
M.getNode());
585 bool ARMDAGToDAGISel::SelectImmShifterOperand(
SDValue N,
588 bool CheckProfitability) {
595 unsigned PowerOfTwo = 0;
597 if (canExtractShiftFromMul(
N, 31, PowerOfTwo, NewMulConst)) {
600 replaceDAGValue(
N.getOperand(1), NewMulConst);
602 Opc = CurDAG->getTargetConstant(
614 BaseReg =
N.getOperand(0);
615 unsigned ShImmVal = 0;
617 if (!
RHS)
return false;
618 ShImmVal =
RHS->getZExtValue() & 31;
624 bool ARMDAGToDAGISel::SelectRegShifterOperand(
SDValue N,
628 bool CheckProfitability) {
638 BaseReg =
N.getOperand(0);
639 unsigned ShImmVal = 0;
641 if (
RHS)
return false;
643 ShReg =
N.getOperand(1);
644 if (CheckProfitability && !isShifterOpProfitable(
N, ShOpcVal, ShImmVal))
656 return CurDAG->haveNoCommonBitsSet(
N, Parent->
getOperand(1));
660 bool ARMDAGToDAGISel::SelectAddrModeImm12(
SDValue N,
667 !CurDAG->isBaseWithConstantOffset(
N)) {
670 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
671 Base = CurDAG->getTargetFrameIndex(
672 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
681 Base =
N.getOperand(0);
689 int RHSC = (
int)
RHS->getSExtValue();
693 if (RHSC > -0x1000 && RHSC < 0x1000) {
694 Base =
N.getOperand(0);
696 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
697 Base = CurDAG->getTargetFrameIndex(
698 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
719 int RHSC = (
int)
RHS->getZExtValue();
728 unsigned ShAmt =
Log2_32(RHSC);
741 !CurDAG->isBaseWithConstantOffset(
N))
748 -0x1000+1, 0x1000, RHSC))
758 Base =
N.getOperand(0);
765 dyn_cast<ConstantSDNode>(
N.getOperand(1).getOperand(1))) {
766 ShAmt = Sh->getZExtValue();
767 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
768 Offset =
N.getOperand(1).getOperand(0);
781 N.getOperand(0).hasOneUse())) {
787 dyn_cast<ConstantSDNode>(
N.getOperand(0).getOperand(1))) {
788 ShAmt = Sh->getZExtValue();
789 if (isShifterOpProfitable(
N.getOperand(0), ShOpcVal, ShAmt)) {
790 Offset =
N.getOperand(0).getOperand(0);
791 Base =
N.getOperand(1);
805 unsigned PowerOfTwo = 0;
807 if (canExtractShiftFromMul(Offset, 31, PowerOfTwo, NewMulConst)) {
809 replaceDAGValue(
Offset.getOperand(1), NewMulConst);
810 Offset = Handle.getValue();
823 unsigned Opcode =
Op->getOpcode();
825 ? cast<LoadSDNode>(
Op)->getAddressingMode()
826 : cast<StoreSDNode>(
Op)->getAddressingMode();
840 ShAmt = Sh->getZExtValue();
841 if (isShifterOpProfitable(
N, ShOpcVal, ShAmt))
859 unsigned Opcode =
Op->getOpcode();
861 ? cast<LoadSDNode>(
Op)->getAddressingMode()
862 : cast<StoreSDNode>(
Op)->getAddressingMode();
879 unsigned Opcode =
Op->getOpcode();
881 ? cast<LoadSDNode>(
Op)->getAddressingMode()
882 : cast<StoreSDNode>(
Op)->getAddressingMode();
902 bool ARMDAGToDAGISel::SelectAddrMode3(
SDValue N,
907 Base =
N.getOperand(0);
914 if (!CurDAG->isBaseWithConstantOffset(
N)) {
917 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
918 Base = CurDAG->getTargetFrameIndex(
919 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
930 -256 + 1, 256, RHSC)) {
931 Base =
N.getOperand(0);
933 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
934 Base = CurDAG->getTargetFrameIndex(
935 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
949 Base =
N.getOperand(0);
958 unsigned Opcode =
Op->getOpcode();
960 ? cast<LoadSDNode>(
Op)->getAddressingMode()
961 : cast<StoreSDNode>(
Op)->getAddressingMode();
980 if (!CurDAG->isBaseWithConstantOffset(
N)) {
983 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
984 Base = CurDAG->getTargetFrameIndex(
985 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
990 Base =
N.getOperand(0);
999 const int Scale = FP16 ? 2 : 4;
1002 Base =
N.getOperand(0);
1004 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1005 Base = CurDAG->getTargetFrameIndex(
1006 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1037 bool ARMDAGToDAGISel::SelectAddrMode5(
SDValue N,
1039 return IsAddressingMode5(
N,
Base, Offset,
false);
1042 bool ARMDAGToDAGISel::SelectAddrMode5FP16(
SDValue N,
1044 return IsAddressingMode5(
N,
Base, Offset,
true);
1053 MemSDNode *MemN = cast<MemSDNode>(Parent);
1055 if (isa<LSBaseSDNode>(MemN) ||
1063 if (MMOAlign.
value() >= MemSize && MemSize > 1)
1090 bool ARMDAGToDAGISel::SelectAddrModePC(
SDValue N,
1095 Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1116 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
1117 return C->getSExtValue() < 0 &&
C->getSExtValue() >= -255;
1124 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N)) {
1126 if (!
NC || !
NC->isZero())
1133 Base =
N.getOperand(0);
1142 return SelectThumbAddrModeRRSext(
N,
Base, Offset);
1146 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(
SDValue N,
unsigned Scale,
1154 if (!CurDAG->isBaseWithConstantOffset(
N)) {
1162 Base =
N.getOperand(0);
1174 Base =
N.getOperand(0);
1186 return SelectThumbAddrModeImm5S(
N, 4,
Base, OffImm);
1192 return SelectThumbAddrModeImm5S(
N, 2,
Base, OffImm);
1198 return SelectThumbAddrModeImm5S(
N, 1,
Base, OffImm);
1201 bool ARMDAGToDAGISel::SelectThumbAddrModeSP(
SDValue N,
1204 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1210 Base = CurDAG->getTargetFrameIndex(
1211 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1216 if (!CurDAG->isBaseWithConstantOffset(
N))
1223 Base =
N.getOperand(0);
1224 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1235 Base = CurDAG->getTargetFrameIndex(
1236 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1247 template <
unsigned Shift>
1250 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1254 Base =
N.getOperand(0);
1275 bool ARMDAGToDAGISel::SelectT2AddrModeImm12(
SDValue N,
1281 !CurDAG->isBaseWithConstantOffset(
N)) {
1284 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1285 Base = CurDAG->getTargetFrameIndex(
1286 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1295 Base =
N.getOperand(0);
1305 if (SelectT2AddrModeImm8(
N,
Base, OffImm))
1309 int RHSC = (
int)
RHS->getZExtValue();
1313 if (RHSC >= 0 && RHSC < 0x1000) {
1314 Base =
N.getOperand(0);
1316 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1317 Base = CurDAG->getTargetFrameIndex(
1318 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1331 template <
unsigned Shift>
1334 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1337 Base =
N.getOperand(0);
1339 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1340 Base = CurDAG->getTargetFrameIndex(
1341 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1358 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(
SDValue N,
1362 !CurDAG->isBaseWithConstantOffset(
N))
1366 int RHSC = (
int)
RHS->getSExtValue();
1370 if ((RHSC >= -255) && (RHSC < 0)) {
1371 Base =
N.getOperand(0);
1373 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1374 Base = CurDAG->getTargetFrameIndex(
1375 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1387 unsigned Opcode =
Op->getOpcode();
1389 ? cast<LoadSDNode>(
Op)->getAddressingMode()
1390 : cast<StoreSDNode>(
Op)->getAddressingMode();
1402 template <
unsigned Shift>
1405 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1409 Base =
N.getOperand(0);
1411 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1412 Base = CurDAG->getTargetFrameIndex(
1413 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1430 template <
unsigned Shift>
1433 return SelectT2AddrModeImm7Offset(
Op,
N, OffImm,
Shift);
1439 unsigned Opcode =
Op->getOpcode();
1443 AM = cast<LoadSDNode>(
Op)->getAddressingMode();
1446 AM = cast<StoreSDNode>(
Op)->getAddressingMode();
1449 AM = cast<MaskedLoadSDNode>(
Op)->getAddressingMode();
1452 AM = cast<MaskedStoreSDNode>(
Op)->getAddressingMode();
1464 : CurDAG->getTargetConstant(-RHSC * (1 <<
Shift),
SDLoc(
N),
1471 template <
int Min,
int Max>
1472 bool ARMDAGToDAGISel::SelectImmediateInRange(
SDValue N,
SDValue &OffImm) {
1481 bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(
SDValue N,
1485 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N))
1490 int RHSC = (
int)
RHS->getZExtValue();
1491 if (RHSC >= 0 && RHSC < 0x1000)
1493 else if (RHSC < 0 && RHSC >= -255)
1499 Base =
N.getOperand(0);
1500 OffReg =
N.getOperand(1);
1514 ShAmt = Sh->getZExtValue();
1515 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1526 unsigned PowerOfTwo = 0;
1528 if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1530 replaceDAGValue(OffReg.
getOperand(1), NewMulConst);
1548 if (
N.getOpcode() !=
ISD::ADD || !CurDAG->isBaseWithConstantOffset(
N))
1556 if (RHSC > 1020 || RHSC % 4 != 0)
1559 Base =
N.getOperand(0);
1561 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1562 Base = CurDAG->getTargetFrameIndex(
1563 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1577 void ARMDAGToDAGISel::transferMemOperands(
SDNode *
N,
SDNode *Result) {
1579 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {
MemOp});
1582 bool ARMDAGToDAGISel::tryARMIndexedLoad(
SDNode *
N) {
1588 EVT LoadedVT =
LD->getMemoryVT();
1591 unsigned Opcode = 0;
1593 if (LoadedVT ==
MVT::i32 && isPre &&
1594 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(), Offset, AMOpc)) {
1595 Opcode = ARM::LDR_PRE_IMM;
1597 }
else if (LoadedVT ==
MVT::i32 && !isPre &&
1598 SelectAddrMode2OffsetImm(
N,
LD->getOffset(), Offset, AMOpc)) {
1599 Opcode = ARM::LDR_POST_IMM;
1602 SelectAddrMode2OffsetReg(
N,
LD->getOffset(), Offset, AMOpc)) {
1603 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1607 SelectAddrMode3Offset(
N,
LD->getOffset(), Offset, AMOpc)) {
1610 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1611 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1614 if (SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1616 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1620 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1622 Opcode = ARM::LDRB_PRE_IMM;
1623 }
else if (!isPre &&
1624 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1626 Opcode = ARM::LDRB_POST_IMM;
1627 }
else if (SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1629 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1635 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1639 CurDAG->getRegister(0,
MVT::i32), Chain };
1642 transferMemOperands(
N, New);
1643 ReplaceNode(
N, New);
1649 CurDAG->getRegister(0,
MVT::i32), Chain };
1652 transferMemOperands(
N, New);
1653 ReplaceNode(
N, New);
1661 bool ARMDAGToDAGISel::tryT1IndexedLoad(
SDNode *
N) {
1663 EVT LoadedVT =
LD->getMemoryVT();
1669 auto *COffs = dyn_cast<ConstantSDNode>(
LD->getOffset());
1670 if (!COffs || COffs->getZExtValue() != 4)
1680 CurDAG->getRegister(0,
MVT::i32), Chain };
1683 transferMemOperands(
N, New);
1684 ReplaceNode(
N, New);
1688 bool ARMDAGToDAGISel::tryT2IndexedLoad(
SDNode *
N) {
1694 EVT LoadedVT =
LD->getMemoryVT();
1698 unsigned Opcode = 0;
1700 if (SelectT2AddrModeImm8Offset(
N,
LD->getOffset(), Offset)) {
1703 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1707 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1709 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1714 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1716 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1728 CurDAG->getRegister(0,
MVT::i32), Chain };
1731 transferMemOperands(
N, New);
1732 ReplaceNode(
N, New);
1739 bool ARMDAGToDAGISel::tryMVEIndexedLoad(
SDNode *
N) {
1741 unsigned Opcode = 0;
1742 bool isSExtLd, isPre;
1752 LoadedVT =
LD->getMemoryVT();
1756 Chain =
LD->getChain();
1763 PredReg = CurDAG->getRegister(0,
MVT::i32);
1768 LoadedVT =
LD->getMemoryVT();
1772 Chain =
LD->getChain();
1779 PredReg =
LD->getMask();
1786 bool CanChangeType = Subtarget->
isLittle() && !isa<MaskedLoadSDNode>(
N);
1790 SelectT2AddrModeImm7Offset(
N, Offset, NewOffset, 1)) {
1792 Opcode = isPre ? ARM::MVE_VLDRHS32_pre : ARM::MVE_VLDRHS32_post;
1794 Opcode = isPre ? ARM::MVE_VLDRHU32_pre : ARM::MVE_VLDRHU32_post;
1796 SelectT2AddrModeImm7Offset(
N, Offset, NewOffset, 0)) {
1798 Opcode = isPre ? ARM::MVE_VLDRBS16_pre : ARM::MVE_VLDRBS16_post;
1800 Opcode = isPre ? ARM::MVE_VLDRBU16_pre : ARM::MVE_VLDRBU16_post;
1802 SelectT2AddrModeImm7Offset(
N, Offset, NewOffset, 0)) {
1804 Opcode = isPre ? ARM::MVE_VLDRBS32_pre : ARM::MVE_VLDRBS32_post;
1806 Opcode = isPre ? ARM::MVE_VLDRBU32_pre : ARM::MVE_VLDRBU32_post;
1807 }
else if (Alignment >=
Align(4) &&
1810 SelectT2AddrModeImm7Offset(
N, Offset, NewOffset, 2))
1811 Opcode = isPre ? ARM::MVE_VLDRWU32_pre : ARM::MVE_VLDRWU32_post;
1812 else if (Alignment >=
Align(2) &&
1815 SelectT2AddrModeImm7Offset(
N, Offset, NewOffset, 1))
1816 Opcode = isPre ? ARM::MVE_VLDRHU16_pre : ARM::MVE_VLDRHU16_post;
1817 else if ((CanChangeType || LoadedVT ==
MVT::v16i8) &&
1818 SelectT2AddrModeImm7Offset(
N, Offset, NewOffset, 0))
1819 Opcode = isPre ? ARM::MVE_VLDRBU8_pre : ARM::MVE_VLDRBU8_post;
1831 transferMemOperands(
N, New);
1835 CurDAG->RemoveDeadNode(
N);
1843 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl,
MVT::i32);
1844 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl,
MVT::i32);
1845 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl,
MVT::i32);
1846 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1847 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1854 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl,
MVT::i32);
1855 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl,
MVT::i32);
1856 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl,
MVT::i32);
1857 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1858 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1864 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1866 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl,
MVT::i32);
1867 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl,
MVT::i32);
1868 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1869 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1875 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1877 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl,
MVT::i32);
1878 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl,
MVT::i32);
1879 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1880 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1888 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl,
MVT::i32);
1889 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl,
MVT::i32);
1890 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl,
MVT::i32);
1891 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl,
MVT::i32);
1892 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl,
MVT::i32);
1893 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1894 V2, SubReg2, V3, SubReg3 };
1895 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1902 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1904 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl,
MVT::i32);
1905 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl,
MVT::i32);
1906 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl,
MVT::i32);
1907 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl,
MVT::i32);
1908 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1909 V2, SubReg2, V3, SubReg3 };
1910 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1917 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1919 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl,
MVT::i32);
1920 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl,
MVT::i32);
1921 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl,
MVT::i32);
1922 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl,
MVT::i32);
1923 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1924 V2, SubReg2, V3, SubReg3 };
1925 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1932 unsigned NumVecs,
bool is64BitVector) {
1933 unsigned NumRegs = NumVecs;
1934 if (!is64BitVector && NumVecs < 3)
1937 unsigned Alignment = cast<ConstantSDNode>(
Align)->getZExtValue();
1938 if (Alignment >= 32 && NumRegs == 4)
1940 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1942 else if (Alignment >= 8)
1947 return CurDAG->getTargetConstant(Alignment, dl,
MVT::i32);
1953 default:
return false;
1954 case ARM::VLD1d8wb_fixed :
return true;
1955 case ARM::VLD1d16wb_fixed :
return true;
1956 case ARM::VLD1d64Qwb_fixed :
return true;
1957 case ARM::VLD1d32wb_fixed :
return true;
1958 case ARM::VLD1d64wb_fixed :
return true;
1959 case ARM::VLD1d8TPseudoWB_fixed :
return true;
1960 case ARM::VLD1d16TPseudoWB_fixed :
return true;
1961 case ARM::VLD1d32TPseudoWB_fixed :
return true;
1962 case ARM::VLD1d64TPseudoWB_fixed :
return true;
1963 case ARM::VLD1d8QPseudoWB_fixed :
return true;
1964 case ARM::VLD1d16QPseudoWB_fixed :
return true;
1965 case ARM::VLD1d32QPseudoWB_fixed :
return true;
1966 case ARM::VLD1d64QPseudoWB_fixed :
return true;
1967 case ARM::VLD1q8wb_fixed :
return true;
1968 case ARM::VLD1q16wb_fixed :
return true;
1969 case ARM::VLD1q32wb_fixed :
return true;
1970 case ARM::VLD1q64wb_fixed :
return true;
1971 case ARM::VLD1DUPd8wb_fixed :
return true;
1972 case ARM::VLD1DUPd16wb_fixed :
return true;
1973 case ARM::VLD1DUPd32wb_fixed :
return true;
1974 case ARM::VLD1DUPq8wb_fixed :
return true;
1975 case ARM::VLD1DUPq16wb_fixed :
return true;
1976 case ARM::VLD1DUPq32wb_fixed :
return true;
1977 case ARM::VLD2d8wb_fixed :
return true;
1978 case ARM::VLD2d16wb_fixed :
return true;
1979 case ARM::VLD2d32wb_fixed :
return true;
1980 case ARM::VLD2q8PseudoWB_fixed :
return true;
1981 case ARM::VLD2q16PseudoWB_fixed :
return true;
1982 case ARM::VLD2q32PseudoWB_fixed :
return true;
1983 case ARM::VLD2DUPd8wb_fixed :
return true;
1984 case ARM::VLD2DUPd16wb_fixed :
return true;
1985 case ARM::VLD2DUPd32wb_fixed :
return true;
1986 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return true;
1987 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return true;
1988 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return true;
1995 default:
return false;
1996 case ARM::VST1d8wb_fixed :
return true;
1997 case ARM::VST1d16wb_fixed :
return true;
1998 case ARM::VST1d32wb_fixed :
return true;
1999 case ARM::VST1d64wb_fixed :
return true;
2000 case ARM::VST1q8wb_fixed :
return true;
2001 case ARM::VST1q16wb_fixed :
return true;
2002 case ARM::VST1q32wb_fixed :
return true;
2003 case ARM::VST1q64wb_fixed :
return true;
2004 case ARM::VST1d8TPseudoWB_fixed :
return true;
2005 case ARM::VST1d16TPseudoWB_fixed :
return true;
2006 case ARM::VST1d32TPseudoWB_fixed :
return true;
2007 case ARM::VST1d64TPseudoWB_fixed :
return true;
2008 case ARM::VST1d8QPseudoWB_fixed :
return true;
2009 case ARM::VST1d16QPseudoWB_fixed :
return true;
2010 case ARM::VST1d32QPseudoWB_fixed :
return true;
2011 case ARM::VST1d64QPseudoWB_fixed :
return true;
2012 case ARM::VST2d8wb_fixed :
return true;
2013 case ARM::VST2d16wb_fixed :
return true;
2014 case ARM::VST2d32wb_fixed :
return true;
2015 case ARM::VST2q8PseudoWB_fixed :
return true;
2016 case ARM::VST2q16PseudoWB_fixed :
return true;
2017 case ARM::VST2q32PseudoWB_fixed :
return true;
2025 &&
"Incorrect fixed stride updating instruction.");
2028 case ARM::VLD1d8wb_fixed:
return ARM::VLD1d8wb_register;
2029 case ARM::VLD1d16wb_fixed:
return ARM::VLD1d16wb_register;
2030 case ARM::VLD1d32wb_fixed:
return ARM::VLD1d32wb_register;
2031 case ARM::VLD1d64wb_fixed:
return ARM::VLD1d64wb_register;
2032 case ARM::VLD1q8wb_fixed:
return ARM::VLD1q8wb_register;
2033 case ARM::VLD1q16wb_fixed:
return ARM::VLD1q16wb_register;
2034 case ARM::VLD1q32wb_fixed:
return ARM::VLD1q32wb_register;
2035 case ARM::VLD1q64wb_fixed:
return ARM::VLD1q64wb_register;
2036 case ARM::VLD1d64Twb_fixed:
return ARM::VLD1d64Twb_register;
2037 case ARM::VLD1d64Qwb_fixed:
return ARM::VLD1d64Qwb_register;
2038 case ARM::VLD1d8TPseudoWB_fixed:
return ARM::VLD1d8TPseudoWB_register;
2039 case ARM::VLD1d16TPseudoWB_fixed:
return ARM::VLD1d16TPseudoWB_register;
2040 case ARM::VLD1d32TPseudoWB_fixed:
return ARM::VLD1d32TPseudoWB_register;
2041 case ARM::VLD1d64TPseudoWB_fixed:
return ARM::VLD1d64TPseudoWB_register;
2042 case ARM::VLD1d8QPseudoWB_fixed:
return ARM::VLD1d8QPseudoWB_register;
2043 case ARM::VLD1d16QPseudoWB_fixed:
return ARM::VLD1d16QPseudoWB_register;
2044 case ARM::VLD1d32QPseudoWB_fixed:
return ARM::VLD1d32QPseudoWB_register;
2045 case ARM::VLD1d64QPseudoWB_fixed:
return ARM::VLD1d64QPseudoWB_register;
2046 case ARM::VLD1DUPd8wb_fixed :
return ARM::VLD1DUPd8wb_register;
2047 case ARM::VLD1DUPd16wb_fixed :
return ARM::VLD1DUPd16wb_register;
2048 case ARM::VLD1DUPd32wb_fixed :
return ARM::VLD1DUPd32wb_register;
2049 case ARM::VLD1DUPq8wb_fixed :
return ARM::VLD1DUPq8wb_register;
2050 case ARM::VLD1DUPq16wb_fixed :
return ARM::VLD1DUPq16wb_register;
2051 case ARM::VLD1DUPq32wb_fixed :
return ARM::VLD1DUPq32wb_register;
2052 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return ARM::VLD2DUPq8OddPseudoWB_register;
2053 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return ARM::VLD2DUPq16OddPseudoWB_register;
2054 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return ARM::VLD2DUPq32OddPseudoWB_register;
2056 case ARM::VST1d8wb_fixed:
return ARM::VST1d8wb_register;
2057 case ARM::VST1d16wb_fixed:
return ARM::VST1d16wb_register;
2058 case ARM::VST1d32wb_fixed:
return ARM::VST1d32wb_register;
2059 case ARM::VST1d64wb_fixed:
return ARM::VST1d64wb_register;
2060 case ARM::VST1q8wb_fixed:
return ARM::VST1q8wb_register;
2061 case ARM::VST1q16wb_fixed:
return ARM::VST1q16wb_register;
2062 case ARM::VST1q32wb_fixed:
return ARM::VST1q32wb_register;
2063 case ARM::VST1q64wb_fixed:
return ARM::VST1q64wb_register;
2064 case ARM::VST1d8TPseudoWB_fixed:
return ARM::VST1d8TPseudoWB_register;
2065 case ARM::VST1d16TPseudoWB_fixed:
return ARM::VST1d16TPseudoWB_register;
2066 case ARM::VST1d32TPseudoWB_fixed:
return ARM::VST1d32TPseudoWB_register;
2067 case ARM::VST1d64TPseudoWB_fixed:
return ARM::VST1d64TPseudoWB_register;
2068 case ARM::VST1d8QPseudoWB_fixed:
return ARM::VST1d8QPseudoWB_register;
2069 case ARM::VST1d16QPseudoWB_fixed:
return ARM::VST1d16QPseudoWB_register;
2070 case ARM::VST1d32QPseudoWB_fixed:
return ARM::VST1d32QPseudoWB_register;
2071 case ARM::VST1d64QPseudoWB_fixed:
return ARM::VST1d64QPseudoWB_register;
2073 case ARM::VLD2d8wb_fixed:
return ARM::VLD2d8wb_register;
2074 case ARM::VLD2d16wb_fixed:
return ARM::VLD2d16wb_register;
2075 case ARM::VLD2d32wb_fixed:
return ARM::VLD2d32wb_register;
2076 case ARM::VLD2q8PseudoWB_fixed:
return ARM::VLD2q8PseudoWB_register;
2077 case ARM::VLD2q16PseudoWB_fixed:
return ARM::VLD2q16PseudoWB_register;
2078 case ARM::VLD2q32PseudoWB_fixed:
return ARM::VLD2q32PseudoWB_register;
2080 case ARM::VST2d8wb_fixed:
return ARM::VST2d8wb_register;
2081 case ARM::VST2d16wb_fixed:
return ARM::VST2d16wb_register;
2082 case ARM::VST2d32wb_fixed:
return ARM::VST2d32wb_register;
2083 case ARM::VST2q8PseudoWB_fixed:
return ARM::VST2q8PseudoWB_register;
2084 case ARM::VST2q16PseudoWB_fixed:
return ARM::VST2q16PseudoWB_register;
2085 case ARM::VST2q32PseudoWB_fixed:
return ARM::VST2q32PseudoWB_register;
2087 case ARM::VLD2DUPd8wb_fixed:
return ARM::VLD2DUPd8wb_register;
2088 case ARM::VLD2DUPd16wb_fixed:
return ARM::VLD2DUPd16wb_register;
2089 case ARM::VLD2DUPd32wb_fixed:
return ARM::VLD2DUPd32wb_register;
2098 auto C = dyn_cast<ConstantSDNode>(Inc);
2102 void ARMDAGToDAGISel::SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2106 assert(Subtarget->hasNEON());
2107 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLD NumVecs out-of-range");
2111 bool IsIntrinsic = !isUpdating;
2113 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2114 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2118 EVT VT =
N->getValueType(0);
2120 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2148 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2153 std::vector<EVT> ResTys;
2154 ResTys.push_back(ResTy);
2165 if (is64BitVector || NumVecs <= 2) {
2166 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2168 Ops.push_back(MemAddr);
2169 Ops.push_back(
Align);
2171 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2182 Ops.push_back(Reg0);
2184 Ops.push_back(Pred);
2185 Ops.push_back(Reg0);
2186 Ops.push_back(Chain);
2187 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2197 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2198 const SDValue OpsA[] = { MemAddr,
Align, Reg0, ImplDef, Pred, Reg0, Chain };
2204 Ops.push_back(
SDValue(VLdA, 1));
2205 Ops.push_back(
Align);
2207 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2209 "only constant post-increment update allowed for VLD3/4");
2211 Ops.push_back(Reg0);
2213 Ops.push_back(
SDValue(VLdA, 0));
2214 Ops.push_back(Pred);
2215 Ops.push_back(Reg0);
2216 Ops.push_back(Chain);
2217 VLd = CurDAG->getMachineNode(QOpcodes1[
OpcodeIndex], dl, ResTys, Ops);
2222 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {
MemOp});
2225 ReplaceNode(
N, VLd);
2231 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2232 ARM::qsub_3 == ARM::qsub_0 + 3,
2233 "Unexpected subreg numbering");
2234 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
2235 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2237 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2241 CurDAG->RemoveDeadNode(
N);
2244 void ARMDAGToDAGISel::SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2248 assert(Subtarget->hasNEON());
2249 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VST NumVecs out-of-range");
2253 bool IsIntrinsic = !isUpdating;
2255 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2256 unsigned Vec0Idx = 3;
2257 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2263 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2265 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2289 std::vector<EVT> ResTys;
2299 if (is64BitVector || NumVecs <= 2) {
2302 SrcReg =
N->getOperand(Vec0Idx);
2303 }
else if (is64BitVector) {
2305 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2306 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2314 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
2315 :
N->getOperand(Vec0Idx + 3);
2320 SDValue Q0 =
N->getOperand(Vec0Idx);
2321 SDValue Q1 =
N->getOperand(Vec0Idx + 1);
2325 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2327 Ops.push_back(MemAddr);
2328 Ops.push_back(
Align);
2330 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2342 Ops.push_back(Reg0);
2344 Ops.push_back(SrcReg);
2345 Ops.push_back(Pred);
2346 Ops.push_back(Reg0);
2347 Ops.push_back(Chain);
2348 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2351 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {
MemOp});
2353 ReplaceNode(
N, VSt);
2361 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2362 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2365 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2366 :
N->getOperand(Vec0Idx + 3);
2371 const SDValue OpsA[] = { MemAddr,
Align, Reg0, RegSeq, Pred, Reg0, Chain };
2375 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {
MemOp});
2379 Ops.push_back(
SDValue(VStA, 0));
2380 Ops.push_back(
Align);
2382 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2384 "only constant post-increment update allowed for VST3/4");
2386 Ops.push_back(Reg0);
2388 Ops.push_back(RegSeq);
2389 Ops.push_back(Pred);
2390 Ops.push_back(Reg0);
2391 Ops.push_back(Chain);
2394 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {
MemOp});
2395 ReplaceNode(
N, VStB);
2398 void ARMDAGToDAGISel::SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
2402 assert(Subtarget->hasNEON());
2403 assert(NumVecs >=2 && NumVecs <= 4 &&
"VLDSTLane NumVecs out-of-range");
2407 bool IsIntrinsic = !isUpdating;
2409 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2410 unsigned Vec0Idx = 3;
2411 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2418 cast<ConstantSDNode>(
N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2419 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2426 if (Alignment > NumBytes)
2428 if (Alignment < 8 && Alignment < NumBytes)
2455 std::vector<EVT> ResTys;
2457 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2471 Ops.push_back(MemAddr);
2472 Ops.push_back(
Align);
2474 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2477 Ops.push_back(IsImmUpdate ? Reg0 : Inc);
2481 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2482 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2491 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2492 :
N->getOperand(Vec0Idx + 3);
2498 Ops.push_back(SuperReg);
2499 Ops.push_back(getI32Imm(Lane, dl));
2500 Ops.push_back(Pred);
2501 Ops.push_back(Reg0);
2502 Ops.push_back(Chain);
2504 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2506 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2507 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {
MemOp});
2509 ReplaceNode(
N, VLdLn);
2515 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2516 ARM::qsub_3 == ARM::qsub_0 + 3,
2517 "Unexpected subreg numbering");
2518 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2519 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2521 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2525 CurDAG->RemoveDeadNode(
N);
2528 template <
typename SDValueVector>
2529 void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2532 Ops.push_back(PredicateMask);
2533 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2536 template <
typename SDValueVector>
2537 void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2541 Ops.push_back(PredicateMask);
2542 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2543 Ops.push_back(Inactive);
2546 template <
typename SDValueVector>
2547 void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc) {
2549 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2550 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2553 template <
typename SDValueVector>
2554 void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2557 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2558 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2560 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, InactiveTy), 0));
2563 void ARMDAGToDAGISel::SelectMVE_WB(
SDNode *
N,
const uint16_t *Opcodes,
2569 switch (
N->getValueType(1).getVectorElementType().getSizeInBits()) {
2571 Opcode = Opcodes[0];
2574 Opcode = Opcodes[1];
2580 Ops.push_back(
N->getOperand(2));
2582 int32_t ImmValue = cast<ConstantSDNode>(
N->getOperand(3))->getZExtValue();
2583 Ops.push_back(getI32Imm(ImmValue, Loc));
2586 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2588 AddEmptyMVEPredicateToOps(Ops, Loc);
2590 Ops.push_back(
N->getOperand(0));
2593 VTs.push_back(
N->getValueType(1));
2594 VTs.push_back(
N->getValueType(0));
2595 VTs.push_back(
N->getValueType(2));
2601 transferMemOperands(
N, New);
2602 CurDAG->RemoveDeadNode(
N);
2607 bool HasSaturationOperand) {
2612 Ops.push_back(
N->getOperand(1));
2613 Ops.push_back(
N->getOperand(2));
2617 int32_t ImmValue = cast<ConstantSDNode>(
N->getOperand(3))->getZExtValue();
2618 Ops.push_back(getI32Imm(ImmValue, Loc));
2620 Ops.push_back(
N->getOperand(3));
2624 if (HasSaturationOperand) {
2625 int32_t SatOp = cast<ConstantSDNode>(
N->getOperand(4))->getZExtValue();
2626 int SatBit = (SatOp == 64 ? 0 : 1);
2627 Ops.push_back(getI32Imm(SatBit, Loc));
2632 Ops.push_back(
getAL(CurDAG, Loc));
2633 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2635 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
makeArrayRef(Ops));
2638 void ARMDAGToDAGISel::SelectMVE_VADCSBC(
SDNode *
N,
uint16_t OpcodeWithCarry,
2640 bool Add,
bool Predicated) {
2645 unsigned FirstInputOp = Predicated ? 2 : 1;
2648 Ops.push_back(
N->getOperand(FirstInputOp));
2649 Ops.push_back(
N->getOperand(FirstInputOp + 1));
2650 SDValue CarryIn =
N->getOperand(FirstInputOp + 2);
2651 ConstantSDNode *CarryInConstant = dyn_cast<ConstantSDNode>(CarryIn);
2654 if (CarryInConstant &&
2655 (CarryInConstant->
getZExtValue() & CarryMask) == CarryExpected) {
2656 Opcode = OpcodeWithNoCarry;
2658 Ops.push_back(CarryIn);
2659 Opcode = OpcodeWithCarry;
2663 AddMVEPredicateToOps(Ops, Loc,
2664 N->getOperand(FirstInputOp + 3),
2665 N->getOperand(FirstInputOp - 1));
2667 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2669 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
makeArrayRef(Ops));
2672 void ARMDAGToDAGISel::SelectMVE_VSHLC(
SDNode *
N,
bool Predicated) {
2678 Ops.push_back(
N->getOperand(1));
2679 Ops.push_back(
N->getOperand(2));
2680 int32_t ImmValue = cast<ConstantSDNode>(
N->getOperand(3))->getZExtValue();
2681 Ops.push_back(getI32Imm(ImmValue, Loc));
2684 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2686 AddEmptyMVEPredicateToOps(Ops, Loc);
2688 CurDAG->SelectNodeTo(
N, ARM::MVE_VSHLC,
N->getVTList(),
makeArrayRef(Ops));
2692 assert(isa<ConstantSDNode>(SDVal) &&
"expected a compile-time constant");
2699 void ARMDAGToDAGISel::SelectBaseMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
2702 size_t Stride,
size_t TySize) {
2703 assert(TySize < Stride &&
"Invalid TySize");
2709 "Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist");
2711 "Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist");
2714 auto OpIsZero = [
N](
size_t OpNo) {
2715 if (
ConstantSDNode *OpConst = dyn_cast<ConstantSDNode>(
N->getOperand(OpNo)))
2716 if (OpConst->getZExtValue() == 0)
2723 bool IsAccum = !(OpIsZero(4) && OpIsZero(5));
2725 const uint16_t *Opcodes = IsUnsigned ? OpcodesU : OpcodesS;
2727 Opcodes += 4 * Stride;
2729 Opcodes += 2 * Stride;
2738 Ops.push_back(
N->getOperand(4));
2739 Ops.push_back(
N->getOperand(5));
2742 Ops.push_back(
N->getOperand(6));
2743 Ops.push_back(
N->getOperand(7));
2746 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(8));
2748 AddEmptyMVEPredicateToOps(Ops, Loc);
2750 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
makeArrayRef(Ops));
2753 void ARMDAGToDAGISel::SelectMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
2756 EVT VecTy =
N->getOperand(6).getValueType();
2769 SelectBaseMVE_VMLLDAV(
N, Predicated, OpcodesS, OpcodesU, 2, SizeIndex);
2772 void ARMDAGToDAGISel::SelectMVE_VRMLLDAVH(
SDNode *
N,
bool Predicated,
2776 N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() ==
2778 "bad vector element size");
2779 SelectBaseMVE_VMLLDAV(
N, Predicated, OpcodesS, OpcodesU, 1, 0);
2782 void ARMDAGToDAGISel::SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
2784 bool HasWriteback) {
2785 EVT VT =
N->getValueType(0);
2791 OurOpcodes = Opcodes[0];
2794 OurOpcodes = Opcodes[1];
2797 OurOpcodes = Opcodes[2];
2805 unsigned PtrOperand = HasWriteback ? 1 : 2;
2808 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, DataTy), 0);
2811 for (
unsigned Stage = 0; Stage < NumVecs - 1; ++Stage) {
2812 SDValue Ops[] = {
Data,
N->getOperand(PtrOperand), Chain};
2814 CurDAG->getMachineNode(OurOpcodes[Stage], Loc, ResultTys, Ops);
2822 SDValue Ops[] = {
Data,
N->getOperand(PtrOperand), Chain};
2824 CurDAG->getMachineNode(OurOpcodes[NumVecs - 1], Loc, ResultTys, Ops);
2828 for (
i = 0;
i < NumVecs;
i++)
2830 CurDAG->getTargetExtractSubreg(ARM::qsub_0 +
i, Loc, VT,
2835 CurDAG->RemoveDeadNode(
N);
2838 void ARMDAGToDAGISel::SelectMVE_VxDUP(
SDNode *
N,
const uint16_t *Opcodes,
2839 bool Wrapping,
bool Predicated) {
2840 EVT VT =
N->getValueType(0);
2846 Opcode = Opcodes[0];
2849 Opcode = Opcodes[1];
2852 Opcode = Opcodes[2];
2863 Inactive =
N->getOperand(OpIdx++);
2865 Ops.push_back(
N->getOperand(OpIdx++));
2867 Ops.push_back(
N->getOperand(OpIdx++));
2869 SDValue ImmOp =
N->getOperand(OpIdx++);
2870 int ImmValue = cast<ConstantSDNode>(ImmOp)->getZExtValue();
2871 Ops.push_back(getI32Imm(ImmValue, Loc));
2874 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(OpIdx), Inactive);
2876 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2878 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
makeArrayRef(Ops));
2882 size_t NumExtraOps,
bool HasAccum) {
2883 bool IsBigEndian = CurDAG->getDataLayout().isBigEndian();
2890 SDValue ImmCorpoc =
N->getOperand(OpIdx++);
2891 uint32_t ImmCoprocVal = cast<ConstantSDNode>(ImmCorpoc)->getZExtValue();
2892 Ops.push_back(getI32Imm(ImmCoprocVal, Loc));
2897 SDValue AccLo =
N->getOperand(OpIdx++);
2898 SDValue AccHi =
N->getOperand(OpIdx++);
2905 for (
size_t I = 0;
I < NumExtraOps;
I++)
2906 Ops.push_back(
N->getOperand(OpIdx++));
2910 uint32_t ImmVal = cast<ConstantSDNode>(
Imm)->getZExtValue();
2911 Ops.push_back(getI32Imm(ImmVal, Loc));
2917 Ops.push_back(Pred);
2918 Ops.push_back(PredReg);
2929 uint16_t SubRegs[2] = {ARM::gsub_0, ARM::gsub_1};
2933 for (
size_t ResIdx = 0; ResIdx < 2; ResIdx++) {
2934 if (
SDValue(
N, ResIdx).use_empty())
2936 SDValue SubReg = CurDAG->getTargetExtractSubreg(SubRegs[ResIdx], Loc,
2941 CurDAG->RemoveDeadNode(
N);
2944 void ARMDAGToDAGISel::SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
2945 bool isUpdating,
unsigned NumVecs,
2949 assert(Subtarget->hasNEON());
2950 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLDDup NumVecs out-of-range");
2954 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2955 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2959 EVT VT =
N->getValueType(0);
2966 if (Alignment > NumBytes)
2968 if (Alignment < 8 && Alignment < NumBytes)
2997 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
3002 std::vector<EVT> ResTys;
3003 ResTys.push_back(ResTy);
3012 Ops.push_back(MemAddr);
3013 Ops.push_back(
Align);
3014 unsigned Opc = is64BitVector ? DOpcodes[
OpcodeIndex]
3023 Ops.push_back(Reg0);
3030 if (is64BitVector || NumVecs == 1) {
3032 }
else if (NumVecs == 2) {
3033 const SDValue OpsA[] = {MemAddr,
Align, Pred, Reg0, Chain};
3039 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
3040 const SDValue OpsA[] = {MemAddr,
Align, ImplDef, Pred, Reg0, Chain};
3043 Ops.push_back(
SDValue(VLdA, 0));
3047 Ops.push_back(Pred);
3048 Ops.push_back(Reg0);
3049 Ops.push_back(Chain);
3051 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
3055 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {
MemOp});
3062 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7,
"Unexpected subreg numbering");
3063 unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
3064 for (
unsigned Vec = 0; Vec != NumVecs; ++Vec) {
3066 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
3072 CurDAG->RemoveDeadNode(
N);
3075 bool ARMDAGToDAGISel::tryInsertVectorElt(
SDNode *
N) {
3076 if (!Subtarget->hasMVEIntegerOps())
3095 if (Lane2 % 2 != 0 || Lane1 != Lane2 + 1)
3122 ExtractLane1 == ExtractLane2 + 1) {
3123 SDValue NewExt = CurDAG->getTargetExtractSubreg(
3125 SDValue NewIns = CurDAG->getTargetInsertSubreg(
3126 ARM::ssub_0 + Lane2 / 2, dl, VT, Ins2.
getOperand(0),
3128 ReplaceUses(Ins1, NewIns);
3135 SDValue Inp1 = CurDAG->getTargetExtractSubreg(
3137 SDValue Inp2 = CurDAG->getTargetExtractSubreg(
3139 if (ExtractLane1 % 2 != 0)
3140 Inp1 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl,
MVT::f32, Inp1), 0);
3141 if (ExtractLane2 % 2 != 0)
3142 Inp2 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl,
MVT::f32, Inp2), 0);
3143 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl,
MVT::f32, Inp2, Inp1);
3145 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl,
MVT::v4f32,
3147 ReplaceUses(Ins1, NewIns);
3155 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl,
MVT::f32, Val2, Val1);
3157 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl,
MVT::v4f32,
3159 ReplaceUses(Ins1, NewIns);
3166 bool ARMDAGToDAGISel::transformFixedFloatingPointConversion(
SDNode *
N,
3169 bool FixedToFloat) {
3170 auto Type =
N->getValueType(0);
3172 if (ScalarBits > 32)
3178 if (ScalarBits == 16 && !FMulFlags.
hasNoInfs() && IsUnsigned)
3203 if (!isa<ConstantSDNode>(ImmNode.
getOperand(0)))
3227 if (!ImmAPF.getExactInverse(&ToConvert))
3230 APSInt Converted(64,
false);
3234 if (!IsExact || !Converted.isPowerOf2())
3237 unsigned FracBits = Converted.logBase2();
3238 if (FracBits > ScalarBits)
3243 AddEmptyMVEPredicateToOps(Ops,
SDLoc(
N),
Type);
3245 unsigned int Opcode;
3246 switch (ScalarBits) {
3249 Opcode = IsUnsigned ? ARM::MVE_VCVTf16u16_fix : ARM::MVE_VCVTf16s16_fix;
3251 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3255 Opcode = IsUnsigned ? ARM::MVE_VCVTf32u32_fix : ARM::MVE_VCVTf32s32_fix;
3257 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3264 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
SDLoc(
N),
Type, Ops));
3268 bool ARMDAGToDAGISel::tryFP_TO_INT(
SDNode *
N,
SDLoc dl) {
3270 if (!Subtarget->hasMVEFloatOps())
3273 if (!
Type.isVector())
3279 SDNode *Node =
N->getOperand(0).getNode();
3284 if (Node->getOperand(0) != Node->getOperand(1))
3289 if (ScalarBits == 16 && !Flags.
hasNoInfs() && IsUnsigned)
3293 switch (ScalarBits) {
3295 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3298 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3302 CurDAG->getConstant(1, dl,
MVT::i32)};
3303 AddEmptyMVEPredicateToOps(Ops, dl,
Type);
3305 ReplaceNode(
N, CurDAG->getMachineNode(Opcode, dl,
Type, Ops));
3312 return transformFixedFloatingPointConversion(
N, Node, IsUnsigned,
false);
3315 bool ARMDAGToDAGISel::tryFMULFixed(
SDNode *
N,
SDLoc dl) {
3317 if (!Subtarget->hasMVEFloatOps())
3319 auto Type =
N->getValueType(0);
3320 if (!
Type.isVector())
3323 auto LHS =
N->getOperand(0);
3327 return transformFixedFloatingPointConversion(
3331 bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(
SDNode *
N,
bool isSigned) {
3332 if (!Subtarget->hasV6T2Ops())
3335 unsigned Opc = isSigned
3336 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
3337 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
3341 unsigned And_imm = 0;
3346 if (And_imm & (And_imm + 1))
3349 unsigned Srl_imm = 0;
3352 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3357 And_imm &= -1U >> Srl_imm;
3361 unsigned LSB = Srl_imm;
3365 if ((LSB +
Width + 1) ==
N->getValueType(0).getSizeInBits()) {
3367 if (Subtarget->isThumb()) {
3368 Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
3369 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3370 CurDAG->getTargetConstant(LSB, dl,
MVT::i32),
3371 getAL(CurDAG, dl), Reg0, Reg0 };
3372 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3381 SDValue Ops[] = {
N->getOperand(0).getOperand(0), ShOpc,
3382 getAL(CurDAG, dl), Reg0, Reg0 };
3383 CurDAG->SelectNodeTo(
N, ARM::MOVsi,
MVT::i32, Ops);
3387 assert(LSB +
Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3388 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3389 CurDAG->getTargetConstant(LSB, dl,
MVT::i32),
3391 getAL(CurDAG, dl), Reg0 };
3392 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3400 unsigned Shl_imm = 0;
3402 assert(Shl_imm > 0 && Shl_imm < 32 &&
"bad amount in shift node!");
3403 unsigned Srl_imm = 0;
3405 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3407 unsigned Width = 32 - Srl_imm - 1;
3408 int LSB = Srl_imm - Shl_imm;
3412 assert(LSB +
Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3413 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3414 CurDAG->getTargetConstant(LSB, dl,
MVT::i32),
3416 getAL(CurDAG, dl), Reg0 };
3417 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3425 unsigned Srl_imm = 0;
3429 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3432 unsigned Width = MSB - LSB;
3434 assert(Srl_imm +
Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3435 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3436 CurDAG->getTargetConstant(Srl_imm, dl,
MVT::i32),
3438 getAL(CurDAG, dl), Reg0 };
3439 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3445 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
3451 if (LSB +
Width > 32)
3455 assert(LSB +
Width <= 32 &&
"Shouldn't create an invalid ubfx");
3456 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3457 CurDAG->getTargetConstant(LSB, dl,
MVT::i32),
3459 getAL(CurDAG, dl), Reg0 };
3460 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3477 bool ARMDAGToDAGISel::tryABSOp(
SDNode *
N){
3478 SDValue SUBSrc0 =
N->getOperand(0);
3479 SDValue SUBSrc1 =
N->getOperand(1);
3480 EVT VT =
N->getValueType(0);
3496 if (XORSrc1 == SUBSrc1 && XORSrc0 == SRASrc0 && XType.
isInteger() &&
3497 SRAConstant !=
nullptr && Size == SRAConstant->
getZExtValue()) {
3499 CurDAG->SelectNodeTo(
N, Opcode, VT, XORSrc0);
3507 void ARMDAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
3509 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
3511 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_8 : ARM::CMP_SWAP_8;
3513 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_16 : ARM::CMP_SWAP_16;
3515 Opcode = ARM::CMP_SWAP_32;
3519 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3521 SDNode *CmpSwap = CurDAG->getMachineNode(
3526 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
3530 CurDAG->RemoveDeadNode(
N);
3535 unsigned FirstOne = A.getBitWidth() - A.countLeadingZeros() - 1;
3536 unsigned LastOne = A.countTrailingZeros();
3537 if (A.countPopulation() != (FirstOne - LastOne + 1))
3539 return std::make_pair(FirstOne, LastOne);
3542 void ARMDAGToDAGISel::SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI) {
3544 SwitchEQNEToPLMI =
false;
3546 if (!Subtarget->isThumb())
3553 if (!
And->hasOneUse())
3557 if (!isa<ConstantSDNode>(Zero) || !cast<ConstantSDNode>(Zero)->
isZero() ||
3561 auto C = dyn_cast<ConstantSDNode>(
And.getOperand(1));
3575 Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
3578 CurDAG->getRegister(0,
MVT::i32) };
3579 return CurDAG->getMachineNode(Opc, dl,
MVT::i32, Ops);
3584 return CurDAG->getMachineNode(Opc, dl,
MVT::i32, Ops);
3588 if (Range->second == 0) {
3590 NewN = EmitShift(ARM::tLSLri,
X, 31 - Range->first);
3591 ReplaceNode(
And.getNode(), NewN);
3592 }
else if (Range->first == 31) {
3594 NewN = EmitShift(ARM::tLSRri,
X, Range->second);
3595 ReplaceNode(
And.getNode(), NewN);
3596 }
else if (Range->first == Range->second) {
3599 NewN = EmitShift(ARM::tLSLri,
X, 31 - Range->first);
3600 ReplaceNode(
And.getNode(), NewN);
3602 SwitchEQNEToPLMI =
true;
3603 }
else if (!Subtarget->hasV6T2Ops()) {
3606 NewN = EmitShift(ARM::tLSLri,
X, 31 - Range->first);
3607 NewN = EmitShift(ARM::tLSRri,
SDValue(NewN, 0),
3608 Range->second + (31 - Range->first));
3609 ReplaceNode(
And.getNode(), NewN);
3617 if (
N->isMachineOpcode()) {
3622 switch (
N->getOpcode()) {
3644 cast<RegisterSDNode>(Ptr.
getOperand(1))->getReg() == ARM::SP &&
3647 CurDAG->getRegister(ARM::SP,
MVT::i32),
3648 CurDAG->getTargetConstant(RHSC, dl,
MVT::i32),
3653 CurDAG->getMachineNode(ARM::tSTRspi, dl,
MVT::Other, Ops);
3655 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3656 ReplaceNode(
N, ResNode);
3663 if (tryWriteRegister(
N))
3667 if (tryReadRegister(
N))
3672 if (tryInlineAsm(
N))
3682 unsigned Val = cast<ConstantSDNode>(
N)->getZExtValue();
3685 SDValue CPIdx = CurDAG->getTargetConstantPool(
3687 TLI->getPointerTy(CurDAG->getDataLayout()));
3690 if (Subtarget->isThumb()) {
3695 CurDAG->getEntryNode()
3702 CurDAG->getTargetConstant(0, dl,
MVT::i32),
3705 CurDAG->getEntryNode()
3718 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3720 ReplaceNode(
N, ResNode);
3729 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
3730 SDValue TFI = CurDAG->getTargetFrameIndex(
3731 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
3738 CurDAG->SelectNodeTo(
N, ARM::tADDframe,
MVT::i32, TFI,
3739 CurDAG->getTargetConstant(0, dl,
MVT::i32));
3742 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
3743 ARM::t2ADDri : ARM::ADDri);
3746 CurDAG->getRegister(0,
MVT::i32) };
3747 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3752 if (tryInsertVectorElt(
N))
3757 if (tryV6T2BitfieldExtractOp(
N,
false))
3762 if (tryV6T2BitfieldExtractOp(
N,
true))
3769 if (tryFP_TO_INT(
N, dl))
3773 if (tryFMULFixed(
N, dl))
3780 unsigned RHSV =
C->getZExtValue();
3783 unsigned ShImm =
Log2_32(RHSV-1);
3790 if (Subtarget->isThumb()) {
3791 SDValue Ops[] = { V, V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3792 CurDAG->SelectNodeTo(
N, ARM::t2ADDrs,
MVT::i32, Ops);
3795 SDValue Ops[] = { V, V, Reg0, ShImmOp,
getAL(CurDAG, dl), Reg0,
3797 CurDAG->SelectNodeTo(
N, ARM::ADDrsi,
MVT::i32, Ops);
3802 unsigned ShImm =
Log2_32(RHSV+1);
3809 if (Subtarget->isThumb()) {
3810 SDValue Ops[] = { V, V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3811 CurDAG->SelectNodeTo(
N, ARM::t2RSBrs,
MVT::i32, Ops);
3814 SDValue Ops[] = { V, V, Reg0, ShImmOp,
getAL(CurDAG, dl), Reg0,
3816 CurDAG->SelectNodeTo(
N, ARM::RSBrsi,
MVT::i32, Ops);
3824 if (tryV6T2BitfieldExtractOp(
N,
false))
3830 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
3831 if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
3838 bool PreferImmediateEncoding =
3839 Subtarget->hasThumb2() && (is_t2_so_imm(
Imm) || is_t2_so_imm_not(
Imm));
3840 if (!PreferImmediateEncoding &&
3846 CurDAG->getConstant(~N1C->getZExtValue(), dl,
MVT::i32);
3850 CurDAG->RepositionNode(
N->getIterator(), NewImm.
getNode());
3852 if (!Subtarget->hasThumb2()) {
3854 N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3856 ReplaceNode(
N, CurDAG->getMachineNode(ARM::tBIC, dl,
MVT::i32, Ops));
3859 SDValue Ops[] = {
N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3863 CurDAG->getMachineNode(ARM::t2BICrr, dl,
MVT::i32, Ops));
3874 EVT VT =
N->getValueType(0);
3877 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
3879 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
3882 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
3883 N1C = dyn_cast<ConstantSDNode>(N1);
3891 unsigned N1CVal = N1C->getZExtValue();
3893 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
3894 (N1CVal & 0xffffU) == 0xffffU &&
3895 (N2CVal & 0xffffU) == 0x0U) {
3896 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
3900 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
3908 unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL :
ARM::UMAAL;
3909 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
3910 N->getOperand(2),
N->getOperand(3),
3912 CurDAG->getRegister(0,
MVT::i32) };
3917 if (Subtarget->isThumb()) {
3918 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3919 N->getOperand(3),
getAL(CurDAG, dl),
3925 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3926 N->getOperand(3),
getAL(CurDAG, dl),
3928 CurDAG->getRegister(0,
MVT::i32) };
3929 ReplaceNode(
N, CurDAG->getMachineNode(
3930 Subtarget->hasV6Ops() ?
ARM::UMLAL : ARM::UMLALv5, dl,
3936 if (Subtarget->isThumb()) {
3937 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3938 N->getOperand(3),
getAL(CurDAG, dl),
3944 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3945 N->getOperand(3),
getAL(CurDAG, dl),
3947 CurDAG->getRegister(0,
MVT::i32) };
3948 ReplaceNode(
N, CurDAG->getMachineNode(
3949 Subtarget->hasV6Ops() ?
ARM::SMLAL : ARM::SMLALv5, dl,
3955 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
3964 if (Subtarget->isThumb())
3965 assert(Subtarget->hasThumb2() &&
3966 "This pattern should not be generated for Thumb");
3968 SDValue SmulLoHi =
N->getOperand(1);
3970 auto *Zero = dyn_cast<ConstantSDNode>(Subc.
getOperand(0));
3972 if (!Zero || Zero->getZExtValue() != 0 ||
3974 N->getOperand(1) != SmulLoHi.
getValue(1) ||
3978 unsigned Opc = Subtarget->
isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
3980 N->getOperand(0),
getAL(CurDAG, dl),
3981 CurDAG->getRegister(0,
MVT::i32) };
3982 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl,
MVT::i32, Ops));
3986 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
3988 if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
3989 if (tryT2IndexedLoad(
N))
3991 }
else if (Subtarget->isThumb()) {
3992 if (tryT1IndexedLoad(
N))
3994 }
else if (tryARMIndexedLoad(
N))
4000 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4007 ReplaceUses(
N, New);
4008 CurDAG->RemoveDeadNode(
N);
4013 N->getOperand(1),
N->getOperand(2),
4015 ReplaceUses(
N, New);
4016 CurDAG->RemoveDeadNode(
N);
4020 SDValue Ops[] = {
N->getOperand(1),
4023 unsigned Opc = ARM::t2LoopEnd;
4025 ReplaceUses(
N, New);
4026 CurDAG->RemoveDeadNode(
N);
4033 const SDValue &Chain =
N->getOperand(0);
4035 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4036 if (RegOffset != CurDAG->getRegister(0,
MVT::i32)) {
4042 RegOffset = CurDAG->getRegister(0,
MVT::i32);
4044 SDValue Ops[] = {
Base, RegOffset, ImmOffset, Chain};
4045 SDNode *
New = CurDAG->getMachineNode(ARM::LOADDUAL, dl,
4051 transferMemOperands(
N, New);
4055 CurDAG->RemoveDeadNode(
N);
4062 const SDValue &Chain =
N->getOperand(0);
4064 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4065 if (RegOffset != CurDAG->getRegister(0,
MVT::i32)) {
4071 RegOffset = CurDAG->getRegister(0,
MVT::i32);
4077 transferMemOperands(
N, New);
4079 CurDAG->RemoveDeadNode(
N);
4083 SDValue Ops[] = {
N->getOperand(1),
4087 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4089 ReplaceUses(
N, Dec);
4090 CurDAG->RemoveDeadNode(
N);
4106 unsigned Opc = Subtarget->isThumb() ?
4107 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
4117 unsigned CC = (unsigned) cast<ConstantSDNode>(N2)->getZExtValue();
4122 uint64_t ID = cast<ConstantSDNode>(
Int->getOperand(1))->getZExtValue();
4125 if (
ID == Intrinsic::loop_decrement_reg) {
4128 cast<ConstantSDNode>(
Int.getOperand(3))->getZExtValue(), dl,
4133 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4136 ReplaceUses(
Int.getNode(), LoopDec);
4140 CurDAG->getMachineNode(ARM::t2LoopEnd, dl,
MVT::Other, EndArgs);
4142 ReplaceUses(
N, LoopEnd);
4143 CurDAG->RemoveDeadNode(
N);
4144 CurDAG->RemoveDeadNode(InFlag.
getNode());
4145 CurDAG->RemoveDeadNode(
Int.getNode());
4150 bool SwitchEQNEToPLMI;
4151 SelectCMPZ(InFlag.
getNode(), SwitchEQNEToPLMI);
4152 InFlag =
N->getOperand(4);
4154 if (SwitchEQNEToPLMI) {
4168 SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
4172 if (
N->getNumValues() == 2) {
4174 ReplaceUses(
SDValue(
N, 1), InFlag);
4178 CurDAG->RemoveDeadNode(
N);
4188 auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(1).getNode());
4189 if (
C &&
C->getSExtValue() < 0 && Subtarget->isThumb()) {
4190 int64_t Addend = -
C->getSExtValue();
4197 if (Addend < 1<<8) {
4201 CurDAG->getRegister(0,
MVT::i32) };
4202 Add = CurDAG->getMachineNode(ARM::t2ADDri, dl,
MVT::i32, Ops);
4204 unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
4206 CurDAG->getTargetConstant(Addend, dl,
MVT::i32),
4208 Add = CurDAG->getMachineNode(Opc, dl,
MVT::i32, Ops);
4224 bool SwitchEQNEToPLMI;
4225 SelectCMPZ(InFlag.
getNode(), SwitchEQNEToPLMI);
4227 if (SwitchEQNEToPLMI) {
4242 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), NewARMcc,
4243 N->getOperand(3),
N->getOperand(4)};
4254 EVT VT =
N->getValueType(0);
4257 case MVT::v8i8: Opc = ARM::VZIPd8;
break;
4271 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg };
4272 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4277 EVT VT =
N->getValueType(0);
4280 case MVT::v8i8: Opc = ARM::VUZPd8;
break;
4294 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg };
4295 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4300 EVT VT =
N->getValueType(0);
4303 case MVT::v8i8: Opc = ARM::VTRNd8;
break;
4316 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg };
4317 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4321 EVT VecVT =
N->getValueType(0);
4325 assert(NumElts == 2 &&
"unexpected type for BUILD_VECTOR");
4327 N, createDRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4333 N, createSRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4336 assert(NumElts == 4 &&
"unexpected type for BUILD_VECTOR");
4338 createQuadSRegsNode(VecVT,
N->getOperand(0),
N->getOperand(1),
4339 N->getOperand(2),
N->getOperand(3)));
4344 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
4346 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
4348 SelectVLDDup(
N,
false,
false, 1, DOpcodes, QOpcodes);
4353 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4355 SelectVLDDup(
N,
false,
false, 2, Opcodes);
4360 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
4361 ARM::VLD3DUPd16Pseudo,
4362 ARM::VLD3DUPd32Pseudo };
4363 SelectVLDDup(
N,
false,
false, 3, Opcodes);
4368 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
4369 ARM::VLD4DUPd16Pseudo,
4370 ARM::VLD4DUPd32Pseudo };
4371 SelectVLDDup(
N,
false,
false, 4, Opcodes);
4376 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
4377 ARM::VLD1DUPd16wb_fixed,
4378 ARM::VLD1DUPd32wb_fixed };
4379 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
4380 ARM::VLD1DUPq16wb_fixed,
4381 ARM::VLD1DUPq32wb_fixed };
4382 SelectVLDDup(
N,
false,
true, 1, DOpcodes, QOpcodes);
4387 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8wb_fixed,
4388 ARM::VLD2DUPd16wb_fixed,
4389 ARM::VLD2DUPd32wb_fixed,
4390 ARM::VLD1q64wb_fixed };
4391 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4392 ARM::VLD2DUPq16EvenPseudo,
4393 ARM::VLD2DUPq32EvenPseudo };
4394 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudoWB_fixed,
4395 ARM::VLD2DUPq16OddPseudoWB_fixed,
4396 ARM::VLD2DUPq32OddPseudoWB_fixed };
4397 SelectVLDDup(
N,
false,
true, 2, DOpcodes, QOpcodes0, QOpcodes1);
4402 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
4403 ARM::VLD3DUPd16Pseudo_UPD,
4404 ARM::VLD3DUPd32Pseudo_UPD,
4405 ARM::VLD1d64TPseudoWB_fixed };
4406 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4407 ARM::VLD3DUPq16EvenPseudo,
4408 ARM::VLD3DUPq32EvenPseudo };
4409 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo_UPD,
4410 ARM::VLD3DUPq16OddPseudo_UPD,
4411 ARM::VLD3DUPq32OddPseudo_UPD };
4412 SelectVLDDup(
N,
false,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4417 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
4418 ARM::VLD4DUPd16Pseudo_UPD,
4419 ARM::VLD4DUPd32Pseudo_UPD,
4420 ARM::VLD1d64QPseudoWB_fixed };
4421 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4422 ARM::VLD4DUPq16EvenPseudo,
4423 ARM::VLD4DUPq32EvenPseudo };
4424 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo_UPD,
4425 ARM::VLD4DUPq16OddPseudo_UPD,
4426 ARM::VLD4DUPq32OddPseudo_UPD };
4427 SelectVLDDup(
N,
false,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4432 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
4433 ARM::VLD1d16wb_fixed,
4434 ARM::VLD1d32wb_fixed,
4435 ARM::VLD1d64wb_fixed };
4436 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
4437 ARM::VLD1q16wb_fixed,
4438 ARM::VLD1q32wb_fixed,
4439 ARM::VLD1q64wb_fixed };
4440 SelectVLD(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4445 if (Subtarget->hasNEON()) {
4446 static const uint16_t DOpcodes[] = {
4447 ARM::VLD2d8wb_fixed, ARM::VLD2d16wb_fixed, ARM::VLD2d32wb_fixed,
4448 ARM::VLD1q64wb_fixed};
4449 static const uint16_t QOpcodes[] = {ARM::VLD2q8PseudoWB_fixed,
4450 ARM::VLD2q16PseudoWB_fixed,
4451 ARM::VLD2q32PseudoWB_fixed};
4452 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4454 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8,
4455 ARM::MVE_VLD21_8_wb};
4456 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
4457 ARM::MVE_VLD21_16_wb};
4458 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
4459 ARM::MVE_VLD21_32_wb};
4460 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4461 SelectMVE_VLD(
N, 2, Opcodes,
true);
4467 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
4468 ARM::VLD3d16Pseudo_UPD,
4469 ARM::VLD3d32Pseudo_UPD,
4470 ARM::VLD1d64TPseudoWB_fixed};
4471 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4472 ARM::VLD3q16Pseudo_UPD,
4473 ARM::VLD3q32Pseudo_UPD };
4474 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
4475 ARM::VLD3q16oddPseudo_UPD,
4476 ARM::VLD3q32oddPseudo_UPD };
4477 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4482 if (Subtarget->hasNEON()) {
4483 static const uint16_t DOpcodes[] = {
4484 ARM::VLD4d8Pseudo_UPD, ARM::VLD4d16Pseudo_UPD, ARM::VLD4d32Pseudo_UPD,
4485 ARM::VLD1d64QPseudoWB_fixed};
4486 static const uint16_t QOpcodes0[] = {ARM::VLD4q8Pseudo_UPD,
4487 ARM::VLD4q16Pseudo_UPD,
4488 ARM::VLD4q32Pseudo_UPD};
4489 static const uint16_t QOpcodes1[] = {ARM::VLD4q8oddPseudo_UPD,
4490 ARM::VLD4q16oddPseudo_UPD,
4491 ARM::VLD4q32oddPseudo_UPD};
4492 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4494 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
4496 ARM::MVE_VLD43_8_wb};
4497 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
4499 ARM::MVE_VLD43_16_wb};
4500 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
4502 ARM::MVE_VLD43_32_wb};
4503 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4504 SelectMVE_VLD(
N, 4, Opcodes,
true);
4510 if (Subtarget->hasNEON()) {
4511 static const uint16_t DOpcodes[] = {
4512 ARM::VLD1q8wb_fixed, ARM::VLD1q16wb_fixed, ARM::VLD1q32wb_fixed,
4513 ARM::VLD1q64wb_fixed};
4514 static const uint16_t QOpcodes[] = {
4515 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4516 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4517 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4524 if (Subtarget->hasNEON()) {
4525 static const uint16_t DOpcodes[] = {
4526 ARM::VLD1d8TPseudoWB_fixed, ARM::VLD1d16TPseudoWB_fixed,
4527 ARM::VLD1d32TPseudoWB_fixed, ARM::VLD1d64TPseudoWB_fixed};
4528 static const uint16_t QOpcodes0[] = {
4529 ARM::VLD1q8LowTPseudo_UPD, ARM::VLD1q16LowTPseudo_UPD,
4530 ARM::VLD1q32LowTPseudo_UPD, ARM::VLD1q64LowTPseudo_UPD};
4531 static const uint16_t QOpcodes1[] = {
4532 ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1q16HighTPseudo_UPD,
4533 ARM::VLD1q32HighTPseudo_UPD, ARM::VLD1q64HighTPseudo_UPD};
4534 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4541 if (Subtarget->hasNEON()) {
4542 static const uint16_t DOpcodes[] = {
4543 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4544 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4545 static const uint16_t QOpcodes0[] = {
4546 ARM::VLD1q8LowQPseudo_UPD, ARM::VLD1q16LowQPseudo_UPD,
4547 ARM::VLD1q32LowQPseudo_UPD, ARM::VLD1q64LowQPseudo_UPD};
4548 static const uint16_t QOpcodes1[] = {
4549 ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1q16HighQPseudo_UPD,
4550 ARM::VLD1q32HighQPseudo_UPD, ARM::VLD1q64HighQPseudo_UPD};
4551 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4558 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
4559 ARM::VLD2LNd16Pseudo_UPD,
4560 ARM::VLD2LNd32Pseudo_UPD };
4561 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
4562 ARM::VLD2LNq32Pseudo_UPD };
4563 SelectVLDSTLane(
N,
true,
true, 2, DOpcodes, QOpcodes);
4568 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
4569 ARM::VLD3LNd16Pseudo_UPD,
4570 ARM::VLD3LNd32Pseudo_UPD };
4571 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
4572 ARM::VLD3LNq32Pseudo_UPD };
4573 SelectVLDSTLane(
N,
true,
true, 3, DOpcodes, QOpcodes);
4578 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
4579 ARM::VLD4LNd16Pseudo_UPD,
4580 ARM::VLD4LNd32Pseudo_UPD };
4581 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
4582 ARM::VLD4LNq32Pseudo_UPD };
4583 SelectVLDSTLane(
N,
true,
true, 4, DOpcodes, QOpcodes);
4588 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
4589 ARM::VST1d16wb_fixed,
4590 ARM::VST1d32wb_fixed,
4591 ARM::VST1d64wb_fixed };
4592 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
4593 ARM::VST1q16wb_fixed,
4594 ARM::VST1q32wb_fixed,
4595 ARM::VST1q64wb_fixed };
4596 SelectVST(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4601 if (Subtarget->hasNEON()) {
4602 static const uint16_t DOpcodes[] = {
4603 ARM::VST2d8wb_fixed, ARM::VST2d16wb_fixed, ARM::VST2d32wb_fixed,
4604 ARM::VST1q64wb_fixed};
4605 static const uint16_t QOpcodes[] = {ARM::VST2q8PseudoWB_fixed,
4606 ARM::VST2q16PseudoWB_fixed,
4607 ARM::VST2q32PseudoWB_fixed};
4608 SelectVST(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4615 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
4616 ARM::VST3d16Pseudo_UPD,
4617 ARM::VST3d32Pseudo_UPD,
4618 ARM::VST1d64TPseudoWB_fixed};
4619 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
4620 ARM::VST3q16Pseudo_UPD,
4621 ARM::VST3q32Pseudo_UPD };
4622 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
4623 ARM::VST3q16oddPseudo_UPD,
4624 ARM::VST3q32oddPseudo_UPD };
4625 SelectVST(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4630 if (Subtarget->hasNEON()) {
4631 static const uint16_t DOpcodes[] = {
4632 ARM::VST4d8Pseudo_UPD, ARM::VST4d16Pseudo_UPD, ARM::VST4d32Pseudo_UPD,
4633 ARM::VST1d64QPseudoWB_fixed};
4634 static const uint16_t QOpcodes0[] = {ARM::VST4q8Pseudo_UPD,
4635 ARM::VST4q16Pseudo_UPD,
4636 ARM::VST4q32Pseudo_UPD};
4637 static const uint16_t QOpcodes1[] = {ARM::VST4q8oddPseudo_UPD,
4638 ARM::VST4q16oddPseudo_UPD,
4639 ARM::VST4q32oddPseudo_UPD};
4640 SelectVST(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4647 if (Subtarget->hasNEON()) {
4648 static const uint16_t DOpcodes[] = { ARM::VST1q8wb_fixed,
4649 ARM::VST1q16wb_fixed,
4650 ARM::VST1q32wb_fixed,
4651 ARM::VST1q64wb_fixed};
4652 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4653 ARM::VST1d16QPseudoWB_fixed,
4654 ARM::VST1d32QPseudoWB_fixed,
4655 ARM::VST1d64QPseudoWB_fixed };
4656 SelectVST(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4663 if (Subtarget->hasNEON()) {
4664 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudoWB_fixed,
4665 ARM::VST1d16TPseudoWB_fixed,
4666 ARM::VST1d32TPseudoWB_fixed,
4667 ARM::VST1d64TPseudoWB_fixed };
4668 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
4669 ARM::VST1q16LowTPseudo_UPD,
4670 ARM::VST1q32LowTPseudo_UPD,
4671 ARM::VST1q64LowTPseudo_UPD };
4672 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo_UPD,
4673 ARM::VST1q16HighTPseudo_UPD,
4674 ARM::VST1q32HighTPseudo_UPD,
4675 ARM::VST1q64HighTPseudo_UPD };
4676 SelectVST(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4683 if (Subtarget->hasNEON()) {
4684 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4685 ARM::VST1d16QPseudoWB_fixed,
4686 ARM::VST1d32QPseudoWB_fixed,
4687 ARM::VST1d64QPseudoWB_fixed };
4688 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
4689 ARM::VST1q16LowQPseudo_UPD,
4690 ARM::VST1q32LowQPseudo_UPD,
4691 ARM::VST1q64LowQPseudo_UPD };
4692 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo_UPD,
4693 ARM::VST1q16HighQPseudo_UPD,
4694 ARM::VST1q32HighQPseudo_UPD,
4695 ARM::VST1q64HighQPseudo_UPD };
4696 SelectVST(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4702 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
4703 ARM::VST2LNd16Pseudo_UPD,
4704 ARM::VST2LNd32Pseudo_UPD };
4705 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
4706 ARM::VST2LNq32Pseudo_UPD };
4707 SelectVLDSTLane(
N,
false,
true, 2, DOpcodes, QOpcodes);
4712 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
4713 ARM::VST3LNd16Pseudo_UPD,
4714 ARM::VST3LNd32Pseudo_UPD };
4715 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
4716 ARM::VST3LNq32Pseudo_UPD };
4717 SelectVLDSTLane(
N,
false,
true, 3, DOpcodes, QOpcodes);
4722 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
4723 ARM::VST4LNd16Pseudo_UPD,
4724 ARM::VST4LNd32Pseudo_UPD };
4725 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
4726 ARM::VST4LNq32Pseudo_UPD };
4727 SelectVLDSTLane(
N,
false,
true, 4, DOpcodes, QOpcodes);
4733 unsigned IntNo = cast<ConstantSDNode>(
N->getOperand(1))->getZExtValue();
4738 case Intrinsic::arm_mrrc:
4739 case Intrinsic::arm_mrrc2: {
4744 if (Subtarget->isThumb())
4745 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
4747 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
4750 Ops.push_back(getI32Imm(cast<ConstantSDNode>(
N->getOperand(2))->getZExtValue(), dl));
4751 Ops.push_back(getI32Imm(cast<ConstantSDNode>(
N->getOperand(3))->getZExtValue(), dl));
4752 Ops.push_back(getI32Imm(cast<ConstantSDNode>(
N->getOperand(4))->getZExtValue(), dl));
4757 if (Opc != ARM::MRRC2) {
4758 Ops.push_back(
getAL(CurDAG, dl));
4759 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
4762 Ops.push_back(Chain);
4767 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
4770 case Intrinsic::arm_ldaexd:
4771 case Intrinsic::arm_ldrexd: {
4774 SDValue MemAddr =
N->getOperand(2);
4775 bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
4777 bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
4778 unsigned NewOpc =
isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
4779 : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
4782 std::vector<EVT> ResTys;
4792 CurDAG->getRegister(0,
MVT::i32), Chain};
4793 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4796 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4806 CurDAG->getTargetConstant(ARM::gsub_0, dl,
MVT::i32);
4807 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4811 ReplaceUses(
SDValue(
N, 0), Result);
4819 CurDAG->getTargetConstant(ARM::gsub_1, dl,
MVT::i32);
4820 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4824 ReplaceUses(
SDValue(
N, 1), Result);
4826 ReplaceUses(
SDValue(
N, 2), OutChain);
4827 CurDAG->RemoveDeadNode(
N);
4830 case Intrinsic::arm_stlexd:
4831 case Intrinsic::arm_strexd: {
4836 SDValue MemAddr =
N->getOperand(4);
4842 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
4846 Ops.push_back(Val0);
4847 Ops.push_back(Val1);
4851 Ops.push_back(MemAddr);
4852 Ops.push_back(
getAL(CurDAG, dl));
4853 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
4854 Ops.push_back(Chain);
4856 bool IsRelease = IntNo == Intrinsic::arm_stlexd;
4857 unsigned NewOpc =
isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
4858 : (IsRelease ? ARM::STLEXD : ARM::STREXD);
4860 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4863 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4869 case Intrinsic::arm_neon_vld1: {
4870 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
4871 ARM::VLD1d32, ARM::VLD1d64 };
4872 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4873 ARM::VLD1q32, ARM::VLD1q64};
4874 SelectVLD(
N,
false, 1, DOpcodes, QOpcodes,
nullptr);
4878 case Intrinsic::arm_neon_vld1x2: {
4879 static const uint16_t DOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4880 ARM::VLD1q32, ARM::VLD1q64 };
4881 static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudo,
4882 ARM::VLD1d16QPseudo,
4883 ARM::VLD1d32QPseudo,
4884 ARM::VLD1d64QPseudo };
4885 SelectVLD(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
4889 case Intrinsic::arm_neon_vld1x3: {
4890 static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudo,
4891 ARM::VLD1d16TPseudo,
4892 ARM::VLD1d32TPseudo,
4893 ARM::VLD1d64TPseudo };
4894 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
4895 ARM::VLD1q16LowTPseudo_UPD,
4896 ARM::VLD1q32LowTPseudo_UPD,
4897 ARM::VLD1q64LowTPseudo_UPD };
4898 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo,
4899 ARM::VLD1q16HighTPseudo,
4900 ARM::VLD1q32HighTPseudo,
4901 ARM::VLD1q64HighTPseudo };
4902 SelectVLD(
N,
false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4906 case Intrinsic::arm_neon_vld1x4: {
4907 static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudo,
4908 ARM::VLD1d16QPseudo,
4909 ARM::VLD1d32QPseudo,
4910 ARM::VLD1d64QPseudo };
4911 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
4912 ARM::VLD1q16LowQPseudo_UPD,
4913 ARM::VLD1q32LowQPseudo_UPD,
4914 ARM::VLD1q64LowQPseudo_UPD };
4915 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo,
4916 ARM::VLD1q16HighQPseudo,
4917 ARM::VLD1q32HighQPseudo,
4918 ARM::VLD1q64HighQPseudo };
4919 SelectVLD(
N,
false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4923 case Intrinsic::arm_neon_vld2: {
4924 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
4925 ARM::VLD2d32, ARM::VLD1q64 };
4926 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
4927 ARM::VLD2q32Pseudo };
4928 SelectVLD(
N,
false, 2, DOpcodes, QOpcodes,
nullptr);
4932 case Intrinsic::arm_neon_vld3: