32#include "llvm/IR/IntrinsicsARM.h"
42#define DEBUG_TYPE "arm-isel"
43#define PASS_NAME "ARM Instruction Selection"
47 cl::desc(
"Disable isel of shifter-op"),
64 ARMDAGToDAGISel() =
delete;
80 inline SDValue getI32Imm(
unsigned Imm,
const SDLoc &dl) {
81 return CurDAG->getTargetConstant(Imm, dl,
MVT::i32);
90 bool hasNoVMLxHazardUse(
SDNode *
N)
const;
91 bool isShifterOpProfitable(
const SDValue &Shift,
95 bool CheckProfitability =
true);
97 SDValue &
B,
bool CheckProfitability =
true);
101 return SelectRegShifterOperand(
N,
A,
B,
C,
false);
105 return SelectImmShifterOperand(
N,
A,
B,
false);
110 return SelectImmShifterOperand(
N,
A,
B,
false);
156 template <
unsigned Shift>
161 template <
unsigned Shift>
167 template <
unsigned Shift>
171 template <
unsigned Shift>
177 template<
int Min,
int Max>
180 inline bool is_so_imm(
unsigned Imm)
const {
184 inline bool is_so_imm_not(
unsigned Imm)
const {
188 inline bool is_t2_so_imm(
unsigned Imm)
const {
192 inline bool is_t2_so_imm_not(
unsigned Imm)
const {
197#include "ARMGenDAGISel.inc"
203 bool tryARMIndexedLoad(
SDNode *
N);
204 bool tryT1IndexedLoad(
SDNode *
N);
205 bool tryT2IndexedLoad(
SDNode *
N);
206 bool tryMVEIndexedLoad(
SDNode *
N);
217 void SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
225 void SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
232 void SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
233 unsigned NumVecs,
const uint16_t *DOpcodes,
237 template <
typename SDValueVector>
238 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
240 template <
typename SDValueVector>
241 void AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
244 template <
typename SDValueVector>
245 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc);
246 template <
typename SDValueVector>
247 void AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
EVT InactiveTy);
250 void SelectMVE_WB(
SDNode *
N,
const uint16_t *Opcodes,
bool Predicated);
254 bool HasSaturationOperand);
258 uint16_t OpcodeWithNoCarry,
bool Add,
bool Predicated);
262 void SelectMVE_VSHLC(
SDNode *
N,
bool Predicated);
272 void SelectBaseMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
274 size_t Stride,
size_t TySize);
278 void SelectMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
const uint16_t *OpcodesS,
282 void SelectMVE_VRMLLDAVH(
SDNode *
N,
bool Predicated,
const uint16_t *OpcodesS,
290 void SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
291 const uint16_t *
const *Opcodes,
bool HasWriteback);
296 bool Wrapping,
bool Predicated);
310 void SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
bool isUpdating,
311 unsigned NumVecs,
const uint16_t *DOpcodes,
312 const uint16_t *QOpcodes0 =
nullptr,
313 const uint16_t *QOpcodes1 =
nullptr);
318 bool tryInsertVectorElt(
SDNode *
N);
323 bool tryReadRegister(
SDNode *
N);
324 bool tryWriteRegister(
SDNode *
N);
328 void SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI);
330 void SelectCMP_SWAP(
SDNode *
N);
335 std::vector<SDValue> &OutOps)
override;
357 bool canExtractShiftFromMul(
const SDValue &
N,
unsigned MaxShift,
358 unsigned &PowerOfTwo,
SDValue &NewMulConst)
const;
366char ARMDAGToDAGISel::ID = 0;
374 Imm = cast<ConstantSDNode>(
N)->getZExtValue();
390 return N->getOpcode() == Opc &&
399 int RangeMin,
int RangeMax,
400 int &ScaledConstant) {
401 assert(Scale > 0 &&
"Invalid scale!");
408 ScaledConstant = (int)
C->getZExtValue();
409 if ((ScaledConstant % Scale) != 0)
412 ScaledConstant /= Scale;
413 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
416void ARMDAGToDAGISel::PreprocessISelDAG() {
417 if (!Subtarget->hasV6T2Ops())
420 bool isThumb2 = Subtarget->isThumb();
437 unsigned And_imm = 0;
447 if (TZ != 1 && TZ != 2)
459 if (And_imm & (And_imm + 1))
464 unsigned Srl_imm = 0;
475 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
478 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
479 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
486 CurDAG->getConstant(Srl_imm + TZ,
SDLoc(Srl),
493 CurDAG->UpdateNodeOperands(&
N, N0, N1);
500bool ARMDAGToDAGISel::hasNoVMLxHazardUse(
SDNode *
N)
const {
504 if (!Subtarget->hasVMLxHazards())
513 if (
Use->isMachineOpcode()) {
515 CurDAG->getSubtarget().getInstrInfo());
521 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
536 return TII->isFpMLxInstruction(Opcode);
542bool ARMDAGToDAGISel::isShifterOpProfitable(
const SDValue &Shift,
551 (ShAmt == 2 || (Subtarget->
isSwift() && ShAmt == 1));
554bool ARMDAGToDAGISel::canExtractShiftFromMul(
const SDValue &
N,
556 unsigned &PowerOfTwo,
563 if (!
N.hasOneUse())
return false;
566 if (!MulConst)
return false;
569 if (!MulConst->
hasOneUse())
return false;
571 if (MulConstVal == 0)
return false;
574 PowerOfTwo = MaxShift;
575 while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
577 if (PowerOfTwo == 0)
return false;
581 unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
582 NewMulConst = CurDAG->getConstant(NewMulConstVal,
SDLoc(
N),
MVT::i32);
585 return NewCost < OldCost;
589 CurDAG->RepositionNode(
N.getNode()->getIterator(),
M.getNode());
593bool ARMDAGToDAGISel::SelectImmShifterOperand(
SDValue N,
596 bool CheckProfitability) {
603 unsigned PowerOfTwo = 0;
605 if (canExtractShiftFromMul(
N, 31, PowerOfTwo, NewMulConst)) {
608 replaceDAGValue(
N.getOperand(1), NewMulConst);
610 Opc = CurDAG->getTargetConstant(
622 BaseReg =
N.getOperand(0);
623 unsigned ShImmVal = 0;
625 if (!RHS)
return false;
626 ShImmVal =
RHS->getZExtValue() & 31;
632bool ARMDAGToDAGISel::SelectRegShifterOperand(
SDValue N,
636 bool CheckProfitability) {
646 BaseReg =
N.getOperand(0);
647 unsigned ShImmVal = 0;
649 if (RHS)
return false;
651 ShReg =
N.getOperand(1);
652 if (CheckProfitability && !isShifterOpProfitable(
N, ShOpcVal, ShImmVal))
664 return CurDAG->haveNoCommonBitsSet(
N, Parent->
getOperand(1));
668bool ARMDAGToDAGISel::SelectAddrModeImm12(
SDValue N,
675 !CurDAG->isBaseWithConstantOffset(
N)) {
678 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
679 Base = CurDAG->getTargetFrameIndex(
680 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
689 Base =
N.getOperand(0);
697 int RHSC = (int)
RHS->getSExtValue();
701 if (RHSC > -0x1000 && RHSC < 0x1000) {
702 Base =
N.getOperand(0);
704 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
705 Base = CurDAG->getTargetFrameIndex(
706 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
727 int RHSC = (int)
RHS->getZExtValue();
736 unsigned ShAmt =
Log2_32(RHSC);
749 !CurDAG->isBaseWithConstantOffset(
N))
756 -0x1000+1, 0x1000, RHSC))
766 Base =
N.getOperand(0);
773 dyn_cast<ConstantSDNode>(
N.getOperand(1).getOperand(1))) {
774 ShAmt = Sh->getZExtValue();
775 if (isShifterOpProfitable(
Offset, ShOpcVal, ShAmt))
776 Offset =
N.getOperand(1).getOperand(0);
789 N.getOperand(0).hasOneUse())) {
795 dyn_cast<ConstantSDNode>(
N.getOperand(0).getOperand(1))) {
796 ShAmt = Sh->getZExtValue();
797 if (isShifterOpProfitable(
N.getOperand(0), ShOpcVal, ShAmt)) {
798 Offset =
N.getOperand(0).getOperand(0);
799 Base =
N.getOperand(1);
813 unsigned PowerOfTwo = 0;
815 if (canExtractShiftFromMul(
Offset, 31, PowerOfTwo, NewMulConst)) {
817 replaceDAGValue(
Offset.getOperand(1), NewMulConst);
818 Offset = Handle.getValue();
829bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(
SDNode *Op,
SDValue N,
831 unsigned Opcode =
Op->getOpcode();
833 ? cast<LoadSDNode>(Op)->getAddressingMode()
834 : cast<StoreSDNode>(Op)->getAddressingMode();
848 ShAmt = Sh->getZExtValue();
849 if (isShifterOpProfitable(
N, ShOpcVal, ShAmt))
865bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(
SDNode *Op,
SDValue N,
867 unsigned Opcode =
Op->getOpcode();
869 ? cast<LoadSDNode>(Op)->getAddressingMode()
870 : cast<StoreSDNode>(Op)->getAddressingMode();
885bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(
SDNode *Op,
SDValue N,
887 unsigned Opcode =
Op->getOpcode();
889 ? cast<LoadSDNode>(Op)->getAddressingMode()
890 : cast<StoreSDNode>(Op)->getAddressingMode();
910bool ARMDAGToDAGISel::SelectAddrMode3(
SDValue N,
915 Base =
N.getOperand(0);
922 if (!CurDAG->isBaseWithConstantOffset(
N)) {
925 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
926 Base = CurDAG->getTargetFrameIndex(
927 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
938 -256 + 1, 256, RHSC)) {
939 Base =
N.getOperand(0);
941 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
942 Base = CurDAG->getTargetFrameIndex(
943 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
957 Base =
N.getOperand(0);
966 unsigned Opcode =
Op->getOpcode();
968 ? cast<LoadSDNode>(Op)->getAddressingMode()
969 : cast<StoreSDNode>(Op)->getAddressingMode();
988 if (!CurDAG->isBaseWithConstantOffset(
N)) {
991 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
992 Base = CurDAG->getTargetFrameIndex(
993 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
998 Base =
N.getOperand(0);
1007 const int Scale = FP16 ? 2 : 4;
1010 Base =
N.getOperand(0);
1012 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1013 Base = CurDAG->getTargetFrameIndex(
1014 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1045bool ARMDAGToDAGISel::SelectAddrMode5(
SDValue N,
1050bool ARMDAGToDAGISel::SelectAddrMode5FP16(
SDValue N,
1059 unsigned Alignment = 0;
1061 MemSDNode *MemN = cast<MemSDNode>(Parent);
1063 if (isa<LSBaseSDNode>(MemN) ||
1071 if (MMOAlign.
value() >= MemSize && MemSize > 1)
1072 Alignment = MemSize;
1084bool ARMDAGToDAGISel::SelectAddrMode6Offset(
SDNode *Op,
SDValue N,
1098bool ARMDAGToDAGISel::SelectAddrModePC(
SDValue N,
1103 Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1124 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1)))
1125 return C->getSExtValue() < 0 &&
C->getSExtValue() >= -255;
1132 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N)) {
1134 if (!
NC || !
NC->isZero())
1141 Base =
N.getOperand(0);
1150 return SelectThumbAddrModeRRSext(
N,
Base,
Offset);
1154ARMDAGToDAGISel::SelectThumbAddrModeImm5S(
SDValue N,
unsigned Scale,
1162 if (!CurDAG->isBaseWithConstantOffset(
N)) {
1170 Base =
N.getOperand(0);
1182 Base =
N.getOperand(0);
1194 return SelectThumbAddrModeImm5S(
N, 4,
Base, OffImm);
1200 return SelectThumbAddrModeImm5S(
N, 2,
Base, OffImm);
1206 return SelectThumbAddrModeImm5S(
N, 1,
Base, OffImm);
1209bool ARMDAGToDAGISel::SelectThumbAddrModeSP(
SDValue N,
1212 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1218 Base = CurDAG->getTargetFrameIndex(
1219 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1224 if (!CurDAG->isBaseWithConstantOffset(
N))
1231 Base =
N.getOperand(0);
1232 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1243 Base = CurDAG->getTargetFrameIndex(
1244 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1255template <
unsigned Shift>
1258 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1262 Base =
N.getOperand(0);
1266 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N),
MVT::i32);
1283bool ARMDAGToDAGISel::SelectT2AddrModeImm12(
SDValue N,
1289 !CurDAG->isBaseWithConstantOffset(
N)) {
1292 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1293 Base = CurDAG->getTargetFrameIndex(
1294 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1303 Base =
N.getOperand(0);
1312 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1313 if (SelectT2AddrModeImm8(
N,
Base, OffImm))
1317 int RHSC = (int)
RHS->getZExtValue();
1321 if (RHSC >= 0 && RHSC < 0x1000) {
1322 Base =
N.getOperand(0);
1324 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1325 Base = CurDAG->getTargetFrameIndex(
1326 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1339template <
unsigned Shift>
1342 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1345 Base =
N.getOperand(0);
1347 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1348 Base = CurDAG->getTargetFrameIndex(
1349 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1355 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N),
MVT::i32);
1366bool ARMDAGToDAGISel::SelectT2AddrModeImm8(
SDValue N,
1370 !CurDAG->isBaseWithConstantOffset(
N))
1373 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1374 int RHSC = (int)
RHS->getSExtValue();
1378 if ((RHSC >= -255) && (RHSC < 0)) {
1379 Base =
N.getOperand(0);
1381 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1382 Base = CurDAG->getTargetFrameIndex(
1383 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1393bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(
SDNode *Op,
SDValue N,
1395 unsigned Opcode =
Op->getOpcode();
1397 ? cast<LoadSDNode>(Op)->getAddressingMode()
1398 : cast<StoreSDNode>(Op)->getAddressingMode();
1410template <
unsigned Shift>
1413 if (
N.getOpcode() ==
ISD::SUB || CurDAG->isBaseWithConstantOffset(
N)) {
1417 Base =
N.getOperand(0);
1419 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1420 Base = CurDAG->getTargetFrameIndex(
1421 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1427 CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N),
MVT::i32);
1438template <
unsigned Shift>
1439bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(
SDNode *Op,
SDValue N,
1441 return SelectT2AddrModeImm7Offset(Op,
N, OffImm, Shift);
1444bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(
SDNode *Op,
SDValue N,
1447 unsigned Opcode =
Op->getOpcode();
1451 AM = cast<LoadSDNode>(Op)->getAddressingMode();
1454 AM = cast<StoreSDNode>(Op)->getAddressingMode();
1457 AM = cast<MaskedLoadSDNode>(Op)->getAddressingMode();
1460 AM = cast<MaskedStoreSDNode>(Op)->getAddressingMode();
1471 ? CurDAG->getTargetConstant(RHSC * (1 << Shift),
SDLoc(
N),
MVT::i32)
1472 : CurDAG->getTargetConstant(-RHSC * (1 << Shift),
SDLoc(
N),
1479template <
int Min,
int Max>
1480bool ARMDAGToDAGISel::SelectImmediateInRange(
SDValue N,
SDValue &OffImm) {
1489bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(
SDValue N,
1493 if (
N.getOpcode() !=
ISD::ADD && !CurDAG->isBaseWithConstantOffset(
N))
1497 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1498 int RHSC = (int)
RHS->getZExtValue();
1499 if (RHSC >= 0 && RHSC < 0x1000)
1501 else if (RHSC < 0 && RHSC >= -255)
1507 Base =
N.getOperand(0);
1508 OffReg =
N.getOperand(1);
1522 ShAmt = Sh->getZExtValue();
1523 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1534 unsigned PowerOfTwo = 0;
1536 if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1538 replaceDAGValue(OffReg.
getOperand(1), NewMulConst);
1556 if (
N.getOpcode() !=
ISD::ADD || !CurDAG->isBaseWithConstantOffset(
N))
1564 if (RHSC > 1020 || RHSC % 4 != 0)
1567 Base =
N.getOperand(0);
1569 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1570 Base = CurDAG->getTargetFrameIndex(
1571 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1585void ARMDAGToDAGISel::transferMemOperands(
SDNode *
N,
SDNode *Result) {
1587 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {
MemOp});
1590bool ARMDAGToDAGISel::tryARMIndexedLoad(
SDNode *
N) {
1596 EVT LoadedVT =
LD->getMemoryVT();
1599 unsigned Opcode = 0;
1601 if (LoadedVT ==
MVT::i32 && isPre &&
1602 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1603 Opcode = ARM::LDR_PRE_IMM;
1605 }
else if (LoadedVT ==
MVT::i32 && !isPre &&
1606 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1607 Opcode = ARM::LDR_POST_IMM;
1610 SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1611 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1615 SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1618 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1619 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1622 if (SelectAddrMode3Offset(
N,
LD->getOffset(),
Offset, AMOpc)) {
1624 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1628 SelectAddrMode2OffsetImmPre(
N,
LD->getOffset(),
Offset, AMOpc)) {
1630 Opcode = ARM::LDRB_PRE_IMM;
1631 }
else if (!isPre &&
1632 SelectAddrMode2OffsetImm(
N,
LD->getOffset(),
Offset, AMOpc)) {
1634 Opcode = ARM::LDRB_POST_IMM;
1635 }
else if (SelectAddrMode2OffsetReg(
N,
LD->getOffset(),
Offset, AMOpc)) {
1637 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1643 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1647 CurDAG->getRegister(0,
MVT::i32), Chain };
1650 transferMemOperands(
N, New);
1651 ReplaceNode(
N, New);
1657 CurDAG->getRegister(0,
MVT::i32), Chain };
1660 transferMemOperands(
N, New);
1661 ReplaceNode(
N, New);
1669bool ARMDAGToDAGISel::tryT1IndexedLoad(
SDNode *
N) {
1671 EVT LoadedVT =
LD->getMemoryVT();
1677 auto *COffs = dyn_cast<ConstantSDNode>(
LD->getOffset());
1678 if (!COffs || COffs->getZExtValue() != 4)
1688 CurDAG->getRegister(0,
MVT::i32), Chain };
1691 transferMemOperands(
N, New);
1692 ReplaceNode(
N, New);
1696bool ARMDAGToDAGISel::tryT2IndexedLoad(
SDNode *
N) {
1702 EVT LoadedVT =
LD->getMemoryVT();
1706 unsigned Opcode = 0;
1708 if (SelectT2AddrModeImm8Offset(
N,
LD->getOffset(),
Offset)) {
1711 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1715 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1717 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1722 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1724 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1736 CurDAG->getRegister(0,
MVT::i32), Chain };
1739 transferMemOperands(
N, New);
1740 ReplaceNode(
N, New);
1747bool ARMDAGToDAGISel::tryMVEIndexedLoad(
SDNode *
N) {
1749 unsigned Opcode = 0;
1750 bool isSExtLd, isPre;
1760 LoadedVT =
LD->getMemoryVT();
1764 Chain =
LD->getChain();
1767 Alignment =
LD->getAlign();
1771 PredReg = CurDAG->getRegister(0,
MVT::i32);
1776 LoadedVT =
LD->getMemoryVT();
1780 Chain =
LD->getChain();
1783 Alignment =
LD->getAlign();
1787 PredReg =
LD->getMask();
1794 bool CanChangeType = Subtarget->
isLittle() && !isa<MaskedLoadSDNode>(
N);
1798 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 1)) {
1800 Opcode = isPre ? ARM::MVE_VLDRHS32_pre : ARM::MVE_VLDRHS32_post;
1802 Opcode = isPre ? ARM::MVE_VLDRHU32_pre : ARM::MVE_VLDRHU32_post;
1804 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0)) {
1806 Opcode = isPre ? ARM::MVE_VLDRBS16_pre : ARM::MVE_VLDRBS16_post;
1808 Opcode = isPre ? ARM::MVE_VLDRBU16_pre : ARM::MVE_VLDRBU16_post;
1810 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0)) {
1812 Opcode = isPre ? ARM::MVE_VLDRBS32_pre : ARM::MVE_VLDRBS32_post;
1814 Opcode = isPre ? ARM::MVE_VLDRBU32_pre : ARM::MVE_VLDRBU32_post;
1815 }
else if (Alignment >=
Align(4) &&
1818 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 2))
1819 Opcode = isPre ? ARM::MVE_VLDRWU32_pre : ARM::MVE_VLDRWU32_post;
1820 else if (Alignment >=
Align(2) &&
1823 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 1))
1824 Opcode = isPre ? ARM::MVE_VLDRHU16_pre : ARM::MVE_VLDRHU16_post;
1825 else if ((CanChangeType || LoadedVT ==
MVT::v16i8) &&
1826 SelectT2AddrModeImm7Offset(
N,
Offset, NewOffset, 0))
1827 Opcode = isPre ? ARM::MVE_VLDRBU8_pre : ARM::MVE_VLDRBU8_post;
1839 transferMemOperands(
N, New);
1843 CurDAG->RemoveDeadNode(
N);
1851 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl,
MVT::i32);
1852 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl,
MVT::i32);
1853 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl,
MVT::i32);
1854 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1855 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1862 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl,
MVT::i32);
1863 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl,
MVT::i32);
1864 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl,
MVT::i32);
1865 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1866 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1872 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1874 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl,
MVT::i32);
1875 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl,
MVT::i32);
1876 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1877 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1883 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1885 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl,
MVT::i32);
1886 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl,
MVT::i32);
1887 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1888 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1896 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl,
MVT::i32);
1897 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl,
MVT::i32);
1898 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl,
MVT::i32);
1899 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl,
MVT::i32);
1900 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl,
MVT::i32);
1901 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1902 V2, SubReg2, V3, SubReg3 };
1903 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1910 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1912 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl,
MVT::i32);
1913 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl,
MVT::i32);
1914 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl,
MVT::i32);
1915 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl,
MVT::i32);
1916 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1917 V2, SubReg2, V3, SubReg3 };
1918 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1925 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1927 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl,
MVT::i32);
1928 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl,
MVT::i32);
1929 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl,
MVT::i32);
1930 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl,
MVT::i32);
1931 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1932 V2, SubReg2, V3, SubReg3 };
1933 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1940 unsigned NumVecs,
bool is64BitVector) {
1941 unsigned NumRegs = NumVecs;
1942 if (!is64BitVector && NumVecs < 3)
1945 unsigned Alignment = cast<ConstantSDNode>(
Align)->getZExtValue();
1946 if (Alignment >= 32 && NumRegs == 4)
1948 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1950 else if (Alignment >= 8)
1955 return CurDAG->getTargetConstant(Alignment, dl,
MVT::i32);
1961 default:
return false;
1962 case ARM::VLD1d8wb_fixed :
return true;
1963 case ARM::VLD1d16wb_fixed :
return true;
1964 case ARM::VLD1d64Qwb_fixed :
return true;
1965 case ARM::VLD1d32wb_fixed :
return true;
1966 case ARM::VLD1d64wb_fixed :
return true;
1967 case ARM::VLD1d8TPseudoWB_fixed :
return true;
1968 case ARM::VLD1d16TPseudoWB_fixed :
return true;
1969 case ARM::VLD1d32TPseudoWB_fixed :
return true;
1970 case ARM::VLD1d64TPseudoWB_fixed :
return true;
1971 case ARM::VLD1d8QPseudoWB_fixed :
return true;
1972 case ARM::VLD1d16QPseudoWB_fixed :
return true;
1973 case ARM::VLD1d32QPseudoWB_fixed :
return true;
1974 case ARM::VLD1d64QPseudoWB_fixed :
return true;
1975 case ARM::VLD1q8wb_fixed :
return true;
1976 case ARM::VLD1q16wb_fixed :
return true;
1977 case ARM::VLD1q32wb_fixed :
return true;
1978 case ARM::VLD1q64wb_fixed :
return true;
1979 case ARM::VLD1DUPd8wb_fixed :
return true;
1980 case ARM::VLD1DUPd16wb_fixed :
return true;
1981 case ARM::VLD1DUPd32wb_fixed :
return true;
1982 case ARM::VLD1DUPq8wb_fixed :
return true;
1983 case ARM::VLD1DUPq16wb_fixed :
return true;
1984 case ARM::VLD1DUPq32wb_fixed :
return true;
1985 case ARM::VLD2d8wb_fixed :
return true;
1986 case ARM::VLD2d16wb_fixed :
return true;
1987 case ARM::VLD2d32wb_fixed :
return true;
1988 case ARM::VLD2q8PseudoWB_fixed :
return true;
1989 case ARM::VLD2q16PseudoWB_fixed :
return true;
1990 case ARM::VLD2q32PseudoWB_fixed :
return true;
1991 case ARM::VLD2DUPd8wb_fixed :
return true;
1992 case ARM::VLD2DUPd16wb_fixed :
return true;
1993 case ARM::VLD2DUPd32wb_fixed :
return true;
1994 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return true;
1995 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return true;
1996 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return true;
2003 default:
return false;
2004 case ARM::VST1d8wb_fixed :
return true;
2005 case ARM::VST1d16wb_fixed :
return true;
2006 case ARM::VST1d32wb_fixed :
return true;
2007 case ARM::VST1d64wb_fixed :
return true;
2008 case ARM::VST1q8wb_fixed :
return true;
2009 case ARM::VST1q16wb_fixed :
return true;
2010 case ARM::VST1q32wb_fixed :
return true;
2011 case ARM::VST1q64wb_fixed :
return true;
2012 case ARM::VST1d8TPseudoWB_fixed :
return true;
2013 case ARM::VST1d16TPseudoWB_fixed :
return true;
2014 case ARM::VST1d32TPseudoWB_fixed :
return true;
2015 case ARM::VST1d64TPseudoWB_fixed :
return true;
2016 case ARM::VST1d8QPseudoWB_fixed :
return true;
2017 case ARM::VST1d16QPseudoWB_fixed :
return true;
2018 case ARM::VST1d32QPseudoWB_fixed :
return true;
2019 case ARM::VST1d64QPseudoWB_fixed :
return true;
2020 case ARM::VST2d8wb_fixed :
return true;
2021 case ARM::VST2d16wb_fixed :
return true;
2022 case ARM::VST2d32wb_fixed :
return true;
2023 case ARM::VST2q8PseudoWB_fixed :
return true;
2024 case ARM::VST2q16PseudoWB_fixed :
return true;
2025 case ARM::VST2q32PseudoWB_fixed :
return true;
2033 &&
"Incorrect fixed stride updating instruction.");
2036 case ARM::VLD1d8wb_fixed:
return ARM::VLD1d8wb_register;
2037 case ARM::VLD1d16wb_fixed:
return ARM::VLD1d16wb_register;
2038 case ARM::VLD1d32wb_fixed:
return ARM::VLD1d32wb_register;
2039 case ARM::VLD1d64wb_fixed:
return ARM::VLD1d64wb_register;
2040 case ARM::VLD1q8wb_fixed:
return ARM::VLD1q8wb_register;
2041 case ARM::VLD1q16wb_fixed:
return ARM::VLD1q16wb_register;
2042 case ARM::VLD1q32wb_fixed:
return ARM::VLD1q32wb_register;
2043 case ARM::VLD1q64wb_fixed:
return ARM::VLD1q64wb_register;
2044 case ARM::VLD1d64Twb_fixed:
return ARM::VLD1d64Twb_register;
2045 case ARM::VLD1d64Qwb_fixed:
return ARM::VLD1d64Qwb_register;
2046 case ARM::VLD1d8TPseudoWB_fixed:
return ARM::VLD1d8TPseudoWB_register;
2047 case ARM::VLD1d16TPseudoWB_fixed:
return ARM::VLD1d16TPseudoWB_register;
2048 case ARM::VLD1d32TPseudoWB_fixed:
return ARM::VLD1d32TPseudoWB_register;
2049 case ARM::VLD1d64TPseudoWB_fixed:
return ARM::VLD1d64TPseudoWB_register;
2050 case ARM::VLD1d8QPseudoWB_fixed:
return ARM::VLD1d8QPseudoWB_register;
2051 case ARM::VLD1d16QPseudoWB_fixed:
return ARM::VLD1d16QPseudoWB_register;
2052 case ARM::VLD1d32QPseudoWB_fixed:
return ARM::VLD1d32QPseudoWB_register;
2053 case ARM::VLD1d64QPseudoWB_fixed:
return ARM::VLD1d64QPseudoWB_register;
2054 case ARM::VLD1DUPd8wb_fixed :
return ARM::VLD1DUPd8wb_register;
2055 case ARM::VLD1DUPd16wb_fixed :
return ARM::VLD1DUPd16wb_register;
2056 case ARM::VLD1DUPd32wb_fixed :
return ARM::VLD1DUPd32wb_register;
2057 case ARM::VLD1DUPq8wb_fixed :
return ARM::VLD1DUPq8wb_register;
2058 case ARM::VLD1DUPq16wb_fixed :
return ARM::VLD1DUPq16wb_register;
2059 case ARM::VLD1DUPq32wb_fixed :
return ARM::VLD1DUPq32wb_register;
2060 case ARM::VLD2DUPq8OddPseudoWB_fixed:
return ARM::VLD2DUPq8OddPseudoWB_register;
2061 case ARM::VLD2DUPq16OddPseudoWB_fixed:
return ARM::VLD2DUPq16OddPseudoWB_register;
2062 case ARM::VLD2DUPq32OddPseudoWB_fixed:
return ARM::VLD2DUPq32OddPseudoWB_register;
2064 case ARM::VST1d8wb_fixed:
return ARM::VST1d8wb_register;
2065 case ARM::VST1d16wb_fixed:
return ARM::VST1d16wb_register;
2066 case ARM::VST1d32wb_fixed:
return ARM::VST1d32wb_register;
2067 case ARM::VST1d64wb_fixed:
return ARM::VST1d64wb_register;
2068 case ARM::VST1q8wb_fixed:
return ARM::VST1q8wb_register;
2069 case ARM::VST1q16wb_fixed:
return ARM::VST1q16wb_register;
2070 case ARM::VST1q32wb_fixed:
return ARM::VST1q32wb_register;
2071 case ARM::VST1q64wb_fixed:
return ARM::VST1q64wb_register;
2072 case ARM::VST1d8TPseudoWB_fixed:
return ARM::VST1d8TPseudoWB_register;
2073 case ARM::VST1d16TPseudoWB_fixed:
return ARM::VST1d16TPseudoWB_register;
2074 case ARM::VST1d32TPseudoWB_fixed:
return ARM::VST1d32TPseudoWB_register;
2075 case ARM::VST1d64TPseudoWB_fixed:
return ARM::VST1d64TPseudoWB_register;
2076 case ARM::VST1d8QPseudoWB_fixed:
return ARM::VST1d8QPseudoWB_register;
2077 case ARM::VST1d16QPseudoWB_fixed:
return ARM::VST1d16QPseudoWB_register;
2078 case ARM::VST1d32QPseudoWB_fixed:
return ARM::VST1d32QPseudoWB_register;
2079 case ARM::VST1d64QPseudoWB_fixed:
return ARM::VST1d64QPseudoWB_register;
2081 case ARM::VLD2d8wb_fixed:
return ARM::VLD2d8wb_register;
2082 case ARM::VLD2d16wb_fixed:
return ARM::VLD2d16wb_register;
2083 case ARM::VLD2d32wb_fixed:
return ARM::VLD2d32wb_register;
2084 case ARM::VLD2q8PseudoWB_fixed:
return ARM::VLD2q8PseudoWB_register;
2085 case ARM::VLD2q16PseudoWB_fixed:
return ARM::VLD2q16PseudoWB_register;
2086 case ARM::VLD2q32PseudoWB_fixed:
return ARM::VLD2q32PseudoWB_register;
2088 case ARM::VST2d8wb_fixed:
return ARM::VST2d8wb_register;
2089 case ARM::VST2d16wb_fixed:
return ARM::VST2d16wb_register;
2090 case ARM::VST2d32wb_fixed:
return ARM::VST2d32wb_register;
2091 case ARM::VST2q8PseudoWB_fixed:
return ARM::VST2q8PseudoWB_register;
2092 case ARM::VST2q16PseudoWB_fixed:
return ARM::VST2q16PseudoWB_register;
2093 case ARM::VST2q32PseudoWB_fixed:
return ARM::VST2q32PseudoWB_register;
2095 case ARM::VLD2DUPd8wb_fixed:
return ARM::VLD2DUPd8wb_register;
2096 case ARM::VLD2DUPd16wb_fixed:
return ARM::VLD2DUPd16wb_register;
2097 case ARM::VLD2DUPd32wb_fixed:
return ARM::VLD2DUPd32wb_register;
2106 auto C = dyn_cast<ConstantSDNode>(Inc);
2110void ARMDAGToDAGISel::SelectVLD(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2114 assert(Subtarget->hasNEON());
2115 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLD NumVecs out-of-range");
2119 bool IsIntrinsic = !isUpdating;
2121 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2122 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2126 EVT VT =
N->getValueType(0);
2128 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2156 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2161 std::vector<EVT> ResTys;
2162 ResTys.push_back(ResTy);
2173 if (is64BitVector || NumVecs <= 2) {
2174 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2179 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2195 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2205 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2206 const SDValue OpsA[] = { MemAddr,
Align, Reg0, ImplDef, Pred, Reg0, Chain };
2215 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2217 "only constant post-increment update allowed for VLD3/4");
2225 VLd = CurDAG->getMachineNode(QOpcodes1[
OpcodeIndex], dl, ResTys, Ops);
2230 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {
MemOp});
2233 ReplaceNode(
N, VLd);
2239 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2240 ARM::qsub_3 == ARM::qsub_0 + 3,
2241 "Unexpected subreg numbering");
2242 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
2243 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2245 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2249 CurDAG->RemoveDeadNode(
N);
2252void ARMDAGToDAGISel::SelectVST(
SDNode *
N,
bool isUpdating,
unsigned NumVecs,
2256 assert(Subtarget->hasNEON());
2257 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VST NumVecs out-of-range");
2261 bool IsIntrinsic = !isUpdating;
2263 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2264 unsigned Vec0Idx = 3;
2265 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2271 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2273 Align = GetVLDSTAlign(
Align, dl, NumVecs, is64BitVector);
2297 std::vector<EVT> ResTys;
2307 if (is64BitVector || NumVecs <= 2) {
2310 SrcReg =
N->getOperand(Vec0Idx);
2311 }
else if (is64BitVector) {
2313 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2314 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2322 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
2323 :
N->getOperand(Vec0Idx + 3);
2328 SDValue Q0 =
N->getOperand(Vec0Idx);
2329 SDValue Q1 =
N->getOperand(Vec0Idx + 1);
2333 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2338 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2356 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2359 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {
MemOp});
2361 ReplaceNode(
N, VSt);
2369 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2370 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2373 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2374 :
N->getOperand(Vec0Idx + 3);
2379 const SDValue OpsA[] = { MemAddr,
Align, Reg0, RegSeq, Pred, Reg0, Chain };
2383 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {
MemOp});
2390 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2392 "only constant post-increment update allowed for VST3/4");
2402 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {
MemOp});
2403 ReplaceNode(
N, VStB);
2406void ARMDAGToDAGISel::SelectVLDSTLane(
SDNode *
N,
bool IsLoad,
bool isUpdating,
2410 assert(Subtarget->hasNEON());
2411 assert(NumVecs >=2 && NumVecs <= 4 &&
"VLDSTLane NumVecs out-of-range");
2415 bool IsIntrinsic = !isUpdating;
2417 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2418 unsigned Vec0Idx = 3;
2419 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2426 cast<ConstantSDNode>(
N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2427 EVT VT =
N->getOperand(Vec0Idx).getValueType();
2430 unsigned Alignment = 0;
2432 Alignment = cast<ConstantSDNode>(
Align)->getZExtValue();
2434 if (Alignment > NumBytes)
2435 Alignment = NumBytes;
2436 if (Alignment < 8 && Alignment < NumBytes)
2439 Alignment = (Alignment & -Alignment);
2463 std::vector<EVT> ResTys;
2465 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2482 SDValue Inc =
N->getOperand(AddrOpIdx + 1);
2485 Ops.
push_back(IsImmUpdate ? Reg0 : Inc);
2489 SDValue V0 =
N->getOperand(Vec0Idx + 0);
2490 SDValue V1 =
N->getOperand(Vec0Idx + 1);
2499 ?
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2500 :
N->getOperand(Vec0Idx + 3);
2512 unsigned Opc = (is64BitVector ? DOpcodes[
OpcodeIndex] :
2514 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2515 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {
MemOp});
2517 ReplaceNode(
N, VLdLn);
2523 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2524 ARM::qsub_3 == ARM::qsub_0 + 3,
2525 "Unexpected subreg numbering");
2526 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2527 for (
unsigned Vec = 0; Vec < NumVecs; ++Vec)
2529 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2533 CurDAG->RemoveDeadNode(
N);
2536template <
typename SDValueVector>
2537void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2540 Ops.push_back(PredicateMask);
2541 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2544template <
typename SDValueVector>
2545void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2549 Ops.push_back(PredicateMask);
2550 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2551 Ops.push_back(Inactive);
2554template <
typename SDValueVector>
2555void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc) {
2557 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2558 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2561template <
typename SDValueVector>
2562void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops,
SDLoc Loc,
2565 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2566 Ops.push_back(CurDAG->getRegister(0,
MVT::i32));
2568 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, InactiveTy), 0));
2577 switch (
N->getValueType(1).getVectorElementType().getSizeInBits()) {
2579 Opcode = Opcodes[0];
2582 Opcode = Opcodes[1];
2590 int32_t ImmValue = cast<ConstantSDNode>(
N->getOperand(3))->getZExtValue();
2591 Ops.
push_back(getI32Imm(ImmValue, Loc));
2594 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2596 AddEmptyMVEPredicateToOps(Ops, Loc);
2609 transferMemOperands(
N, New);
2610 CurDAG->RemoveDeadNode(
N);
2615 bool HasSaturationOperand) {
2625 int32_t ImmValue = cast<ConstantSDNode>(
N->getOperand(3))->getZExtValue();
2626 Ops.
push_back(getI32Imm(ImmValue, Loc));
2632 if (HasSaturationOperand) {
2633 int32_t SatOp = cast<ConstantSDNode>(
N->getOperand(4))->getZExtValue();
2634 int SatBit = (SatOp == 64 ? 0 : 1);
2643 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2646void ARMDAGToDAGISel::SelectMVE_VADCSBC(
SDNode *
N,
uint16_t OpcodeWithCarry,
2648 bool Add,
bool Predicated) {
2653 unsigned FirstInputOp = Predicated ? 2 : 1;
2657 Ops.
push_back(
N->getOperand(FirstInputOp + 1));
2658 SDValue CarryIn =
N->getOperand(FirstInputOp + 2);
2659 ConstantSDNode *CarryInConstant = dyn_cast<ConstantSDNode>(CarryIn);
2662 if (CarryInConstant &&
2663 (CarryInConstant->
getZExtValue() & CarryMask) == CarryExpected) {
2664 Opcode = OpcodeWithNoCarry;
2667 Opcode = OpcodeWithCarry;
2671 AddMVEPredicateToOps(Ops, Loc,
2672 N->getOperand(FirstInputOp + 3),
2673 N->getOperand(FirstInputOp - 1));
2675 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2677 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2680void ARMDAGToDAGISel::SelectMVE_VSHLC(
SDNode *
N,
bool Predicated) {
2688 int32_t ImmValue = cast<ConstantSDNode>(
N->getOperand(3))->getZExtValue();
2689 Ops.
push_back(getI32Imm(ImmValue, Loc));
2692 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(4));
2694 AddEmptyMVEPredicateToOps(Ops, Loc);
2696 CurDAG->SelectNodeTo(
N, ARM::MVE_VSHLC,
N->getVTList(),
ArrayRef(Ops));
2700 assert(isa<ConstantSDNode>(SDVal) &&
"expected a compile-time constant");
2707void ARMDAGToDAGISel::SelectBaseMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
2710 size_t Stride,
size_t TySize) {
2711 assert(TySize < Stride &&
"Invalid TySize");
2717 "Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist");
2719 "Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist");
2722 auto OpIsZero = [
N](
size_t OpNo) {
2728 bool IsAccum = !(OpIsZero(4) && OpIsZero(5));
2730 const uint16_t *Opcodes = IsUnsigned ? OpcodesU : OpcodesS;
2732 Opcodes += 4 * Stride;
2734 Opcodes += 2 * Stride;
2751 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(8));
2753 AddEmptyMVEPredicateToOps(Ops, Loc);
2755 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2758void ARMDAGToDAGISel::SelectMVE_VMLLDAV(
SDNode *
N,
bool Predicated,
2761 EVT VecTy =
N->getOperand(6).getValueType();
2774 SelectBaseMVE_VMLLDAV(
N, Predicated, OpcodesS, OpcodesU, 2, SizeIndex);
2777void ARMDAGToDAGISel::SelectMVE_VRMLLDAVH(
SDNode *
N,
bool Predicated,
2781 N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() ==
2783 "bad vector element size");
2784 SelectBaseMVE_VMLLDAV(
N, Predicated, OpcodesS, OpcodesU, 1, 0);
2787void ARMDAGToDAGISel::SelectMVE_VLD(
SDNode *
N,
unsigned NumVecs,
2789 bool HasWriteback) {
2790 EVT VT =
N->getValueType(0);
2796 OurOpcodes = Opcodes[0];
2799 OurOpcodes = Opcodes[1];
2802 OurOpcodes = Opcodes[2];
2810 unsigned PtrOperand = HasWriteback ? 1 : 2;
2813 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, DataTy), 0);
2816 for (
unsigned Stage = 0; Stage < NumVecs - 1; ++Stage) {
2817 SDValue Ops[] = {
Data,
N->getOperand(PtrOperand), Chain};
2819 CurDAG->getMachineNode(OurOpcodes[Stage], Loc, ResultTys, Ops);
2827 SDValue Ops[] = {
Data,
N->getOperand(PtrOperand), Chain};
2829 CurDAG->getMachineNode(OurOpcodes[NumVecs - 1], Loc, ResultTys, Ops);
2833 for (i = 0; i < NumVecs; i++)
2835 CurDAG->getTargetExtractSubreg(ARM::qsub_0 + i, Loc, VT,
2840 CurDAG->RemoveDeadNode(
N);
2843void ARMDAGToDAGISel::SelectMVE_VxDUP(
SDNode *
N,
const uint16_t *Opcodes,
2844 bool Wrapping,
bool Predicated) {
2845 EVT VT =
N->getValueType(0);
2851 Opcode = Opcodes[0];
2854 Opcode = Opcodes[1];
2857 Opcode = Opcodes[2];
2868 Inactive =
N->getOperand(OpIdx++);
2874 SDValue ImmOp =
N->getOperand(OpIdx++);
2875 int ImmValue = cast<ConstantSDNode>(ImmOp)->getZExtValue();
2876 Ops.
push_back(getI32Imm(ImmValue, Loc));
2879 AddMVEPredicateToOps(Ops, Loc,
N->getOperand(OpIdx), Inactive);
2881 AddEmptyMVEPredicateToOps(Ops, Loc,
N->getValueType(0));
2883 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
ArrayRef(Ops));
2887 size_t NumExtraOps,
bool HasAccum) {
2888 bool IsBigEndian = CurDAG->getDataLayout().isBigEndian();
2895 SDValue ImmCorpoc =
N->getOperand(OpIdx++);
2896 uint32_t ImmCoprocVal = cast<ConstantSDNode>(ImmCorpoc)->getZExtValue();
2897 Ops.
push_back(getI32Imm(ImmCoprocVal, Loc));
2902 SDValue AccLo =
N->getOperand(OpIdx++);
2903 SDValue AccHi =
N->getOperand(OpIdx++);
2910 for (
size_t I = 0;
I < NumExtraOps;
I++)
2915 uint32_t ImmVal = cast<ConstantSDNode>(Imm)->getZExtValue();
2934 uint16_t SubRegs[2] = {ARM::gsub_0, ARM::gsub_1};
2938 for (
size_t ResIdx = 0; ResIdx < 2; ResIdx++) {
2939 if (
SDValue(
N, ResIdx).use_empty())
2941 SDValue SubReg = CurDAG->getTargetExtractSubreg(SubRegs[ResIdx], Loc,
2946 CurDAG->RemoveDeadNode(
N);
2949void ARMDAGToDAGISel::SelectVLDDup(
SDNode *
N,
bool IsIntrinsic,
2950 bool isUpdating,
unsigned NumVecs,
2954 assert(Subtarget->hasNEON());
2955 assert(NumVecs >= 1 && NumVecs <= 4 &&
"VLDDup NumVecs out-of-range");
2959 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2960 if (!SelectAddrMode6(
N,
N->getOperand(AddrOpIdx), MemAddr,
Align))
2964 EVT VT =
N->getValueType(0);
2967 unsigned Alignment = 0;
2969 Alignment = cast<ConstantSDNode>(
Align)->getZExtValue();
2971 if (Alignment > NumBytes)
2972 Alignment = NumBytes;
2973 if (Alignment < 8 && Alignment < NumBytes)
2976 Alignment = (Alignment & -Alignment);
3002 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
3007 std::vector<EVT> ResTys;
3008 ResTys.push_back(ResTy);
3019 unsigned Opc = is64BitVector ? DOpcodes[
OpcodeIndex]
3035 if (is64BitVector || NumVecs == 1) {
3037 }
else if (NumVecs == 2) {
3038 const SDValue OpsA[] = {MemAddr,
Align, Pred, Reg0, Chain};
3044 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
3045 const SDValue OpsA[] = {MemAddr,
Align, ImplDef, Pred, Reg0, Chain};
3056 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
3060 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {
MemOp});
3067 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7,
"Unexpected subreg numbering");
3068 unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
3069 for (
unsigned Vec = 0; Vec != NumVecs; ++Vec) {
3071 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
3077 CurDAG->RemoveDeadNode(
N);
3080bool ARMDAGToDAGISel::tryInsertVectorElt(
SDNode *
N) {
3081 if (!Subtarget->hasMVEIntegerOps())
3100 if (Lane2 % 2 != 0 || Lane1 != Lane2 + 1)
3127 ExtractLane1 == ExtractLane2 + 1) {
3128 SDValue NewExt = CurDAG->getTargetExtractSubreg(
3130 SDValue NewIns = CurDAG->getTargetInsertSubreg(
3131 ARM::ssub_0 + Lane2 / 2, dl, VT, Ins2.
getOperand(0),
3133 ReplaceUses(Ins1, NewIns);
3139 if (VT ==
MVT::v8i16 && Subtarget->hasFullFP16()) {
3140 SDValue Inp1 = CurDAG->getTargetExtractSubreg(
3142 SDValue Inp2 = CurDAG->getTargetExtractSubreg(
3144 if (ExtractLane1 % 2 != 0)
3145 Inp1 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl,
MVT::f32, Inp1), 0);
3146 if (ExtractLane2 % 2 != 0)
3147 Inp2 =
SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl,
MVT::f32, Inp2), 0);
3148 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl,
MVT::f32, Inp2, Inp1);
3150 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl,
MVT::v4f32,
3152 ReplaceUses(Ins1, NewIns);
3159 if (VT ==
MVT::v8f16 && Subtarget->hasFullFP16()) {
3160 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl,
MVT::f32, Val2, Val1);
3162 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl,
MVT::v4f32,
3164 ReplaceUses(Ins1, NewIns);
3171bool ARMDAGToDAGISel::transformFixedFloatingPointConversion(
SDNode *
N,
3174 bool FixedToFloat) {
3175 auto Type =
N->getValueType(0);
3177 if (ScalarBits > 32)
3183 if (ScalarBits == 16 && !FMulFlags.
hasNoInfs() && IsUnsigned)
3208 if (!isa<ConstantSDNode>(ImmNode.
getOperand(0)))
3214 APFloat(ScalarBits == 32 ? APFloat::IEEEsingle() : APFloat::IEEEhalf(),
3215 APInt(ScalarBits, Imm));
3232 if (!ImmAPF.getExactInverse(&ToConvert))
3235 APSInt Converted(64,
false);
3239 if (!IsExact || !Converted.isPowerOf2())
3242 unsigned FracBits = Converted.logBase2();
3243 if (FracBits > ScalarBits)
3248 AddEmptyMVEPredicateToOps(Ops,
SDLoc(
N),
Type);
3250 unsigned int Opcode;
3251 switch (ScalarBits) {
3254 Opcode = IsUnsigned ? ARM::MVE_VCVTf16u16_fix : ARM::MVE_VCVTf16s16_fix;
3256 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3260 Opcode = IsUnsigned ? ARM::MVE_VCVTf32u32_fix : ARM::MVE_VCVTf32s32_fix;
3262 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3269 ReplaceNode(
N, CurDAG->getMachineNode(Opcode,
SDLoc(
N),
Type, Ops));
3273bool ARMDAGToDAGISel::tryFP_TO_INT(
SDNode *
N,
SDLoc dl) {
3275 if (!Subtarget->hasMVEFloatOps())
3278 if (!
Type.isVector())
3289 if (
Node->getOperand(0) !=
Node->getOperand(1))
3294 if (ScalarBits == 16 && !
Flags.hasNoInfs() && IsUnsigned)
3298 switch (ScalarBits) {
3300 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3303 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3307 CurDAG->getConstant(1, dl,
MVT::i32)};
3308 AddEmptyMVEPredicateToOps(Ops, dl,
Type);
3310 ReplaceNode(
N, CurDAG->getMachineNode(Opcode, dl,
Type, Ops));
3317 return transformFixedFloatingPointConversion(
N,
Node, IsUnsigned,
false);
3320bool ARMDAGToDAGISel::tryFMULFixed(
SDNode *
N,
SDLoc dl) {
3322 if (!Subtarget->hasMVEFloatOps())
3324 auto Type =
N->getValueType(0);
3325 if (!
Type.isVector())
3328 auto LHS =
N->getOperand(0);
3332 return transformFixedFloatingPointConversion(
3336bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(
SDNode *
N,
bool isSigned) {
3337 if (!Subtarget->hasV6T2Ops())
3341 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
3342 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
3346 unsigned And_imm = 0;
3351 if (And_imm & (And_imm + 1))
3354 unsigned Srl_imm = 0;
3357 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3362 And_imm &= -1U >> Srl_imm;
3366 unsigned LSB = Srl_imm;
3370 if ((LSB + Width + 1) ==
N->getValueType(0).getSizeInBits()) {
3372 if (Subtarget->isThumb()) {
3373 Opc =
isSigned ? ARM::t2ASRri : ARM::t2LSRri;
3374 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3375 CurDAG->getTargetConstant(LSB, dl,
MVT::i32),
3376 getAL(CurDAG, dl), Reg0, Reg0 };
3377 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3386 SDValue Ops[] = {
N->getOperand(0).getOperand(0), ShOpc,
3387 getAL(CurDAG, dl), Reg0, Reg0 };
3388 CurDAG->SelectNodeTo(
N, ARM::MOVsi,
MVT::i32, Ops);
3392 assert(LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3393 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3394 CurDAG->getTargetConstant(LSB, dl,
MVT::i32),
3395 CurDAG->getTargetConstant(Width, dl,
MVT::i32),
3396 getAL(CurDAG, dl), Reg0 };
3397 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3405 unsigned Shl_imm = 0;
3407 assert(Shl_imm > 0 && Shl_imm < 32 &&
"bad amount in shift node!");
3408 unsigned Srl_imm = 0;
3410 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3412 unsigned Width = 32 - Srl_imm - 1;
3413 int LSB = Srl_imm - Shl_imm;
3417 assert(LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3418 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3419 CurDAG->getTargetConstant(LSB, dl,
MVT::i32),
3420 CurDAG->getTargetConstant(Width, dl,
MVT::i32),
3421 getAL(CurDAG, dl), Reg0 };
3422 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3430 unsigned Srl_imm = 0;
3434 assert(Srl_imm > 0 && Srl_imm < 32 &&
"bad amount in shift node!");
3437 unsigned Width = MSB - LSB;
3439 assert(Srl_imm + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx");
3440 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3441 CurDAG->getTargetConstant(Srl_imm, dl,
MVT::i32),
3442 CurDAG->getTargetConstant(Width, dl,
MVT::i32),
3443 getAL(CurDAG, dl), Reg0 };
3444 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3450 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
3456 if (LSB + Width > 32)
3460 assert(LSB + Width <= 32 &&
"Shouldn't create an invalid ubfx");
3461 SDValue Ops[] = {
N->getOperand(0).getOperand(0),
3462 CurDAG->getTargetConstant(LSB, dl,
MVT::i32),
3463 CurDAG->getTargetConstant(Width - 1, dl,
MVT::i32),
3464 getAL(CurDAG, dl), Reg0 };
3465 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3482bool ARMDAGToDAGISel::tryABSOp(
SDNode *
N){
3483 SDValue SUBSrc0 =
N->getOperand(0);
3484 SDValue SUBSrc1 =
N->getOperand(1);
3485 EVT VT =
N->getValueType(0);
3501 if (XORSrc1 == SUBSrc1 && XORSrc0 == SRASrc0 && XType.
isInteger() &&
3503 unsigned Opcode = Subtarget->
isThumb2() ? ARM::t2ABS : ARM::ABS;
3504 CurDAG->SelectNodeTo(
N, Opcode, VT, XORSrc0);
3512void ARMDAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
3514 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
3516 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_8 : ARM::CMP_SWAP_8;
3518 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_16 : ARM::CMP_SWAP_16;
3520 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_32 : ARM::CMP_SWAP_32;
3524 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3526 SDNode *CmpSwap = CurDAG->getMachineNode(
3531 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
3535 CurDAG->RemoveDeadNode(
N);
3538static std::optional<std::pair<unsigned, unsigned>>
3540 unsigned FirstOne =
A.getBitWidth() -
A.countl_zero() - 1;
3541 unsigned LastOne =
A.countr_zero();
3542 if (
A.popcount() != (FirstOne - LastOne + 1))
3543 return std::nullopt;
3544 return std::make_pair(FirstOne, LastOne);
3547void ARMDAGToDAGISel::SelectCMPZ(
SDNode *
N,
bool &SwitchEQNEToPLMI) {
3549 SwitchEQNEToPLMI =
false;
3551 if (!Subtarget->isThumb())
3558 if (!
And->hasOneUse())
3562 if (!isa<ConstantSDNode>(Zero) || !cast<ConstantSDNode>(Zero)->
isZero() ||
3566 auto C = dyn_cast<ConstantSDNode>(
And.getOperand(1));
3580 Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
3583 CurDAG->getRegister(0,
MVT::i32) };
3584 return CurDAG->getMachineNode(Opc, dl,
MVT::i32, Ops);
3587 CurDAG->getTargetConstant(Imm, dl,
MVT::i32),
3589 return CurDAG->getMachineNode(Opc, dl,
MVT::i32, Ops);
3593 if (
Range->second == 0) {
3595 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3596 ReplaceNode(
And.getNode(), NewN);
3597 }
else if (
Range->first == 31) {
3599 NewN = EmitShift(ARM::tLSRri,
X,
Range->second);
3600 ReplaceNode(
And.getNode(), NewN);
3604 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3605 ReplaceNode(
And.getNode(), NewN);
3607 SwitchEQNEToPLMI =
true;
3608 }
else if (!Subtarget->hasV6T2Ops()) {
3611 NewN = EmitShift(ARM::tLSLri,
X, 31 -
Range->first);
3612 NewN = EmitShift(ARM::tLSRri,
SDValue(NewN, 0),
3614 ReplaceNode(
And.getNode(), NewN);
3619 unsigned Opc128[3]) {
3621 "Unexpected vector shuffle length");
3634void ARMDAGToDAGISel::Select(
SDNode *
N) {
3637 if (
N->isMachineOpcode()) {
3642 switch (
N->getOpcode()) {
3664 cast<RegisterSDNode>(
Ptr.getOperand(1))->getReg() == ARM::SP &&
3665 Ptr.getOperand(0) ==
ST->getChain()) {
3667 CurDAG->getRegister(ARM::SP,
MVT::i32),
3668 CurDAG->getTargetConstant(RHSC, dl,
MVT::i32),
3673 CurDAG->getMachineNode(ARM::tSTRspi, dl,
MVT::Other, Ops);
3675 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3676 ReplaceNode(
N, ResNode);
3683 if (tryWriteRegister(
N))
3687 if (tryReadRegister(
N))
3692 if (tryInlineAsm(
N))
3702 unsigned Val = cast<ConstantSDNode>(
N)->getZExtValue();
3705 SDValue CPIdx = CurDAG->getTargetConstantPool(
3707 TLI->getPointerTy(CurDAG->getDataLayout()));
3710 if (Subtarget->isThumb()) {
3715 CurDAG->getEntryNode()
3722 CurDAG->getTargetConstant(0, dl,
MVT::i32),
3725 CurDAG->getEntryNode()
3738 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {
MemOp});
3740 ReplaceNode(
N, ResNode);
3749 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
3750 SDValue TFI = CurDAG->getTargetFrameIndex(
3751 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
3758 CurDAG->SelectNodeTo(
N, ARM::tADDframe,
MVT::i32, TFI,
3759 CurDAG->getTargetConstant(0, dl,
MVT::i32));
3762 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
3763 ARM::t2ADDri : ARM::ADDri);
3766 CurDAG->getRegister(0,
MVT::i32) };
3767 CurDAG->SelectNodeTo(
N, Opc,
MVT::i32, Ops);
3772 if (tryInsertVectorElt(
N))
3777 if (tryV6T2BitfieldExtractOp(
N,
false))
3782 if (tryV6T2BitfieldExtractOp(
N,
true))
3789 if (tryFP_TO_INT(
N, dl))
3793 if (tryFMULFixed(
N, dl))
3800 unsigned RHSV =
C->getZExtValue();
3803 unsigned ShImm =
Log2_32(RHSV-1);
3810 if (Subtarget->isThumb()) {
3811 SDValue Ops[] = {
V,
V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3812 CurDAG->SelectNodeTo(
N, ARM::t2ADDrs,
MVT::i32, Ops);
3817 CurDAG->SelectNodeTo(
N, ARM::ADDrsi,
MVT::i32, Ops);
3822 unsigned ShImm =
Log2_32(RHSV+1);
3829 if (Subtarget->isThumb()) {
3830 SDValue Ops[] = {
V,
V, ShImmOp,
getAL(CurDAG, dl), Reg0, Reg0 };
3831 CurDAG->SelectNodeTo(
N, ARM::t2RSBrs,
MVT::i32, Ops);
3836 CurDAG->SelectNodeTo(
N, ARM::RSBrsi,
MVT::i32, Ops);
3844 if (tryV6T2BitfieldExtractOp(
N,
false))
3850 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
3851 if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
3858 bool PreferImmediateEncoding =
3859 Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
3860 if (!PreferImmediateEncoding &&
3866 CurDAG->getConstant(~N1C->getZExtValue(), dl,
MVT::i32);
3870 CurDAG->RepositionNode(
N->getIterator(), NewImm.
getNode());
3872 if (!Subtarget->hasThumb2()) {
3874 N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3876 ReplaceNode(
N, CurDAG->getMachineNode(ARM::tBIC, dl,
MVT::i32, Ops));
3879 SDValue Ops[] = {
N->getOperand(0), NewImm,
getAL(CurDAG, dl),
3883 CurDAG->getMachineNode(ARM::t2BICrr, dl,
MVT::i32, Ops));
3894 EVT VT =
N->getValueType(0);
3897 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
3899 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
3902 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
3903 N1C = dyn_cast<ConstantSDNode>(N1);
3911 unsigned N1CVal = N1C->getZExtValue();
3913 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
3914 (N1CVal & 0xffffU) == 0xffffU &&
3915 (N2CVal & 0xffffU) == 0x0U) {
3916 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
3920 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
3928 unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
3929 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
3930 N->getOperand(2),
N->getOperand(3),
3932 CurDAG->getRegister(0,
MVT::i32) };
3937 if (Subtarget->isThumb()) {
3938 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3939 N->getOperand(3),
getAL(CurDAG, dl),
3945 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3946 N->getOperand(3),
getAL(CurDAG, dl),
3948 CurDAG->getRegister(0,
MVT::i32) };
3949 ReplaceNode(
N, CurDAG->getMachineNode(
3950 Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
3956 if (Subtarget->isThumb()) {
3957 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3958 N->getOperand(3),
getAL(CurDAG, dl),
3964 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
3965 N->getOperand(3),
getAL(CurDAG, dl),
3967 CurDAG->getRegister(0,
MVT::i32) };
3968 ReplaceNode(
N, CurDAG->getMachineNode(
3969 Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
3975 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
3984 if (Subtarget->isThumb())
3985 assert(Subtarget->hasThumb2() &&
3986 "This pattern should not be generated for Thumb");
3988 SDValue SmulLoHi =
N->getOperand(1);
3993 N->getOperand(1) != SmulLoHi.
getValue(1) ||
3997 unsigned Opc = Subtarget->
isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
3999 N->getOperand(0),
getAL(CurDAG, dl),
4000 CurDAG->getRegister(0,
MVT::i32) };
4001 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl,
MVT::i32, Ops));
4005 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4007 if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
4008 if (tryT2IndexedLoad(
N))
4010 }
else if (Subtarget->isThumb()) {
4011 if (tryT1IndexedLoad(
N))
4013 }
else if (tryARMIndexedLoad(
N))
4019 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(
N))
4026 ReplaceUses(
N, New);
4027 CurDAG->RemoveDeadNode(
N);
4032 N->getOperand(1),
N->getOperand(2),
4034 ReplaceUses(
N, New);
4035 CurDAG->RemoveDeadNode(
N);
4039 SDValue Ops[] = {
N->getOperand(1),
4042 unsigned Opc = ARM::t2LoopEnd;
4044 ReplaceUses(
N, New);
4045 CurDAG->RemoveDeadNode(
N);
4052 const SDValue &Chain =
N->getOperand(0);
4054 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4055 if (RegOffset != CurDAG->getRegister(0,
MVT::i32)) {
4061 RegOffset = CurDAG->getRegister(0,
MVT::i32);
4063 SDValue Ops[] = {
Base, RegOffset, ImmOffset, Chain};
4064 SDNode *
New = CurDAG->getMachineNode(ARM::LOADDUAL, dl,
4070 transferMemOperands(
N, New);
4074 CurDAG->RemoveDeadNode(
N);
4081 const SDValue &Chain =
N->getOperand(0);
4083 SelectAddrMode3(
Addr,
Base, RegOffset, ImmOffset);
4084 if (RegOffset != CurDAG->getRegister(0,
MVT::i32)) {
4090 RegOffset = CurDAG->getRegister(0,
MVT::i32);
4096 transferMemOperands(
N, New);
4098 CurDAG->RemoveDeadNode(
N);
4102 SDValue Ops[] = {
N->getOperand(1),
4106 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4108 ReplaceUses(
N, Dec);
4109 CurDAG->RemoveDeadNode(
N);
4125 unsigned Opc = Subtarget->isThumb() ?
4126 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
4136 unsigned CC = (
unsigned) cast<ConstantSDNode>(N2)->getZExtValue();
4141 uint64_t ID = cast<ConstantSDNode>(
Int->getOperand(1))->getZExtValue();
4144 if (
ID == Intrinsic::loop_decrement_reg) {
4147 cast<ConstantSDNode>(
Int.getOperand(3))->getZExtValue(), dl,
4152 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4155 ReplaceUses(
Int.getNode(), LoopDec);
4159 CurDAG->getMachineNode(ARM::t2LoopEnd, dl,
MVT::Other, EndArgs);
4161 ReplaceUses(
N, LoopEnd);
4162 CurDAG->RemoveDeadNode(
N);
4163 CurDAG->RemoveDeadNode(InFlag.
getNode());
4164 CurDAG->RemoveDeadNode(
Int.getNode());
4169 bool SwitchEQNEToPLMI;
4170 SelectCMPZ(InFlag.
getNode(), SwitchEQNEToPLMI);
4171 InFlag =
N->getOperand(4);
4173 if (SwitchEQNEToPLMI) {
4187 SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
4191 if (
N->getNumValues() == 2) {
4193 ReplaceUses(
SDValue(
N, 1), InFlag);
4197 CurDAG->RemoveDeadNode(
N);
4207 auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(1).getNode());
4208 if (
C &&
C->getSExtValue() < 0 && Subtarget->isThumb()) {
4209 int64_t Addend = -
C->getSExtValue();
4216 if (Addend < 1<<8) {
4220 CurDAG->getRegister(0,
MVT::i32) };
4221 Add = CurDAG->getMachineNode(ARM::t2ADDri, dl,
MVT::i32, Ops);
4223 unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
4225 CurDAG->getTargetConstant(Addend, dl,
MVT::i32),
4227 Add = CurDAG->getMachineNode(Opc, dl,
MVT::i32, Ops);
4243 bool SwitchEQNEToPLMI;
4244 SelectCMPZ(InFlag.
getNode(), SwitchEQNEToPLMI);
4246 if (SwitchEQNEToPLMI) {
4261 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), NewARMcc,
4262 N->getOperand(3),
N->getOperand(4)};
4271 EVT VT =
N->getValueType(0);
4273 unsigned Opc64[] = {ARM::VZIPd8, ARM::VZIPd16, ARM::VTRNd32};
4274 unsigned Opc128[] = {ARM::VZIPq8, ARM::VZIPq16, ARM::VZIPq32};
4278 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4279 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4283 EVT VT =
N->getValueType(0);
4285 unsigned Opc64[] = {ARM::VUZPd8, ARM::VUZPd16, ARM::VTRNd32};
4286 unsigned Opc128[] = {ARM::VUZPq8, ARM::VUZPq16, ARM::VUZPq32};
4290 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4291 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4295 EVT VT =
N->getValueType(0);
4296 unsigned Opc64[] = {ARM::VTRNd8, ARM::VTRNd16, ARM::VTRNd32};
4297 unsigned Opc128[] = {ARM::VTRNq8, ARM::VTRNq16, ARM::VTRNq32};
4301 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1), Pred, PredReg};
4302 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4306 EVT VecVT =
N->getValueType(0);
4310 assert(NumElts == 2 &&
"unexpected type for BUILD_VECTOR");
4312 N, createDRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4318 N, createSRegPairNode(VecVT,
N->getOperand(0),
N->getOperand(1)));
4321 assert(NumElts == 4 &&
"unexpected type for BUILD_VECTOR");
4323 createQuadSRegsNode(VecVT,
N->getOperand(0),
N->getOperand(1),
4324 N->getOperand(2),
N->getOperand(3)));
4329 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
4331 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
4333 SelectVLDDup(
N,
false,
false, 1, DOpcodes, QOpcodes);
4338 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4340 SelectVLDDup(
N,
false,
false, 2, Opcodes);
4345 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
4346 ARM::VLD3DUPd16Pseudo,
4347 ARM::VLD3DUPd32Pseudo };
4348 SelectVLDDup(
N,
false,
false, 3, Opcodes);
4353 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
4354 ARM::VLD4DUPd16Pseudo,
4355 ARM::VLD4DUPd32Pseudo };
4356 SelectVLDDup(
N,
false,
false, 4, Opcodes);
4361 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
4362 ARM::VLD1DUPd16wb_fixed,
4363 ARM::VLD1DUPd32wb_fixed };
4364 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
4365 ARM::VLD1DUPq16wb_fixed,
4366 ARM::VLD1DUPq32wb_fixed };
4367 SelectVLDDup(
N,
false,
true, 1, DOpcodes, QOpcodes);
4372 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8wb_fixed,
4373 ARM::VLD2DUPd16wb_fixed,
4374 ARM::VLD2DUPd32wb_fixed,
4375 ARM::VLD1q64wb_fixed };
4376 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4377 ARM::VLD2DUPq16EvenPseudo,
4378 ARM::VLD2DUPq32EvenPseudo };
4379 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudoWB_fixed,
4380 ARM::VLD2DUPq16OddPseudoWB_fixed,
4381 ARM::VLD2DUPq32OddPseudoWB_fixed };
4382 SelectVLDDup(
N,
false,
true, 2, DOpcodes, QOpcodes0, QOpcodes1);
4387 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
4388 ARM::VLD3DUPd16Pseudo_UPD,
4389 ARM::VLD3DUPd32Pseudo_UPD,
4390 ARM::VLD1d64TPseudoWB_fixed };
4391 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4392 ARM::VLD3DUPq16EvenPseudo,
4393 ARM::VLD3DUPq32EvenPseudo };
4394 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo_UPD,
4395 ARM::VLD3DUPq16OddPseudo_UPD,
4396 ARM::VLD3DUPq32OddPseudo_UPD };
4397 SelectVLDDup(
N,
false,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4402 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
4403 ARM::VLD4DUPd16Pseudo_UPD,
4404 ARM::VLD4DUPd32Pseudo_UPD,
4405 ARM::VLD1d64QPseudoWB_fixed };
4406 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4407 ARM::VLD4DUPq16EvenPseudo,
4408 ARM::VLD4DUPq32EvenPseudo };
4409 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo_UPD,
4410 ARM::VLD4DUPq16OddPseudo_UPD,
4411 ARM::VLD4DUPq32OddPseudo_UPD };
4412 SelectVLDDup(
N,
false,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4417 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
4418 ARM::VLD1d16wb_fixed,
4419 ARM::VLD1d32wb_fixed,
4420 ARM::VLD1d64wb_fixed };
4421 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
4422 ARM::VLD1q16wb_fixed,
4423 ARM::VLD1q32wb_fixed,
4424 ARM::VLD1q64wb_fixed };
4425 SelectVLD(
N,
true, 1, DOpcodes, QOpcodes,
nullptr);
4430 if (Subtarget->hasNEON()) {
4431 static const uint16_t DOpcodes[] = {
4432 ARM::VLD2d8wb_fixed, ARM::VLD2d16wb_fixed, ARM::VLD2d32wb_fixed,
4433 ARM::VLD1q64wb_fixed};
4434 static const uint16_t QOpcodes[] = {ARM::VLD2q8PseudoWB_fixed,
4435 ARM::VLD2q16PseudoWB_fixed,
4436 ARM::VLD2q32PseudoWB_fixed};
4437 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4439 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8,
4440 ARM::MVE_VLD21_8_wb};
4441 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
4442 ARM::MVE_VLD21_16_wb};
4443 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
4444 ARM::MVE_VLD21_32_wb};
4445 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4446 SelectMVE_VLD(
N, 2, Opcodes,
true);
4452 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
4453 ARM::VLD3d16Pseudo_UPD,
4454 ARM::VLD3d32Pseudo_UPD,
4455 ARM::VLD1d64TPseudoWB_fixed};
4456 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4457 ARM::VLD3q16Pseudo_UPD,
4458 ARM::VLD3q32Pseudo_UPD };
4459 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
4460 ARM::VLD3q16oddPseudo_UPD,
4461 ARM::VLD3q32oddPseudo_UPD };
4462 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4467 if (Subtarget->hasNEON()) {
4468 static const uint16_t DOpcodes[] = {
4469 ARM::VLD4d8Pseudo_UPD, ARM::VLD4d16Pseudo_UPD, ARM::VLD4d32Pseudo_UPD,
4470 ARM::VLD1d64QPseudoWB_fixed};
4471 static const uint16_t QOpcodes0[] = {ARM::VLD4q8Pseudo_UPD,
4472 ARM::VLD4q16Pseudo_UPD,
4473 ARM::VLD4q32Pseudo_UPD};
4474 static const uint16_t QOpcodes1[] = {ARM::VLD4q8oddPseudo_UPD,
4475 ARM::VLD4q16oddPseudo_UPD,
4476 ARM::VLD4q32oddPseudo_UPD};
4477 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4479 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
4481 ARM::MVE_VLD43_8_wb};
4482 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
4484 ARM::MVE_VLD43_16_wb};
4485 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
4487 ARM::MVE_VLD43_32_wb};
4488 static const uint16_t *
const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4489 SelectMVE_VLD(
N, 4, Opcodes,
true);
4495 if (Subtarget->hasNEON()) {
4496 static const uint16_t DOpcodes[] = {
4497 ARM::VLD1q8wb_fixed, ARM::VLD1q16wb_fixed, ARM::VLD1q32wb_fixed,
4498 ARM::VLD1q64wb_fixed};
4499 static const uint16_t QOpcodes[] = {
4500 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4501 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4502 SelectVLD(
N,
true, 2, DOpcodes, QOpcodes,
nullptr);
4509 if (Subtarget->hasNEON()) {
4510 static const uint16_t DOpcodes[] = {
4511 ARM::VLD1d8TPseudoWB_fixed, ARM::VLD1d16TPseudoWB_fixed,
4512 ARM::VLD1d32TPseudoWB_fixed, ARM::VLD1d64TPseudoWB_fixed};
4513 static const uint16_t QOpcodes0[] = {
4514 ARM::VLD1q8LowTPseudo_UPD, ARM::VLD1q16LowTPseudo_UPD,
4515 ARM::VLD1q32LowTPseudo_UPD, ARM::VLD1q64LowTPseudo_UPD};
4516 static const uint16_t QOpcodes1[] = {
4517 ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1q16HighTPseudo_UPD,
4518 ARM::VLD1q32HighTPseudo_UPD, ARM::VLD1q64HighTPseudo_UPD};
4519 SelectVLD(
N,
true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4526 if (Subtarget->hasNEON()) {
4527 static const uint16_t DOpcodes[] = {
4528 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4529 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4530 static const uint16_t QOpcodes0[] = {
4531 ARM::VLD1q8LowQPseudo_UPD, ARM::VLD1q16LowQPseudo_UPD,
4532 ARM::VLD1q32LowQPseudo_UPD, ARM::VLD1q64LowQPseudo_UPD};
4533 static const uint16_t QOpcodes1[] = {
4534 ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1q16HighQPseudo_UPD,
4535 ARM::VLD1q32HighQPseudo_UPD, ARM::VLD1q64HighQPseudo_UPD};
4536 SelectVLD(
N,
true, 4, DOpcodes, QOpcodes0, QOpcodes1);