22#include "llvm/IR/IntrinsicsAArch64.h"
31#define DEBUG_TYPE "aarch64-isel"
32#define PASS_NAME "AArch64 Instruction Selection"
47 AArch64DAGToDAGISel() =
delete;
64 std::vector<SDValue> &OutOps)
override;
66 template <
signed Low,
signed High,
signed Scale>
74 return SelectShiftedRegister(
N,
false, Reg, Shift);
77 return SelectShiftedRegister(
N,
true, Reg, Shift);
80 return SelectAddrModeIndexed7S(
N, 1,
Base, OffImm);
83 return SelectAddrModeIndexed7S(
N, 2,
Base, OffImm);
86 return SelectAddrModeIndexed7S(
N, 4,
Base, OffImm);
89 return SelectAddrModeIndexed7S(
N, 8,
Base, OffImm);
92 return SelectAddrModeIndexed7S(
N, 16,
Base, OffImm);
95 return SelectAddrModeIndexedBitWidth(
N,
true, 9, 16,
Base, OffImm);
98 return SelectAddrModeIndexedBitWidth(
N,
false, 6, 16,
Base, OffImm);
101 return SelectAddrModeIndexed(
N, 1,
Base, OffImm);
104 return SelectAddrModeIndexed(
N, 2,
Base, OffImm);
107 return SelectAddrModeIndexed(
N, 4,
Base, OffImm);
110 return SelectAddrModeIndexed(
N, 8,
Base, OffImm);
113 return SelectAddrModeIndexed(
N, 16,
Base, OffImm);
116 return SelectAddrModeUnscaled(
N, 1,
Base, OffImm);
119 return SelectAddrModeUnscaled(
N, 2,
Base, OffImm);
122 return SelectAddrModeUnscaled(
N, 4,
Base, OffImm);
125 return SelectAddrModeUnscaled(
N, 8,
Base, OffImm);
128 return SelectAddrModeUnscaled(
N, 16,
Base, OffImm);
130 template <
unsigned Size,
unsigned Max>
134 bool Found = SelectAddrModeIndexed(
N,
Size,
Base, OffImm);
136 if (
auto *CI = dyn_cast<ConstantSDNode>(OffImm)) {
137 int64_t
C = CI->getSExtValue();
145 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
152 return SelectAddrModeWRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
158 return SelectAddrModeXRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
163 N =
N->getOperand(0);
165 !isa<ConstantSDNode>(
N->getOperand(1)))
167 EVT VT =
N->getValueType(0);
168 EVT LVT =
N->getOperand(0).getValueType();
169 unsigned Index =
N->getConstantOperandVal(1);
173 Res =
N->getOperand(0);
181 EVT VT =
Op.getValueType();
182 unsigned ShtAmt =
N->getConstantOperandVal(1);
189 Op.getOperand(1).getConstantOperandVal(0)
190 <<
Op.getOperand(1).getConstantOperandVal(1));
192 isa<ConstantSDNode>(
Op.getOperand(1).getOperand(0)))
194 Op.getOperand(1).getConstantOperandVal(0));
198 if (Imm != 1ULL << (ShtAmt - 1))
201 Res1 =
Op.getOperand(0);
202 Res2 = CurDAG->getTargetConstant(ShtAmt,
SDLoc(
N), MVT::i32);
206 bool SelectDupZeroOrUndef(
SDValue N) {
207 switch(
N->getOpcode()) {
212 auto Opnd0 =
N->getOperand(0);
227 switch(
N->getOpcode()) {
230 auto Opnd0 =
N->getOperand(0);
242 bool SelectDupNegativeZero(
SDValue N) {
243 switch(
N->getOpcode()) {
247 return Const && Const->isZero() && Const->isNegative();
254 template<MVT::SimpleValueType VT>
256 return SelectSVEAddSubImm(
N, VT, Imm, Shift);
259 template <MVT::SimpleValueType VT,
bool Negate>
261 return SelectSVEAddSubSSatImm(
N, VT, Imm, Shift, Negate);
264 template <MVT::SimpleValueType VT>
266 return SelectSVECpyDupImm(
N, VT, Imm, Shift);
269 template <MVT::SimpleValueType VT,
bool Invert = false>
271 return SelectSVELogicalImm(
N, VT, Imm, Invert);
274 template <MVT::SimpleValueType VT>
276 return SelectSVEArithImm(
N, VT, Imm);
279 template <
unsigned Low,
unsigned High,
bool AllowSaturation = false>
281 return SelectSVEShiftImm(
N,
Low,
High, AllowSaturation, Imm);
288 EVT EltVT =
N->getValueType(0).getVectorElementType();
289 return SelectSVEShiftImm(
N->getOperand(0), 1,
295 template<
signed Min,
signed Max,
signed Scale,
bool Shift>
297 if (!isa<ConstantSDNode>(
N))
300 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
302 MulImm = 1LL << MulImm;
304 if ((MulImm % std::abs(Scale)) != 0)
308 if ((MulImm >= Min) && (MulImm <= Max)) {
309 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
316 template <
signed Max,
signed Scale>
318 if (!isa<ConstantSDNode>(
N))
321 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
323 if (MulImm >= 0 && MulImm <= Max) {
325 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
332 template <
unsigned BaseReg,
unsigned Max>
334 if (
auto *CI = dyn_cast<ConstantSDNode>(
N)) {
340 Imm = CurDAG->getRegister(BaseReg +
C, MVT::Other);
363 const unsigned SubRegs[]);
365 void SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
367 bool tryIndexedLoad(
SDNode *
N);
369 void SelectPtrauthAuth(
SDNode *
N);
370 void SelectPtrauthResign(
SDNode *
N);
372 bool trySelectStackSlotTagP(
SDNode *
N);
375 void SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
377 void SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
379 void SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
380 void SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
381 void SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
382 unsigned Opc_rr,
unsigned Opc_ri,
383 bool IsIntr =
false);
384 void SelectContiguousMultiVectorLoad(
SDNode *
N,
unsigned NumVecs,
385 unsigned Scale,
unsigned Opc_ri,
387 void SelectDestructiveMultiIntrinsic(
SDNode *
N,
unsigned NumVecs,
388 bool IsZmMulti,
unsigned Opcode,
389 bool HasPred =
false);
390 void SelectPExtPair(
SDNode *
N,
unsigned Opc);
391 void SelectWhilePair(
SDNode *
N,
unsigned Opc);
392 void SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
393 void SelectClamp(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
394 void SelectUnaryMultiIntrinsic(
SDNode *
N,
unsigned NumOutVecs,
395 bool IsTupleInput,
unsigned Opc);
396 void SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
398 template <
unsigned MaxIdx,
unsigned Scale>
399 void SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
unsigned BaseReg,
401 void SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
402 unsigned Op,
unsigned MaxIdx,
unsigned Scale,
403 unsigned BaseReg = 0);
406 template <
int64_t Min,
int64_t Max>
410 template <
unsigned Scale>
412 return SelectSVERegRegAddrMode(
N, Scale,
Base,
Offset);
415 void SelectMultiVectorLuti(
SDNode *
Node,
unsigned NumOutVecs,
unsigned Opc,
418 template <
unsigned MaxIdx,
unsigned Scale>
423 void SelectStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
424 void SelectPostStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
425 void SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
426 void SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
427 void SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
428 unsigned Opc_rr,
unsigned Opc_ri);
429 std::tuple<unsigned, SDValue, SDValue>
430 findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
unsigned Opc_ri,
434 bool tryBitfieldExtractOp(
SDNode *
N);
435 bool tryBitfieldExtractOpFromSExt(
SDNode *
N);
436 bool tryBitfieldInsertOp(
SDNode *
N);
437 bool tryBitfieldInsertInZeroOp(
SDNode *
N);
438 bool tryShiftAmountMod(
SDNode *
N);
440 bool tryReadRegister(
SDNode *
N);
441 bool tryWriteRegister(
SDNode *
N);
443 bool trySelectCastFixedLengthToScalableVector(
SDNode *
N);
444 bool trySelectCastScalableToFixedLengthVector(
SDNode *
N);
449#include "AArch64GenDAGISel.inc"
457 return SelectAddrModeIndexedBitWidth(
N,
true, 7,
Size,
Base, OffImm);
459 bool SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
unsigned BW,
472 bool isWorthFoldingALU(
SDValue V,
bool LSL =
false)
const;
473 bool isWorthFoldingAddr(
SDValue V,
unsigned Size)
const;
474 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
477 template<
unsigned RegW
idth>
479 return SelectCVTFixedPosOperand(
N, FixedPos, RegWidth);
482 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
484 template<
unsigned RegW
idth>
486 return SelectCVTFixedPosRecipOperand(
N, FixedPos, RegWidth);
492 bool SelectCMP_SWAP(
SDNode *
N);
502 bool AllowSaturation,
SDValue &Imm);
510 bool SelectAllActivePredicate(
SDValue N);
520 ID, std::make_unique<AArch64DAGToDAGISel>(tm, OptLevel)) {}
524char AArch64DAGToDAGISelLegacy::ID = 0;
532 Imm =
C->getZExtValue();
549 return N->getOpcode() == Opc &&
560 return Imm == ImmExpected;
564bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
566 std::vector<SDValue> &OutOps) {
567 switch(ConstraintID) {
570 case InlineAsm::ConstraintCode::m:
571 case InlineAsm::ConstraintCode::o:
572 case InlineAsm::ConstraintCode::Q:
578 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(), dl, MVT::i64);
580 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
581 dl,
Op.getValueType(),
583 OutOps.push_back(NewOp);
599 if (!isa<ConstantSDNode>(
N.getNode()))
602 uint64_t Immed =
N.getNode()->getAsZExtVal();
605 if (Immed >> 12 == 0) {
607 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
615 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
616 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
629 if (!isa<ConstantSDNode>(
N.getNode()))
633 uint64_t Immed =
N.getNode()->getAsZExtVal();
641 if (
N.getValueType() == MVT::i32)
644 Immed = ~Immed + 1ULL;
645 if (Immed & 0xFFFFFFFFFF000000ULL)
648 Immed &= 0xFFFFFFULL;
649 return SelectArithImmed(CurDAG->getConstant(Immed,
SDLoc(
N), MVT::i32), Val,
656 switch (
N.getOpcode()) {
675 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
678 unsigned ShiftVal = CSD->getZExtValue();
687 if (!isa<MemSDNode>(*UI))
689 if (!isa<MemSDNode>(*UII))
696bool AArch64DAGToDAGISel::isWorthFoldingAddr(
SDValue V,
unsigned Size)
const {
699 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
704 if (Subtarget->hasAddrLSLSlow14() && (
Size == 2 ||
Size == 16))
726bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(
SDValue N,
SDValue &Reg,
728 EVT VT =
N.getValueType();
729 if (VT != MVT::i32 && VT != MVT::i64)
732 if (
N->getOpcode() !=
ISD::AND || !
N->hasOneUse())
738 unsigned LHSOpcode =
LHS->getOpcode();
752 unsigned LowZBits, MaskLen;
756 unsigned BitWidth =
N.getValueSizeInBits();
763 if (LowZBits <= ShiftAmtC || (
BitWidth != LowZBits + MaskLen))
766 NewShiftC = LowZBits - ShiftAmtC;
767 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
773 NewShiftC = LowZBits + ShiftAmtC;
786 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
788 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
792 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC,
DL, VT);
794 Reg =
SDValue(CurDAG->getMachineNode(NewShiftOp,
DL, VT,
LHS->getOperand(0),
795 NewShiftAmt, BitWidthMinus1),
798 Shift = CurDAG->getTargetConstant(ShVal,
DL, MVT::i32);
810 SrcVT = cast<VTSDNode>(
N.getOperand(1))->getVT();
812 SrcVT =
N.getOperand(0).getValueType();
814 if (!IsLoadStore && SrcVT == MVT::i8)
816 else if (!IsLoadStore && SrcVT == MVT::i16)
818 else if (SrcVT == MVT::i32)
820 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
825 EVT SrcVT =
N.getOperand(0).getValueType();
826 if (!IsLoadStore && SrcVT == MVT::i8)
828 else if (!IsLoadStore && SrcVT == MVT::i16)
830 else if (SrcVT == MVT::i32)
832 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
860bool AArch64DAGToDAGISel::isWorthFoldingALU(
SDValue V,
bool LSL)
const {
863 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
868 if (LSL && Subtarget->hasALULSLFast() &&
V.getOpcode() ==
ISD::SHL &&
869 V.getConstantOperandVal(1) <= 4 &&
882bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
884 if (SelectShiftedRegisterFromAnd(
N, Reg, Shift))
894 unsigned BitSize =
N.getValueSizeInBits();
895 unsigned Val =
RHS->getZExtValue() & (BitSize - 1);
898 Reg =
N.getOperand(0);
899 Shift = CurDAG->getTargetConstant(ShVal,
SDLoc(
N), MVT::i32);
900 return isWorthFoldingALU(
N,
true);
911 if (
N.getValueType() == MVT::i32)
919template<
signed Low,
signed High,
signed Scale>
921 if (!isa<ConstantSDNode>(
N))
924 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
925 if ((MulImm % std::abs(Scale)) == 0) {
926 int64_t RDVLImm = MulImm / Scale;
927 if ((RDVLImm >=
Low) && (RDVLImm <=
High)) {
928 Imm = CurDAG->getTargetConstant(RDVLImm,
SDLoc(
N), MVT::i32);
938bool AArch64DAGToDAGISel::SelectArithExtendedRegister(
SDValue N,
SDValue &Reg,
940 unsigned ShiftVal = 0;
955 Reg =
N.getOperand(0).getOperand(0);
961 Reg =
N.getOperand(0);
966 unsigned Opc =
N.getOpcode();
967 return Opc !=
ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
984 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
986 return isWorthFoldingALU(
N);
991bool AArch64DAGToDAGISel::SelectArithUXTXRegister(
SDValue N,
SDValue &Reg,
993 unsigned ShiftVal = 0;
1007 Reg =
N.getOperand(0);
1008 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
1010 return isWorthFoldingALU(
N);
1019 for (
auto *
Use :
N->uses()) {
1046bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
1047 unsigned BW,
unsigned Size,
1054 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1056 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1062 if (CurDAG->isBaseWithConstantOffset(
N)) {
1063 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1065 int64_t RHSC =
RHS->getSExtValue();
1067 int64_t
Range = 0x1LL << (BW - 1);
1069 if ((RHSC & (
Size - 1)) == 0 && RHSC >= -(
Range << Scale) &&
1070 RHSC < (
Range << Scale)) {
1071 Base =
N.getOperand(0);
1073 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1076 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1085 if ((RHSC & (
Size - 1)) == 0 && RHSC < (
Range << Scale)) {
1086 Base =
N.getOperand(0);
1088 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1091 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1102 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1109bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
1115 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1117 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1123 dyn_cast<GlobalAddressSDNode>(
N.getOperand(1).getNode());
1124 Base =
N.getOperand(0);
1125 OffImm =
N.getOperand(1);
1134 if (CurDAG->isBaseWithConstantOffset(
N)) {
1135 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1136 int64_t RHSC = (int64_t)
RHS->getZExtValue();
1139 Base =
N.getOperand(0);
1141 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1144 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1152 if (SelectAddrModeUnscaled(
N,
Size,
Base, OffImm))
1160 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1169bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
1172 if (!CurDAG->isBaseWithConstantOffset(
N))
1174 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1175 int64_t RHSC =
RHS->getSExtValue();
1176 if (RHSC >= -256 && RHSC < 256) {
1177 Base =
N.getOperand(0);
1179 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1181 Base = CurDAG->getTargetFrameIndex(
1184 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i64);
1194 CurDAG->
getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1201bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
1221 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1227 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1230 return isWorthFoldingAddr(
N,
Size);
1233bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
1245 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
1253 if (!isa<MemSDNode>(*UI))
1258 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1261 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1262 SelectExtendedSHL(RHS,
Size,
true,
Offset, SignExtend)) {
1264 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1269 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1270 SelectExtendedSHL(LHS,
Size,
true,
Offset, SignExtend)) {
1272 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1277 DoShift = CurDAG->getTargetConstant(
false, dl, MVT::i32);
1281 if (IsExtendedRegisterWorthFolding &&
1288 if (isWorthFoldingAddr(LHS,
Size))
1293 if (IsExtendedRegisterWorthFolding &&
1300 if (isWorthFoldingAddr(RHS,
Size))
1312 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1315 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1317 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1318 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1322bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
1337 if (!isa<MemSDNode>(*UI))
1352 if (isa<ConstantSDNode>(RHS)) {
1353 int64_t ImmOff = (int64_t)
RHS->getAsZExtVal();
1363 CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
1366 N = CurDAG->getNode(
ISD::ADD,
DL, MVT::i64, LHS, MOVIV);
1370 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1373 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1374 SelectExtendedSHL(RHS,
Size,
false,
Offset, SignExtend)) {
1376 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1381 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1382 SelectExtendedSHL(LHS,
Size,
false,
Offset, SignExtend)) {
1384 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1391 SignExtend = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1392 DoShift = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1398 static const unsigned RegClassIDs[] = {
1399 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1400 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1401 AArch64::dsub2, AArch64::dsub3};
1407 static const unsigned RegClassIDs[] = {
1408 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1409 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1410 AArch64::qsub2, AArch64::qsub3};
1416 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1417 AArch64::ZPR3RegClassID,
1418 AArch64::ZPR4RegClassID};
1419 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1420 AArch64::zsub2, AArch64::zsub3};
1430 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1431 AArch64::ZPR4Mul4RegClassID};
1432 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1433 AArch64::zsub2, AArch64::zsub3};
1438 const unsigned RegClassIDs[],
1439 const unsigned SubRegs[]) {
1442 if (Regs.
size() == 1)
1453 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2],
DL, MVT::i32));
1456 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1458 Ops.
push_back(CurDAG->getTargetConstant(SubRegs[i],
DL, MVT::i32));
1462 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped, Ops);
1466void AArch64DAGToDAGISel::SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1469 EVT VT =
N->getValueType(0);
1471 unsigned ExtOff = isExt;
1474 unsigned Vec0Off = ExtOff + 1;
1476 N->op_begin() + Vec0Off + NumVecs);
1483 Ops.
push_back(
N->getOperand(NumVecs + ExtOff + 1));
1484 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1487static std::tuple<SDValue, SDValue>
1507 auto *ConstDiscN = dyn_cast<ConstantSDNode>(ConstDisc);
1508 if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue()))
1513 AddrDisc = DAG->
getRegister(AArch64::XZR, MVT::i64);
1515 return std::make_tuple(
1520void AArch64DAGToDAGISel::SelectPtrauthAuth(
SDNode *
N) {
1525 SDValue AUTDisc =
N->getOperand(3);
1527 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1528 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
1530 SDValue AUTAddrDisc, AUTConstDisc;
1531 std::tie(AUTConstDisc, AUTAddrDisc) =
1534 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
1535 AArch64::X16, Val,
SDValue());
1536 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.
getValue(1)};
1538 SDNode *AUT = CurDAG->getMachineNode(AArch64::AUT,
DL, MVT::i64, Ops);
1539 ReplaceNode(
N, AUT);
1543void AArch64DAGToDAGISel::SelectPtrauthResign(
SDNode *
N) {
1548 SDValue AUTDisc =
N->getOperand(3);
1550 SDValue PACDisc =
N->getOperand(5);
1552 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1553 unsigned PACKeyC = cast<ConstantSDNode>(PACKey)->getZExtValue();
1555 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
1556 PACKey = CurDAG->getTargetConstant(PACKeyC,
DL, MVT::i64);
1558 SDValue AUTAddrDisc, AUTConstDisc;
1559 std::tie(AUTConstDisc, AUTAddrDisc) =
1562 SDValue PACAddrDisc, PACConstDisc;
1563 std::tie(PACConstDisc, PACAddrDisc) =
1566 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
1567 AArch64::X16, Val,
SDValue());
1569 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1570 PACConstDisc, PACAddrDisc, X16Copy.
getValue(1)};
1572 SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC,
DL, MVT::i64, Ops);
1573 ReplaceNode(
N, AUTPAC);
1577bool AArch64DAGToDAGISel::tryIndexedLoad(
SDNode *
N) {
1579 if (
LD->isUnindexed())
1581 EVT VT =
LD->getMemoryVT();
1582 EVT DstVT =
N->getValueType(0);
1589 unsigned Opcode = 0;
1592 bool InsertTo64 =
false;
1594 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1595 else if (VT == MVT::i32) {
1597 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1599 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1601 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1607 }
else if (VT == MVT::i16) {
1609 if (DstVT == MVT::i64)
1610 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1612 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1614 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1615 InsertTo64 = DstVT == MVT::i64;
1620 }
else if (VT == MVT::i8) {
1622 if (DstVT == MVT::i64)
1623 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1625 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1627 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1628 InsertTo64 = DstVT == MVT::i64;
1633 }
else if (VT == MVT::f16) {
1634 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1635 }
else if (VT == MVT::bf16) {
1636 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1637 }
else if (VT == MVT::f32) {
1638 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1640 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1642 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1650 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1652 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1657 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {
MemOp});
1662 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1664 SDValue(CurDAG->getMachineNode(
1665 AArch64::SUBREG_TO_REG, dl, MVT::i64,
1666 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1671 ReplaceUses(
SDValue(
N, 0), LoadedVal);
1674 CurDAG->RemoveDeadNode(
N);
1678void AArch64DAGToDAGISel::SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1679 unsigned SubRegIdx) {
1681 EVT VT =
N->getValueType(0);
1687 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1689 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1691 for (
unsigned i = 0; i < NumVecs; ++i)
1693 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1699 if (
auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(
N)) {
1701 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
1704 CurDAG->RemoveDeadNode(
N);
1707void AArch64DAGToDAGISel::SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
1708 unsigned Opc,
unsigned SubRegIdx) {
1710 EVT VT =
N->getValueType(0);
1717 const EVT ResTys[] = {MVT::i64,
1718 MVT::Untyped, MVT::Other};
1720 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1728 ReplaceUses(
SDValue(
N, 0), SuperReg);
1730 for (
unsigned i = 0; i < NumVecs; ++i)
1732 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1736 CurDAG->RemoveDeadNode(
N);
1742std::tuple<unsigned, SDValue, SDValue>
1743AArch64DAGToDAGISel::findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
1749 SDValue NewOffset = OldOffset;
1751 const bool IsRegImm = SelectAddrModeIndexedSVE<-8, 7>(
1752 N, OldBase, NewBase, NewOffset);
1756 const bool IsRegReg =
1757 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
1760 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
1773template <SelectTypeKind Kind>
1785 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
1790 if (EltVT != MVT::i1)
1794 if (EltVT == MVT::bf16)
1796 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
1826void AArch64DAGToDAGISel::SelectPExtPair(
SDNode *
N,
unsigned Opc) {
1829 if (
Imm->getZExtValue() > 1)
1833 EVT VT =
N->getValueType(0);
1834 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1835 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1838 for (
unsigned I = 0;
I < 2; ++
I)
1839 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1840 AArch64::psub0 +
I,
DL, VT, SuperReg));
1842 CurDAG->RemoveDeadNode(
N);
1845void AArch64DAGToDAGISel::SelectWhilePair(
SDNode *
N,
unsigned Opc) {
1847 EVT VT =
N->getValueType(0);
1849 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1851 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1854 for (
unsigned I = 0;
I < 2; ++
I)
1855 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1856 AArch64::psub0 +
I,
DL, VT, SuperReg));
1858 CurDAG->RemoveDeadNode(
N);
1861void AArch64DAGToDAGISel::SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
1863 EVT VT =
N->getValueType(0);
1865 SDValue Ops = createZTuple(Regs);
1869 for (
unsigned i = 0; i < NumVecs; ++i)
1870 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1871 AArch64::zsub0 + i,
DL, VT, SuperReg));
1873 CurDAG->RemoveDeadNode(
N);
1876void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(
SDNode *
N,
1881 assert(Opcode != 0 &&
"Unexpected opcode");
1884 EVT VT =
N->getValueType(0);
1885 unsigned FirstVecIdx = HasPred ? 2 : 1;
1887 auto GetMultiVecOperand = [=](
unsigned StartIdx) {
1889 N->op_begin() + StartIdx + NumVecs);
1890 return createZMulTuple(Regs);
1893 SDValue Zdn = GetMultiVecOperand(FirstVecIdx);
1897 Zm = GetMultiVecOperand(NumVecs + FirstVecIdx);
1899 Zm =
N->getOperand(NumVecs + FirstVecIdx);
1903 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped,
1904 N->getOperand(1), Zdn, Zm);
1906 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped, Zdn, Zm);
1908 for (
unsigned i = 0; i < NumVecs; ++i)
1909 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1910 AArch64::zsub0 + i,
DL, VT, SuperReg));
1912 CurDAG->RemoveDeadNode(
N);
1915void AArch64DAGToDAGISel::SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
1916 unsigned Scale,
unsigned Opc_ri,
1917 unsigned Opc_rr,
bool IsIntr) {
1918 assert(Scale < 5 &&
"Invalid scaling value.");
1920 EVT VT =
N->getValueType(0);
1926 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
1927 N, Opc_rr, Opc_ri,
N->getOperand(IsIntr ? 3 : 2),
1928 CurDAG->getTargetConstant(0,
DL, MVT::i64), Scale);
1930 SDValue Ops[] = {
N->getOperand(IsIntr ? 2 : 1),
1934 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1936 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1938 for (
unsigned i = 0; i < NumVecs; ++i)
1939 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1940 AArch64::zsub0 + i,
DL, VT, SuperReg));
1943 unsigned ChainIdx = NumVecs;
1945 CurDAG->RemoveDeadNode(
N);
1948void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(
SDNode *
N,
1953 assert(Scale < 4 &&
"Invalid scaling value.");
1955 EVT VT =
N->getValueType(0);
1963 findAddrModeSVELoadStore(
N, Opc_rr, Opc_ri,
Base,
Offset, Scale);
1969 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1971 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1973 for (
unsigned i = 0; i < NumVecs; ++i)
1974 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1975 AArch64::zsub0 + i,
DL, VT, SuperReg));
1978 unsigned ChainIdx = NumVecs;
1980 CurDAG->RemoveDeadNode(
N);
1983void AArch64DAGToDAGISel::SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
1985 if (
N->getValueType(0) != MVT::nxv4f32)
1987 SelectUnaryMultiIntrinsic(
N, NumVecs,
true, Opcode);
1990void AArch64DAGToDAGISel::SelectMultiVectorLuti(
SDNode *
Node,
1991 unsigned NumOutVecs,
1994 if (
Imm->getZExtValue() > MaxImm)
1998 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
2002 EVT VT =
Node->getValueType(0);
2005 CurDAG->getMachineNode(Opc,
DL, {MVT::Untyped, MVT::Other}, Ops);
2008 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
2009 ReplaceUses(
SDValue(
Node,
I), CurDAG->getTargetExtractSubreg(
2010 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2013 unsigned ChainIdx = NumOutVecs;
2015 CurDAG->RemoveDeadNode(
Node);
2018void AArch64DAGToDAGISel::SelectClamp(
SDNode *
N,
unsigned NumVecs,
2021 EVT VT =
N->getValueType(0);
2024 SDValue Zd = createZMulTuple(Regs);
2025 SDValue Zn =
N->getOperand(1 + NumVecs);
2026 SDValue Zm =
N->getOperand(2 + NumVecs);
2032 for (
unsigned i = 0; i < NumVecs; ++i)
2033 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
2034 AArch64::zsub0 + i,
DL, VT, SuperReg));
2036 CurDAG->RemoveDeadNode(
N);
2066template <
unsigned MaxIdx,
unsigned Scale>
2067void AArch64DAGToDAGISel::SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
2068 unsigned BaseReg,
unsigned Op) {
2069 unsigned TileNum = 0;
2070 if (BaseReg != AArch64::ZA)
2071 TileNum =
N->getConstantOperandVal(2);
2077 if (BaseReg == AArch64::ZA)
2078 SliceBase =
N->getOperand(2);
2080 SliceBase =
N->getOperand(3);
2082 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2088 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
2090 EVT VT =
N->getValueType(0);
2091 for (
unsigned I = 0;
I < NumVecs; ++
I)
2093 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2096 unsigned ChainIdx = NumVecs;
2098 CurDAG->RemoveDeadNode(
N);
2101void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
2102 unsigned Op,
unsigned MaxIdx,
2103 unsigned Scale,
unsigned BaseReg) {
2107 SDValue SliceBase =
N->getOperand(2);
2108 if (BaseReg != AArch64::ZA)
2109 SliceBase =
N->getOperand(3);
2112 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2119 if (BaseReg != AArch64::ZA )
2124 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
2126 EVT VT =
N->getValueType(0);
2127 for (
unsigned I = 0;
I < NumVecs; ++
I)
2129 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2133 unsigned ChainIdx = NumVecs;
2135 CurDAG->RemoveDeadNode(
N);
2138void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(
SDNode *
N,
2139 unsigned NumOutVecs,
2143 EVT VT =
N->getValueType(0);
2144 unsigned NumInVecs =
N->getNumOperands() - 1;
2148 assert((NumInVecs == 2 || NumInVecs == 4) &&
2149 "Don't know how to handle multi-register input!");
2151 N->op_begin() + 1 + NumInVecs);
2155 for (
unsigned I = 0;
I < NumInVecs;
I++)
2159 SDNode *Res = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
2162 for (
unsigned I = 0;
I < NumOutVecs;
I++)
2163 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2164 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2165 CurDAG->RemoveDeadNode(
N);
2168void AArch64DAGToDAGISel::SelectStore(
SDNode *
N,
unsigned NumVecs,
2171 EVT VT =
N->getOperand(2)->getValueType(0);
2178 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
N->getOperand(0)};
2179 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2183 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2188void AArch64DAGToDAGISel::SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
2189 unsigned Scale,
unsigned Opc_rr,
2195 SDValue RegSeq = createZTuple(Regs);
2200 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
2201 N, Opc_rr, Opc_ri,
N->getOperand(NumVecs + 3),
2202 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2204 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
2208 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2220 if (
auto FINode = dyn_cast<FrameIndexSDNode>(
N)) {
2221 int FI = FINode->getIndex();
2223 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
2230void AArch64DAGToDAGISel::SelectPostStore(
SDNode *
N,
unsigned NumVecs,
2233 EVT VT =
N->getOperand(2)->getValueType(0);
2234 const EVT ResTys[] = {MVT::i64,
2243 N->getOperand(NumVecs + 1),
2244 N->getOperand(NumVecs + 2),
2246 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2286void AArch64DAGToDAGISel::SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
2289 EVT VT =
N->getValueType(0);
2297 WidenVector(*CurDAG));
2301 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2303 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2305 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2306 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2307 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2311 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2312 AArch64::qsub2, AArch64::qsub3 };
2313 for (
unsigned i = 0; i < NumVecs; ++i) {
2314 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2321 CurDAG->RemoveDeadNode(
N);
2324void AArch64DAGToDAGISel::SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
2327 EVT VT =
N->getValueType(0);
2335 WidenVector(*CurDAG));
2339 const EVT ResTys[] = {MVT::i64,
2342 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2345 CurDAG->getTargetConstant(LaneNo, dl,
2347 N->getOperand(NumVecs + 2),
2348 N->getOperand(NumVecs + 3),
2350 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2362 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2363 AArch64::qsub2, AArch64::qsub3 };
2364 for (
unsigned i = 0; i < NumVecs; ++i) {
2365 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
2375 CurDAG->RemoveDeadNode(
N);
2378void AArch64DAGToDAGISel::SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
2381 EVT VT =
N->getOperand(2)->getValueType(0);
2389 WidenVector(*CurDAG));
2393 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2395 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2396 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2397 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
2401 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2406void AArch64DAGToDAGISel::SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
2409 EVT VT =
N->getOperand(2)->getValueType(0);
2417 WidenVector(*CurDAG));
2421 const EVT ResTys[] = {MVT::i64,
2424 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2426 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2427 N->getOperand(NumVecs + 2),
2428 N->getOperand(NumVecs + 3),
2430 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2434 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2441 unsigned &LSB,
unsigned &MSB,
2442 unsigned NumberOfIgnoredLowBits,
2443 bool BiggerPattern) {
2445 "N must be a AND operation to call this function");
2447 EVT VT =
N->getValueType(0);
2452 assert((VT == MVT::i32 || VT == MVT::i64) &&
2453 "Type checking must have been done before calling this function");
2467 const SDNode *Op0 =
N->getOperand(0).getNode();
2471 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
2474 if (AndImm & (AndImm + 1))
2477 bool ClampMSB =
false;
2497 ClampMSB = (VT == MVT::i32);
2498 }
else if (BiggerPattern) {
2504 Opd0 =
N->getOperand(0);
2510 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.
getSizeInBits())) {
2513 <<
": Found large shift immediate, this should not happen\n"));
2519 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(AndImm)
2520 : llvm::countr_one<uint64_t>(AndImm)) -
2527 MSB = MSB > 31 ? 31 : MSB;
2529 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2534 SDValue &Opd0,
unsigned &Immr,
2538 EVT VT =
N->getValueType(0);
2540 assert((VT == MVT::i32 || VT == MVT::i64) &&
2541 "Type checking must have been done before calling this function");
2545 Op =
Op->getOperand(0);
2546 VT =
Op->getValueType(0);
2555 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
2559 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2560 Opd0 =
Op.getOperand(0);
2562 Imms = ShiftImm + Width - 1;
2590 Opd0 =
N->getOperand(0).getOperand(0);
2600 Opc =
N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2607 unsigned &Immr,
unsigned &Imms,
2608 bool BiggerPattern) {
2610 "N must be a SHR/SRA operation to call this function");
2612 EVT VT =
N->getValueType(0);
2617 assert((VT == MVT::i32 || VT == MVT::i64) &&
2618 "Type checking must have been done before calling this function");
2628 Opd0 =
N->getOperand(0).getOperand(0);
2629 }
else if (VT == MVT::i32 &&
N->getOpcode() ==
ISD::SRL &&
2635 Opd0 =
N->getOperand(0).getOperand(0);
2638 assert(VT == MVT::i64 &&
"the promoted type should be i64");
2639 }
else if (BiggerPattern) {
2643 Opd0 =
N->getOperand(0);
2652 <<
": Found large shift immediate, this should not happen\n"));
2661 "bad amount in shift node!");
2662 int immr = SrlImm - ShlImm;
2667 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
2669 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
2673bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(
SDNode *
N) {
2676 EVT VT =
N->getValueType(0);
2677 EVT NarrowVT =
N->getOperand(0)->getValueType(0);
2678 if (VT != MVT::i64 || NarrowVT != MVT::i32)
2689 unsigned Immr = ShiftImm;
2691 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2692 CurDAG->getTargetConstant(Imms, dl, VT)};
2693 CurDAG->SelectNodeTo(
N, AArch64::SBFMXri, VT, Ops);
2698 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
2699 unsigned NumberOfIgnoredLowBits = 0,
2700 bool BiggerPattern =
false) {
2701 if (
N->getValueType(0) != MVT::i32 &&
N->getValueType(0) != MVT::i64)
2704 switch (
N->getOpcode()) {
2706 if (!
N->isMachineOpcode())
2711 NumberOfIgnoredLowBits, BiggerPattern);
2720 unsigned NOpc =
N->getMachineOpcode();
2724 case AArch64::SBFMWri:
2725 case AArch64::UBFMWri:
2726 case AArch64::SBFMXri:
2727 case AArch64::UBFMXri:
2729 Opd0 =
N->getOperand(0);
2730 Immr =
N->getConstantOperandVal(1);
2731 Imms =
N->getConstantOperandVal(2);
2738bool AArch64DAGToDAGISel::tryBitfieldExtractOp(
SDNode *
N) {
2739 unsigned Opc, Immr, Imms;
2744 EVT VT =
N->getValueType(0);
2749 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
2750 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
2751 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
2753 SDNode *
BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
2754 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
2760 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2761 CurDAG->getTargetConstant(Imms, dl, VT)};
2762 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
2771 unsigned NumberOfIgnoredHighBits,
EVT VT) {
2772 assert((VT == MVT::i32 || VT == MVT::i64) &&
2773 "i32 or i64 mask type expected!");
2779 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
2780 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
2803 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2813 APInt OpUsefulBits(UsefulBits);
2817 OpUsefulBits <<= MSB - Imm + 1;
2822 OpUsefulBits <<= Imm;
2824 OpUsefulBits <<= MSB + 1;
2827 OpUsefulBits <<= OpUsefulBits.
getBitWidth() - Imm;
2833 UsefulBits &= OpUsefulBits;
2839 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2841 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2849 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2850 APInt Mask(UsefulBits);
2851 Mask.clearAllBits();
2859 Mask.lshrInPlace(ShiftAmt);
2865 Mask.lshrInPlace(ShiftAmt);
2877 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2879 cast<const ConstantSDNode>(
Op.getOperand(3).getNode())->getZExtValue();
2881 APInt OpUsefulBits(UsefulBits);
2895 OpUsefulBits <<= Width;
2898 if (
Op.getOperand(1) == Orig) {
2900 Mask = ResultUsefulBits & OpUsefulBits;
2904 if (
Op.getOperand(0) == Orig)
2906 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2912 OpUsefulBits <<= Width;
2914 OpUsefulBits <<= LSB;
2916 if (
Op.getOperand(1) == Orig) {
2918 Mask = ResultUsefulBits & OpUsefulBits;
2919 Mask.lshrInPlace(LSB);
2922 if (
Op.getOperand(0) == Orig)
2923 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2940 case AArch64::ANDSWri:
2941 case AArch64::ANDSXri:
2942 case AArch64::ANDWri:
2943 case AArch64::ANDXri:
2947 case AArch64::UBFMWri:
2948 case AArch64::UBFMXri:
2951 case AArch64::ORRWrs:
2952 case AArch64::ORRXrs:
2957 case AArch64::BFMWri:
2958 case AArch64::BFMXri:
2961 case AArch64::STRBBui:
2962 case AArch64::STURBBi:
2968 case AArch64::STRHHui:
2969 case AArch64::STURHHi:
2982 unsigned Bitwidth =
Op.getScalarValueSizeInBits();
2984 UsefulBits =
APInt(Bitwidth, 0);
2993 UsersUsefulBits |= UsefulBitsForUse;
2998 UsefulBits &= UsersUsefulBits;
3008 EVT VT =
Op.getValueType();
3011 unsigned UBFMOpc =
BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3014 if (ShlAmount > 0) {
3017 UBFMOpc, dl, VT,
Op,
3022 assert(ShlAmount < 0 &&
"expected right shift");
3023 int ShrAmount = -ShlAmount;
3049 bool BiggerPattern,
SDValue &Src,
3050 int &DstLSB,
int &Width) {
3051 EVT VT =
Op.getValueType();
3060 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
3064 switch (
Op.getOpcode()) {
3069 NonZeroBits, Src, DstLSB, Width);
3072 NonZeroBits, Src, DstLSB, Width);
3085 EVT VT =
Op.getValueType();
3086 assert((VT == MVT::i32 || VT == MVT::i64) &&
3087 "Caller guarantees VT is one of i32 or i64");
3100 assert((~AndImm & NonZeroBits) == 0 &&
3101 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3130 if (!BiggerPattern && !AndOp0.
hasOneUse())
3149 <<
"Found large Width in bit-field-positioning -- this indicates no "
3150 "proper combining / constant folding was performed\n");
3159 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3174 "Op.getNode() should be a SHL node to call this function");
3176 "Op.getNode() should shift ShlImm to call this function");
3183 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3207 EVT VT =
Op.getValueType();
3208 assert((VT == MVT::i32 || VT == MVT::i64) &&
3209 "Caller guarantees that type is i32 or i64");
3216 if (!BiggerPattern && !
Op.hasOneUse())
3225 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3233 assert(VT == MVT::i32 || VT == MVT::i64);
3244 EVT VT =
N->getValueType(0);
3245 if (VT != MVT::i32 && VT != MVT::i64)
3263 if (!
And.hasOneUse() ||
3273 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3280 if ((OrImm & NotKnownZero) != 0) {
3292 unsigned ImmS = Width - 1;
3298 bool IsBFI = LSB != 0;
3303 unsigned OrChunks = 0, BFIChunks = 0;
3304 for (
unsigned Shift = 0; Shift <
BitWidth; Shift += 16) {
3305 if (((OrImm >> Shift) & 0xFFFF) != 0)
3307 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3310 if (BFIChunks > OrChunks)
3316 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3324 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3333 if (!Dst.hasOneUse())
3336 EVT VT = Dst.getValueType();
3337 assert((VT == MVT::i32 || VT == MVT::i64) &&
3338 "Caller should guarantee that VT is one of i32 or i64");
3346 SDValue DstOp0 = Dst.getOperand(0);
3366 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3367 unsigned MaskWidth =
3370 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3376 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1,
DL, VT));
3377 ShiftedOperand =
SDValue(UBFMNode, 0);
3387 ShiftedOperand = Dst.getOperand(0);
3394 ShiftedOperand = Dst.getOperand(0);
3406 const bool BiggerPattern) {
3407 EVT VT =
N->getValueType(0);
3408 assert(
N->getOpcode() ==
ISD::OR &&
"Expect N to be an OR node");
3409 assert(((
N->getOperand(0) == OrOpd0 &&
N->getOperand(1) == OrOpd1) ||
3410 (
N->getOperand(1) == OrOpd0 &&
N->getOperand(0) == OrOpd1)) &&
3411 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3412 assert((VT == MVT::i32 || VT == MVT::i64) &&
3413 "Expect result type to be i32 or i64 since N is combinable to BFM");
3420 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3423 if (BiggerPattern) {
3437 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3446 assert((!BiggerPattern) &&
"BiggerPattern should be handled above");
3508 EVT VT =
N->getValueType(0);
3509 if (VT != MVT::i32 && VT != MVT::i64)
3517 unsigned NumberOfIgnoredLowBits = UsefulBits.
countr_zero();
3518 unsigned NumberOfIgnoredHighBits = UsefulBits.
countl_zero();
3538 for (
int I = 0;
I < 4; ++
I) {
3541 unsigned ImmR, ImmS;
3542 bool BiggerPattern =
I / 2;
3543 SDValue OrOpd0Val =
N->getOperand(
I % 2);
3545 SDValue OrOpd1Val =
N->getOperand((
I + 1) % 2);
3551 NumberOfIgnoredLowBits, BiggerPattern)) {
3554 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3555 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3560 Width = ImmS - ImmR + 1;
3571 Src, DstLSB, Width)) {
3579 assert((VT == MVT::i32 || VT == MVT::i64) &&
"unexpected OR operand");
3589 APInt BitsToBeInserted =
3592 if ((BitsToBeInserted & ~Known.
Zero) != 0)
3616 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3649 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3651 if (Src->hasOneUse() &&
3654 Src = Src->getOperand(0);
3664 unsigned ImmS = Width - 1;
3670 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3678bool AArch64DAGToDAGISel::tryBitfieldInsertOp(
SDNode *
N) {
3687 CurDAG->SelectNodeTo(
N, TargetOpcode::IMPLICIT_DEF,
N->getValueType(0));
3700bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(
SDNode *
N) {
3704 EVT VT =
N->getValueType(0);
3705 if (VT != MVT::i32 && VT != MVT::i64)
3711 Op0, DstLSB, Width))
3717 unsigned ImmS = Width - 1;
3720 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR,
DL, VT),
3721 CurDAG->getTargetConstant(ImmS,
DL, VT)};
3722 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3723 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3729bool AArch64DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
3730 EVT VT =
N->getValueType(0);
3733 switch (
N->getOpcode()) {
3735 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
3738 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
3741 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
3744 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
3752 if (VT == MVT::i32) {
3755 }
else if (VT == MVT::i64) {
3761 SDValue ShiftAmt =
N->getOperand(1);
3781 (Add0Imm %
Size == 0)) {
3787 if (SubVT == MVT::i32) {
3788 NegOpc = AArch64::SUBWrr;
3789 ZeroReg = AArch64::WZR;
3791 assert(SubVT == MVT::i64);
3792 NegOpc = AArch64::SUBXrr;
3793 ZeroReg = AArch64::XZR;
3796 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3798 CurDAG->getMachineNode(NegOpc,
DL, SubVT, Zero, Add1);
3799 NewShiftAmt =
SDValue(Neg, 0);
3807 if (SubVT == MVT::i32) {
3808 NotOpc = AArch64::ORNWrr;
3809 ZeroReg = AArch64::WZR;
3811 assert(SubVT == MVT::i64);
3812 NotOpc = AArch64::ORNXrr;
3813 ZeroReg = AArch64::XZR;
3816 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3818 CurDAG->getMachineNode(NotOpc,
DL, SubVT, Zero, Add1);
3819 NewShiftAmt =
SDValue(Not, 0);
3840 else if (VT == MVT::i64 && NewShiftAmt->
getValueType(0) == MVT::i32) {
3841 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32,
DL, MVT::i32);
3843 AArch64::SUBREG_TO_REG,
DL, VT,
3844 CurDAG->getTargetConstant(0,
DL, MVT::i64), NewShiftAmt,
SubReg);
3845 NewShiftAmt =
SDValue(Ext, 0);
3848 SDValue Ops[] = {
N->getOperand(0), NewShiftAmt};
3849 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3856 bool isReciprocal) {
3859 FVal = CN->getValueAPF();
3860 else if (
LoadSDNode *LN = dyn_cast<LoadSDNode>(
N)) {
3863 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
3867 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
3868 FVal = cast<ConstantFP>(CN->
getConstVal())->getValueAPF();
3891 if (!IsExact || !IntVal.isPowerOf2())
3893 unsigned FBits = IntVal.logBase2();
3897 if (FBits == 0 || FBits > RegWidth)
return false;
3903bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
3904 unsigned RegWidth) {
3909bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(
SDValue N,
3911 unsigned RegWidth) {
3921 RegString.
split(Fields,
':');
3923 if (Fields.
size() == 1)
3927 &&
"Invalid number of fields in read register string");
3930 bool AllIntFields =
true;
3934 AllIntFields &= !
Field.getAsInteger(10, IntField);
3939 "Unexpected non-integer value in special register string.");
3944 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
3945 (Ops[3] << 3) | (Ops[4]);
3952bool AArch64DAGToDAGISel::tryReadRegister(
SDNode *
N) {
3953 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3954 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3959 unsigned Opcode64Bit = AArch64::MRS;
3964 const auto *TheReg =
3966 if (TheReg && TheReg->Readable &&
3967 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3968 Imm = TheReg->Encoding;
3974 if (!ReadIs128Bit && RegString->getString() ==
"pc") {
3975 Opcode64Bit = AArch64::ADR;
3983 SDValue InChain =
N->getOperand(0);
3984 SDValue SysRegImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
3985 if (!ReadIs128Bit) {
3986 CurDAG->SelectNodeTo(
N, Opcode64Bit, MVT::i64, MVT::Other ,
3987 {SysRegImm, InChain});
3991 {MVT::Untyped , MVT::Other },
3992 {SysRegImm, InChain});
3996 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64,
DL, MVT::i64,
3998 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64,
DL, MVT::i64,
4004 ReplaceUses(
SDValue(
N, 2), OutChain);
4013bool AArch64DAGToDAGISel::tryWriteRegister(
SDNode *
N) {
4014 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
4015 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
4020 if (!WriteIs128Bit) {
4026 auto trySelectPState = [&](
auto PMapper,
unsigned State) {
4028 assert(isa<ConstantSDNode>(
N->getOperand(2)) &&
4029 "Expected a constant integer expression.");
4030 unsigned Reg = PMapper->Encoding;
4031 uint64_t Immed =
N->getConstantOperandVal(2);
4032 CurDAG->SelectNodeTo(
4033 N, State, MVT::Other, CurDAG->getTargetConstant(Reg,
DL, MVT::i32),
4034 CurDAG->getTargetConstant(Immed,
DL, MVT::i16),
N->getOperand(0));
4040 if (trySelectPState(
4041 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
4042 AArch64::MSRpstateImm4))
4044 if (trySelectPState(
4045 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
4046 AArch64::MSRpstateImm1))
4056 if (TheReg && TheReg->Writeable &&
4057 TheReg->haveFeatures(Subtarget->getFeatureBits()))
4058 Imm = TheReg->Encoding;
4066 SDValue InChain =
N->getOperand(0);
4067 if (!WriteIs128Bit) {
4068 CurDAG->SelectNodeTo(
N, AArch64::MSR, MVT::Other,
4069 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
4070 N->getOperand(2), InChain);
4074 SDNode *Pair = CurDAG->getMachineNode(
4075 TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped ,
4076 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(),
DL,
4079 CurDAG->getTargetConstant(AArch64::sube64,
DL, MVT::i32),
4081 CurDAG->getTargetConstant(AArch64::subo64,
DL, MVT::i32)});
4083 CurDAG->SelectNodeTo(
N, AArch64::MSRR, MVT::Other,
4084 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
4092bool AArch64DAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
4094 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
4097 if (Subtarget->hasLSE())
return false;
4099 if (MemTy == MVT::i8)
4100 Opcode = AArch64::CMP_SWAP_8;
4101 else if (MemTy == MVT::i16)
4102 Opcode = AArch64::CMP_SWAP_16;
4103 else if (MemTy == MVT::i32)
4104 Opcode = AArch64::CMP_SWAP_32;
4105 else if (MemTy == MVT::i64)
4106 Opcode = AArch64::CMP_SWAP_64;
4110 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4111 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
4113 SDNode *CmpSwap = CurDAG->getMachineNode(
4115 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
4118 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
4122 CurDAG->RemoveDeadNode(
N);
4129 if (!isa<ConstantSDNode>(
N))
4141 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4142 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4149 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4150 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4154 if (Val <= 65280 && Val % 256 == 0) {
4155 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4156 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4167bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(
SDValue N,
MVT VT,
4170 if (!isa<ConstantSDNode>(
N))
4174 int64_t Val = cast<ConstantSDNode>(
N)
4191 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4192 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4199 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4200 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4204 if (Val <= 65280 && Val % 256 == 0) {
4205 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4206 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4219 if (!isa<ConstantSDNode>(
N))
4223 int64_t Val = cast<ConstantSDNode>(
N)
4231 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4232 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4238 if (Val >= -128 && Val <= 127) {
4239 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4240 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4244 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) {
4245 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4246 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF,
DL, MVT::i32);
4257bool AArch64DAGToDAGISel::SelectSVESignedArithImm(
SDValue N,
SDValue &Imm) {
4258 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4259 int64_t ImmVal = CNode->getSExtValue();
4261 if (ImmVal >= -128 && ImmVal < 128) {
4262 Imm = CurDAG->getTargetConstant(ImmVal,
DL, MVT::i32);
4270 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4271 uint64_t ImmVal = CNode->getZExtValue();
4281 ImmVal &= 0xFFFFFFFF;
4290 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4299 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4300 uint64_t ImmVal = CNode->getZExtValue();
4310 ImmVal |= ImmVal << 8;
4311 ImmVal |= ImmVal << 16;
4312 ImmVal |= ImmVal << 32;
4316 ImmVal |= ImmVal << 16;
4317 ImmVal |= ImmVal << 32;
4320 ImmVal &= 0xFFFFFFFF;
4321 ImmVal |= ImmVal << 32;
4331 Imm = CurDAG->getTargetConstant(encoding,
DL, MVT::i64);
4346 if (
auto *CN = dyn_cast<ConstantSDNode>(
N)) {
4347 uint64_t ImmVal = CN->getZExtValue();
4354 if (ImmVal >
High) {
4355 if (!AllowSaturation)
4360 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4367bool AArch64DAGToDAGISel::trySelectStackSlotTagP(
SDNode *
N) {
4371 if (!(isa<FrameIndexSDNode>(
N->getOperand(1)))) {
4383 int FI = cast<FrameIndexSDNode>(
N->getOperand(1))->getIndex();
4384 SDValue FiOp = CurDAG->getTargetFrameIndex(
4386 int TagOffset =
N->getConstantOperandVal(3);
4388 SDNode *Out = CurDAG->getMachineNode(
4389 AArch64::TAGPstack,
DL, MVT::i64,
4390 {FiOp, CurDAG->getTargetConstant(0,
DL, MVT::i64),
N->getOperand(2),
4391 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4392 ReplaceNode(
N, Out);
4396void AArch64DAGToDAGISel::SelectTagP(
SDNode *
N) {
4397 assert(isa<ConstantSDNode>(
N->getOperand(3)) &&
4398 "llvm.aarch64.tagp third argument must be an immediate");
4399 if (trySelectStackSlotTagP(
N))
4406 int TagOffset =
N->getConstantOperandVal(3);
4407 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP,
DL, MVT::i64,
4408 {
N->getOperand(1),
N->getOperand(2)});
4409 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr,
DL, MVT::i64,
4410 {
SDValue(N1, 0),
N->getOperand(2)});
4411 SDNode *N3 = CurDAG->getMachineNode(
4412 AArch64::ADDG,
DL, MVT::i64,
4413 {
SDValue(N2, 0), CurDAG->getTargetConstant(0,
DL, MVT::i64),
4414 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4418bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(
SDNode *
N) {
4422 if (
N->getConstantOperandVal(2) != 0)
4424 if (!
N->getOperand(0).isUndef())
4428 EVT VT =
N->getValueType(0);
4429 EVT InVT =
N->getOperand(1).getValueType();
4440 "Expected to insert into a packed scalable vector!");
4443 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4444 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4445 N->getOperand(1), RC));
4449bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(
SDNode *
N) {
4453 if (
N->getConstantOperandVal(1) != 0)
4457 EVT VT =
N->getValueType(0);
4458 EVT InVT =
N->getOperand(0).getValueType();
4469 "Expected to extract from a packed scalable vector!");
4472 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4473 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4474 N->getOperand(0), RC));
4478bool AArch64DAGToDAGISel::trySelectXAR(
SDNode *
N) {
4483 EVT VT =
N->getValueType(0);
4495 (Subtarget->hasSVE2() ||
4496 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4505 if (!TLI->isAllActivePredicate(*CurDAG, N0.
getOperand(0)) ||
4506 !TLI->isAllActivePredicate(*CurDAG, N1.
getOperand(0)))
4513 APInt ShlAmt, ShrAmt;
4526 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
4527 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
4528 AArch64::XAR_ZZZI_D})) {
4529 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
4535 if (!Subtarget->hasSHA3())
4557 if (ShAmt + HsAmt != 64)
4561 CurDAG->SelectNodeTo(
N, AArch64::XAR, N0.
getValueType(), Ops);
4566void AArch64DAGToDAGISel::Select(
SDNode *
Node) {
4568 if (
Node->isMachineOpcode()) {
4570 Node->setNodeId(-1);
4575 EVT VT =
Node->getValueType(0);
4577 switch (
Node->getOpcode()) {
4582 if (SelectCMP_SWAP(
Node))
4588 if (tryReadRegister(
Node))
4594 if (tryWriteRegister(
Node))
4601 if (tryIndexedLoad(
Node))
4610 if (tryBitfieldExtractOp(
Node))
4612 if (tryBitfieldInsertInZeroOp(
Node))
4617 if (tryShiftAmountMod(
Node))
4622 if (tryBitfieldExtractOpFromSExt(
Node))
4627 if (tryBitfieldInsertOp(
Node))
4629 if (trySelectXAR(
Node))
4634 if (trySelectCastScalableToFixedLengthVector(
Node))
4640 if (trySelectCastFixedLengthToScalableVector(
Node))
4649 if (ConstNode->
isZero()) {
4650 if (VT == MVT::i32) {
4652 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::WZR, MVT::i32);
4653 ReplaceNode(
Node,
New.getNode());
4655 }
else if (VT == MVT::i64) {
4657 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::XZR, MVT::i64);
4658 ReplaceNode(
Node,
New.getNode());
4667 int FI = cast<FrameIndexSDNode>(
Node)->getIndex();
4670 SDValue TFI = CurDAG->getTargetFrameIndex(
4673 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0,
DL, MVT::i32),
4674 CurDAG->getTargetConstant(Shifter,
DL, MVT::i32) };
4675 CurDAG->SelectNodeTo(
Node, AArch64::ADDXri, MVT::i64, Ops);
4679 unsigned IntNo =
Node->getConstantOperandVal(1);
4683 case Intrinsic::aarch64_gcsss: {
4687 SDValue Zero = CurDAG->getCopyFromReg(Chain,
DL, AArch64::XZR, MVT::i64);
4689 CurDAG->getMachineNode(AArch64::GCSSS1,
DL, MVT::Other, Val, Chain);
4690 SDNode *SS2 = CurDAG->getMachineNode(AArch64::GCSSS2,
DL, MVT::i64,
4691 MVT::Other, Zero,
SDValue(SS1, 0));
4692 ReplaceNode(
Node, SS2);
4695 case Intrinsic::aarch64_ldaxp:
4696 case Intrinsic::aarch64_ldxp: {
4698 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
4703 SDNode *Ld = CurDAG->getMachineNode(
Op,
DL, MVT::i64, MVT::i64,
4704 MVT::Other, MemAddr, Chain);
4708 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4709 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4710 ReplaceNode(
Node, Ld);
4713 case Intrinsic::aarch64_stlxp:
4714 case Intrinsic::aarch64_stxp: {
4716 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
4724 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
4726 SDNode *St = CurDAG->getMachineNode(
Op,
DL, MVT::i32, MVT::Other, Ops);
4729 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4730 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4732 ReplaceNode(
Node, St);
4735 case Intrinsic::aarch64_neon_ld1x2:
4736 if (VT == MVT::v8i8) {
4737 SelectLoad(
Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
4739 }
else if (VT == MVT::v16i8) {
4740 SelectLoad(
Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
4742 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4743 SelectLoad(
Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
4745 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4746 SelectLoad(
Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
4748 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4749 SelectLoad(
Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
4751 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4752 SelectLoad(
Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
4754 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4755 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4757 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4758 SelectLoad(
Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
4762 case Intrinsic::aarch64_neon_ld1x3:
4763 if (VT == MVT::v8i8) {
4764 SelectLoad(
Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
4766 }
else if (VT == MVT::v16i8) {
4767 SelectLoad(
Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
4769 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4770 SelectLoad(
Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
4772 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4773 SelectLoad(
Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
4775 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4776 SelectLoad(
Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
4778 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4779 SelectLoad(
Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
4781 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4782 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4784 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4785 SelectLoad(
Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
4789 case Intrinsic::aarch64_neon_ld1x4:
4790 if (VT == MVT::v8i8) {
4791 SelectLoad(
Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
4793 }
else if (VT == MVT::v16i8) {
4794 SelectLoad(
Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
4796 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4797 SelectLoad(
Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
4799 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4800 SelectLoad(
Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
4802 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4803 SelectLoad(
Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
4805 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4806 SelectLoad(
Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
4808 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4809 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4811 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4812 SelectLoad(
Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
4816 case Intrinsic::aarch64_neon_ld2:
4817 if (VT == MVT::v8i8) {
4818 SelectLoad(
Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
4820 }
else if (VT == MVT::v16i8) {
4821 SelectLoad(
Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
4823 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4824 SelectLoad(
Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
4826 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4827 SelectLoad(
Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
4829 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4830 SelectLoad(
Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
4832 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4833 SelectLoad(
Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
4835 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4836 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4838 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4839 SelectLoad(
Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
4843 case Intrinsic::aarch64_neon_ld3:
4844 if (VT == MVT::v8i8) {
4845 SelectLoad(
Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
4847 }
else if (VT == MVT::v16i8) {
4848 SelectLoad(
Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
4850 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4851 SelectLoad(
Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
4853 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4854 SelectLoad(
Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
4856 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4857 SelectLoad(
Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
4859 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4860 SelectLoad(
Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
4862 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4863 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4865 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4866 SelectLoad(
Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
4870 case Intrinsic::aarch64_neon_ld4:
4871 if (VT == MVT::v8i8) {
4872 SelectLoad(
Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
4874 }
else if (VT == MVT::v16i8) {
4875 SelectLoad(
Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
4877 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4878 SelectLoad(
Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
4880 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4881 SelectLoad(
Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
4883 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4884 SelectLoad(
Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
4886 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4887 SelectLoad(
Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
4889 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4890 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4892 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4893 SelectLoad(
Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
4897 case Intrinsic::aarch64_neon_ld2r:
4898 if (VT == MVT::v8i8) {
4899 SelectLoad(
Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
4901 }
else if (VT == MVT::v16i8) {
4902 SelectLoad(
Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
4904 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4905 SelectLoad(
Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
4907 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4908 SelectLoad(
Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
4910 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4911 SelectLoad(
Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
4913 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4914 SelectLoad(
Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
4916 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4917 SelectLoad(
Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
4919 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4920 SelectLoad(
Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
4924 case Intrinsic::aarch64_neon_ld3r:
4925 if (VT == MVT::v8i8) {
4926 SelectLoad(
Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
4928 }
else if (VT == MVT::v16i8) {
4929 SelectLoad(
Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
4931 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4932 SelectLoad(
Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
4934 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4935 SelectLoad(
Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
4937 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4938 SelectLoad(
Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
4940 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4941 SelectLoad(
Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
4943 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4944 SelectLoad(
Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
4946 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4947 SelectLoad(
Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
4951 case Intrinsic::aarch64_neon_ld4r:
4952 if (VT == MVT::v8i8) {
4953 SelectLoad(
Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
4955 }
else if (VT == MVT::v16i8) {
4956 SelectLoad(
Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
4958 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4959 SelectLoad(
Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
4961 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4962 SelectLoad(
Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
4964 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4965 SelectLoad(
Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
4967 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4968 SelectLoad(
Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
4970 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4971 SelectLoad(
Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
4973 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4974 SelectLoad(
Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
4978 case Intrinsic::aarch64_neon_ld2lane:
4979 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4980 SelectLoadLane(
Node, 2, AArch64::LD2i8);
4982 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4983 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4984 SelectLoadLane(
Node, 2, AArch64::LD2i16);
4986 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4988 SelectLoadLane(
Node, 2, AArch64::LD2i32);
4990 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4992 SelectLoadLane(
Node, 2, AArch64::LD2i64);
4996 case Intrinsic::aarch64_neon_ld3lane:
4997 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4998 SelectLoadLane(
Node, 3, AArch64::LD3i8);
5000 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5001 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5002 SelectLoadLane(
Node, 3, AArch64::LD3i16);
5004 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5006 SelectLoadLane(
Node, 3, AArch64::LD3i32);
5008 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5010 SelectLoadLane(
Node, 3, AArch64::LD3i64);
5014 case Intrinsic::aarch64_neon_ld4lane:
5015 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5016 SelectLoadLane(
Node, 4, AArch64::LD4i8);
5018 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5019 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5020 SelectLoadLane(
Node, 4, AArch64::LD4i16);
5022 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5024 SelectLoadLane(
Node, 4, AArch64::LD4i32);
5026 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5028 SelectLoadLane(
Node, 4, AArch64::LD4i64);
5032 case Intrinsic::aarch64_ld64b:
5033 SelectLoad(
Node, 8, AArch64::LD64B, AArch64::x8sub_0);
5035 case Intrinsic::aarch64_sve_ld2q_sret: {
5036 SelectPredicatedLoad(
Node, 2, 4, AArch64::LD2Q_IMM, AArch64::LD2Q,
true);
5039 case Intrinsic::aarch64_sve_ld3q_sret: {
5040 SelectPredicatedLoad(
Node, 3, 4, AArch64::LD3Q_IMM, AArch64::LD3Q,
true);
5043 case Intrinsic::aarch64_sve_ld4q_sret: {
5044 SelectPredicatedLoad(
Node, 4, 4, AArch64::LD4Q_IMM, AArch64::LD4Q,
true);
5047 case Intrinsic::aarch64_sve_ld2_sret: {
5048 if (VT == MVT::nxv16i8) {
5049 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B,
5052 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5053 VT == MVT::nxv8bf16) {
5054 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H,
5057 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5058 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W,
5061 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5062 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D,
5068 case Intrinsic::aarch64_sve_ld1_pn_x2: {
5069 if (VT == MVT::nxv16i8) {
5070 if (Subtarget->hasSME2())
5071 SelectContiguousMultiVectorLoad(
5072 Node, 2, 0, AArch64::LD1B_2Z_IMM_PSEUDO, AArch64::LD1B_2Z_PSEUDO);
5073 else if (Subtarget->hasSVE2p1())
5074 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LD1B_2Z_IMM,
5079 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5080 VT == MVT::nxv8bf16) {
5081 if (Subtarget->hasSME2())
5082 SelectContiguousMultiVectorLoad(
5083 Node, 2, 1, AArch64::LD1H_2Z_IMM_PSEUDO, AArch64::LD1H_2Z_PSEUDO);
5084 else if (Subtarget->hasSVE2p1())
5085 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LD1H_2Z_IMM,
5090 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5091 if (Subtarget->hasSME2())
5092 SelectContiguousMultiVectorLoad(
5093 Node, 2, 2, AArch64::LD1W_2Z_IMM_PSEUDO, AArch64::LD1W_2Z_PSEUDO);
5094 else if (Subtarget->hasSVE2p1())
5095 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LD1W_2Z_IMM,
5100 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5101 if (Subtarget->hasSME2())
5102 SelectContiguousMultiVectorLoad(
5103 Node, 2, 3, AArch64::LD1D_2Z_IMM_PSEUDO, AArch64::LD1D_2Z_PSEUDO);
5104 else if (Subtarget->hasSVE2p1())
5105 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LD1D_2Z_IMM,
5113 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5114 if (VT == MVT::nxv16i8) {
5115 if (Subtarget->hasSME2())
5116 SelectContiguousMultiVectorLoad(
5117 Node, 4, 0, AArch64::LD1B_4Z_IMM_PSEUDO, AArch64::LD1B_4Z_PSEUDO);
5118 else if (Subtarget->hasSVE2p1())
5119 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LD1B_4Z_IMM,
5124 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5125 VT == MVT::nxv8bf16) {
5126 if (Subtarget->hasSME2())
5127 SelectContiguousMultiVectorLoad(
5128 Node, 4, 1, AArch64::LD1H_4Z_IMM_PSEUDO, AArch64::LD1H_4Z_PSEUDO);
5129 else if (Subtarget->hasSVE2p1())
5130 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LD1H_4Z_IMM,
5135 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5136 if (Subtarget->hasSME2())
5137 SelectContiguousMultiVectorLoad(
5138 Node, 4, 2, AArch64::LD1W_4Z_IMM_PSEUDO, AArch64::LD1W_4Z_PSEUDO);
5139 else if (Subtarget->hasSVE2p1())
5140 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LD1W_4Z_IMM,
5145 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5146 if (Subtarget->hasSME2())
5147 SelectContiguousMultiVectorLoad(
5148 Node, 4, 3, AArch64::LD1D_4Z_IMM_PSEUDO, AArch64::LD1D_4Z_PSEUDO);
5149 else if (Subtarget->hasSVE2p1())
5150 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LD1D_4Z_IMM,
5158 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5159 if (VT == MVT::nxv16i8) {
5160 if (Subtarget->hasSME2())
5161 SelectContiguousMultiVectorLoad(
Node, 2, 0,
5162 AArch64::LDNT1B_2Z_IMM_PSEUDO,
5163 AArch64::LDNT1B_2Z_PSEUDO);
5164 else if (Subtarget->hasSVE2p1())
5165 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LDNT1B_2Z_IMM,
5166 AArch64::LDNT1B_2Z);
5170 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5171 VT == MVT::nxv8bf16) {
5172 if (Subtarget->hasSME2())
5173 SelectContiguousMultiVectorLoad(
Node, 2, 1,
5174 AArch64::LDNT1H_2Z_IMM_PSEUDO,
5175 AArch64::LDNT1H_2Z_PSEUDO);
5176 else if (Subtarget->hasSVE2p1())
5177 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LDNT1H_2Z_IMM,
5178 AArch64::LDNT1H_2Z);
5182 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5183 if (Subtarget->hasSME2())
5184 SelectContiguousMultiVectorLoad(
Node, 2, 2,
5185 AArch64::LDNT1W_2Z_IMM_PSEUDO,
5186 AArch64::LDNT1W_2Z_PSEUDO);
5187 else if (Subtarget->hasSVE2p1())
5188 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LDNT1W_2Z_IMM,
5189 AArch64::LDNT1W_2Z);
5193 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5194 if (Subtarget->hasSME2())
5195 SelectContiguousMultiVectorLoad(
Node, 2, 3,
5196 AArch64::LDNT1D_2Z_IMM_PSEUDO,
5197 AArch64::LDNT1D_2Z_PSEUDO);
5198 else if (Subtarget->hasSVE2p1())
5199 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LDNT1D_2Z_IMM,
5200 AArch64::LDNT1D_2Z);
5207 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5208 if (VT == MVT::nxv16i8) {
5209 if (Subtarget->hasSME2())
5210 SelectContiguousMultiVectorLoad(
Node, 4, 0,
5211 AArch64::LDNT1B_4Z_IMM_PSEUDO,
5212 AArch64::LDNT1B_4Z_PSEUDO);
5213 else if (Subtarget->hasSVE2p1())
5214 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LDNT1B_4Z_IMM,
5215 AArch64::LDNT1B_4Z);
5219 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5220 VT == MVT::nxv8bf16) {
5221 if (Subtarget->hasSME2())
5222 SelectContiguousMultiVectorLoad(
Node, 4, 1,
5223 AArch64::LDNT1H_4Z_IMM_PSEUDO,
5224 AArch64::LDNT1H_4Z_PSEUDO);
5225 else if (Subtarget->hasSVE2p1())
5226 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LDNT1H_4Z_IMM,
5227 AArch64::LDNT1H_4Z);
5231 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5232 if (Subtarget->hasSME2())
5233 SelectContiguousMultiVectorLoad(
Node, 4, 2,
5234 AArch64::LDNT1W_4Z_IMM_PSEUDO,
5235 AArch64::LDNT1W_4Z_PSEUDO);
5236 else if (Subtarget->hasSVE2p1())
5237 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LDNT1W_4Z_IMM,
5238 AArch64::LDNT1W_4Z);
5242 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5243 if (Subtarget->hasSME2())
5244 SelectContiguousMultiVectorLoad(
Node, 4, 3,
5245 AArch64::LDNT1D_4Z_IMM_PSEUDO,
5246 AArch64::LDNT1D_4Z_PSEUDO);
5247 else if (Subtarget->hasSVE2p1())
5248 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LDNT1D_4Z_IMM,
5249 AArch64::LDNT1D_4Z);
5256 case Intrinsic::aarch64_sve_ld3_sret: {
5257 if (VT == MVT::nxv16i8) {
5258 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B,
5261 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5262 VT == MVT::nxv8bf16) {
5263 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H,
5266 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5267 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W,
5270 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5271 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D,
5277 case Intrinsic::aarch64_sve_ld4_sret: {
5278 if (VT == MVT::nxv16i8) {
5279 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B,
5282 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5283 VT == MVT::nxv8bf16) {
5284 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H,
5287 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5288 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W,
5291 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5292 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D,
5298 case Intrinsic::aarch64_sme_read_hor_vg2: {
5299 if (VT == MVT::nxv16i8) {
5300 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5301 AArch64::MOVA_2ZMXI_H_B);
5303 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5304 VT == MVT::nxv8bf16) {
5305 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5306 AArch64::MOVA_2ZMXI_H_H);
5308 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5309 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5310 AArch64::MOVA_2ZMXI_H_S);
5312 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5313 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5314 AArch64::MOVA_2ZMXI_H_D);
5319 case Intrinsic::aarch64_sme_read_ver_vg2: {
5320 if (VT == MVT::nxv16i8) {
5321 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5322 AArch64::MOVA_2ZMXI_V_B);
5324 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5325 VT == MVT::nxv8bf16) {
5326 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5327 AArch64::MOVA_2ZMXI_V_H);
5329 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5330 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5331 AArch64::MOVA_2ZMXI_V_S);
5333 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5334 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5335 AArch64::MOVA_2ZMXI_V_D);
5340 case Intrinsic::aarch64_sme_read_hor_vg4: {
5341 if (VT == MVT::nxv16i8) {
5342 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5343 AArch64::MOVA_4ZMXI_H_B);
5345 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5346 VT == MVT::nxv8bf16) {
5347 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5348 AArch64::MOVA_4ZMXI_H_H);
5350 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5351 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAS0,
5352 AArch64::MOVA_4ZMXI_H_S);
5354 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5355 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAD0,
5356 AArch64::MOVA_4ZMXI_H_D);
5361 case Intrinsic::aarch64_sme_read_ver_vg4: {
5362 if (VT == MVT::nxv16i8) {
5363 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5364 AArch64::MOVA_4ZMXI_V_B);
5366 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5367 VT == MVT::nxv8bf16) {
5368 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5369 AArch64::MOVA_4ZMXI_V_H);
5371 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5372 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAS0,
5373 AArch64::MOVA_4ZMXI_V_S);
5375 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5376 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAD0,
5377 AArch64::MOVA_4ZMXI_V_D);
5382 case Intrinsic::aarch64_sme_read_vg1x2: {
5383 SelectMultiVectorMove<7, 1>(
Node, 2, AArch64::ZA,
5384 AArch64::MOVA_VG2_2ZMXI);
5387 case Intrinsic::aarch64_sme_read_vg1x4: {
5388 SelectMultiVectorMove<7, 1>(
Node, 4, AArch64::ZA,
5389 AArch64::MOVA_VG4_4ZMXI);
5392 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5393 if (VT == MVT::nxv16i8) {
5394 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_B_PSEUDO, 14, 2);
5396 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5397 VT == MVT::nxv8bf16) {
5398 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_H_PSEUDO, 6, 2);
5400 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5401 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_S_PSEUDO, 2, 2);
5403 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5404 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_D_PSEUDO, 0, 2);
5409 case Intrinsic::aarch64_sme_readz_vert_x2: {
5410 if (VT == MVT::nxv16i8) {
5411 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_B_PSEUDO, 14, 2);
5413 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5414 VT == MVT::nxv8bf16) {
5415 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_H_PSEUDO, 6, 2);
5417 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5418 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_S_PSEUDO, 2, 2);
5420 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5421 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_D_PSEUDO, 0, 2);
5426 case Intrinsic::aarch64_sme_readz_horiz_x4: {
5427 if (VT == MVT::nxv16i8) {
5428 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_B_PSEUDO, 12, 4);
5430 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5431 VT == MVT::nxv8bf16) {
5432 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_H_PSEUDO, 4, 4);
5434 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5435 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_S_PSEUDO, 0, 4);
5437 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5438 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_D_PSEUDO, 0, 4);
5443 case Intrinsic::aarch64_sme_readz_vert_x4: {
5444 if (VT == MVT::nxv16i8) {
5445 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_B_PSEUDO, 12, 4);
5447 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5448 VT == MVT::nxv8bf16) {
5449 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_H_PSEUDO, 4, 4);
5451 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5452 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_S_PSEUDO, 0, 4);
5454 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5455 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_D_PSEUDO, 0, 4);
5460 case Intrinsic::aarch64_sme_readz_x2: {
5461 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, 7, 1,
5465 case Intrinsic::aarch64_sme_readz_x4: {
5466 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, 7, 1,
5470 case Intrinsic::swift_async_context_addr: {
5473 SDValue CopyFP = CurDAG->getCopyFromReg(Chain,
DL, AArch64::FP, MVT::i64);
5475 CurDAG->getMachineNode(AArch64::SUBXri,
DL, MVT::i64, CopyFP,
5476 CurDAG->getTargetConstant(8,
DL, MVT::i32),
5477 CurDAG->getTargetConstant(0,
DL, MVT::i32)),
5481 CurDAG->RemoveDeadNode(
Node);
5483 auto &MF = CurDAG->getMachineFunction();
5484 MF.getFrameInfo().setFrameAddressIsTaken(
true);
5488 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
5489 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5490 Node->getValueType(0),
5491 {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
5492 AArch64::LUTI2_4ZTZI_S}))
5494 SelectMultiVectorLuti(
Node, 4, Opc, 3);
5497 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
5498 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5499 Node->getValueType(0),
5500 {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
5502 SelectMultiVectorLuti(
Node, 4, Opc, 1);
5505 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
5506 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5507 Node->getValueType(0),
5508 {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
5509 AArch64::LUTI2_2ZTZI_S}))
5511 SelectMultiVectorLuti(
Node, 2, Opc, 7);
5514 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
5515 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5516 Node->getValueType(0),
5517 {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
5518 AArch64::LUTI4_2ZTZI_S}))
5520 SelectMultiVectorLuti(
Node, 2, Opc, 3);
5526 unsigned IntNo =
Node->getConstantOperandVal(0);
5530 case Intrinsic::aarch64_tagp:
5534 case Intrinsic::ptrauth_auth:
5535 SelectPtrauthAuth(
Node);
5538 case Intrinsic::ptrauth_resign:
5539 SelectPtrauthResign(
Node);
5542 case Intrinsic::aarch64_neon_tbl2:
5543 SelectTable(
Node, 2,
5544 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
5547 case Intrinsic::aarch64_neon_tbl3:
5548 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
5549 : AArch64::TBLv16i8Three,
5552 case Intrinsic::aarch64_neon_tbl4:
5553 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
5554 : AArch64::TBLv16i8Four,
5557 case Intrinsic::aarch64_neon_tbx2:
5558 SelectTable(
Node, 2,
5559 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
5562 case Intrinsic::aarch64_neon_tbx3:
5563 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
5564 : AArch64::TBXv16i8Three,
5567 case Intrinsic::aarch64_neon_tbx4:
5568 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
5569 : AArch64::TBXv16i8Four,
5572 case Intrinsic::aarch64_sve_srshl_single_x2:
5573 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5574 Node->getValueType(0),
5575 {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
5576 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
5577 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5579 case Intrinsic::aarch64_sve_srshl_single_x4:
5580 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5581 Node->getValueType(0),
5582 {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
5583 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
5584 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5586 case Intrinsic::aarch64_sve_urshl_single_x2:
5587 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5588 Node->getValueType(0),
5589 {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
5590 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
5591 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5593 case Intrinsic::aarch64_sve_urshl_single_x4:
5594 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5595 Node->getValueType(0),
5596 {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
5597 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
5598 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5600 case Intrinsic::aarch64_sve_srshl_x2:
5601 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5602 Node->getValueType(0),
5603 {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
5604 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
5605 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5607 case Intrinsic::aarch64_sve_srshl_x4:
5608 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5609 Node->getValueType(0),
5610 {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
5611 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
5612 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5614 case Intrinsic::aarch64_sve_urshl_x2:
5615 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5616 Node->getValueType(0),
5617 {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
5618 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
5619 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5621 case Intrinsic::aarch64_sve_urshl_x4:
5622 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5623 Node->getValueType(0),
5624 {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
5625 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
5626 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5628 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
5629 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5630 Node->getValueType(0),
5631 {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
5632 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
5633 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5635 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
5636 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5637 Node->getValueType(0),
5638 {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
5639 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
5640 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5642 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
5643 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5644 Node->getValueType(0),
5645 {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
5646 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
5647 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5649 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
5650 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5651 Node->getValueType(0),
5652 {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
5653 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
5654 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5656 case Intrinsic::aarch64_sve_whilege_x2:
5657 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5658 Node->getValueType(0),
5659 {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
5660 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
5661 SelectWhilePair(
Node,
Op);
5663 case Intrinsic::aarch64_sve_whilegt_x2:
5664 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5665 Node->getValueType(0),
5666 {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
5667 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
5668 SelectWhilePair(
Node,
Op);
5670 case Intrinsic::aarch64_sve_whilehi_x2:
5671 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5672 Node->getValueType(0),
5673 {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
5674 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
5675 SelectWhilePair(
Node,
Op);
5677 case Intrinsic::aarch64_sve_whilehs_x2:
5678 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5679 Node->getValueType(0),
5680 {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
5681 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
5682 SelectWhilePair(
Node,
Op);
5684 case Intrinsic::aarch64_sve_whilele_x2:
5685 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5686 Node->getValueType(0),
5687 {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
5688 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
5689 SelectWhilePair(
Node,
Op);
5691 case Intrinsic::aarch64_sve_whilelo_x2:
5692 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5693 Node->getValueType(0),
5694 {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
5695 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
5696 SelectWhilePair(
Node,
Op);
5698 case Intrinsic::aarch64_sve_whilels_x2:
5699 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5700 Node->getValueType(0),
5701 {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
5702 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
5703 SelectWhilePair(
Node,
Op);
5705 case Intrinsic::aarch64_sve_whilelt_x2:
5706 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5707 Node->getValueType(0),
5708 {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
5709 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
5710 SelectWhilePair(
Node,
Op);
5712 case Intrinsic::aarch64_sve_smax_single_x2:
5713 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5714 Node->getValueType(0),
5715 {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
5716 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
5717 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5719 case Intrinsic::aarch64_sve_umax_single_x2:
5720 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5721 Node->getValueType(0),
5722 {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
5723 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
5724 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5726 case Intrinsic::aarch64_sve_fmax_single_x2:
5727 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5728 Node->getValueType(0),
5729 {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
5730 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
5731 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5733 case Intrinsic::aarch64_sve_smax_single_x4:
5734 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5735 Node->getValueType(0),
5736 {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
5737 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
5738 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5740 case Intrinsic::aarch64_sve_umax_single_x4:
5741 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5742 Node->getValueType(0),
5743 {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
5744 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
5745 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5747 case Intrinsic::aarch64_sve_fmax_single_x4:
5748 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5749 Node->getValueType(0),
5750 {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
5751 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
5752 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5754 case Intrinsic::aarch64_sve_smin_single_x2:
5755 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5756 Node->getValueType(0),
5757 {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
5758 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
5759 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5761 case Intrinsic::aarch64_sve_umin_single_x2:
5762 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5763 Node->getValueType(0),
5764 {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
5765 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
5766 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5768 case Intrinsic::aarch64_sve_fmin_single_x2:
5769 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5770 Node->getValueType(0),
5771 {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
5772 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
5773 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5775 case Intrinsic::aarch64_sve_smin_single_x4:
5776 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5777 Node->getValueType(0),
5778 {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
5779 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
5780 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5782 case Intrinsic::aarch64_sve_umin_single_x4:
5783 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5784 Node->getValueType(0),
5785 {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
5786 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
5787 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5789 case Intrinsic::aarch64_sve_fmin_single_x4:
5790 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5791 Node->getValueType(0),
5792 {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
5793 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
5794 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5796 case Intrinsic::aarch64_sve_smax_x2:
5797 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5798 Node->getValueType(0),
5799 {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
5800 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
5801 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5803 case Intrinsic::aarch64_sve_umax_x2:
5804 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5805 Node->getValueType(0),
5806 {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
5807 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
5808 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5810 case Intrinsic::aarch64_sve_fmax_x2:
5811 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5812 Node->getValueType(0),
5813 {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
5814 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
5815 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5817 case Intrinsic::aarch64_sve_smax_x4:
5818 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5819 Node->getValueType(0),
5820 {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
5821 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
5822 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5824 case Intrinsic::aarch64_sve_umax_x4:
5825 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5826 Node->getValueType(0),
5827 {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
5828 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
5829 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5831 case Intrinsic::aarch64_sve_fmax_x4:
5832 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5833 Node->getValueType(0),
5834 {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
5835 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
5836 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5838 case Intrinsic::aarch64_sve_smin_x2:
5839 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5840 Node->getValueType(0),
5841 {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
5842 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
5843 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5845 case Intrinsic::aarch64_sve_umin_x2:
5846 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5847 Node->getValueType(0),
5848 {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
5849 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
5850 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5852 case Intrinsic::aarch64_sve_fmin_x2:
5853 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5854 Node->getValueType(0),
5855 {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
5856 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
5857 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5859 case Intrinsic::aarch64_sve_smin_x4:
5860 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5861 Node->getValueType(0),
5862 {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
5863 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
5864 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5866 case Intrinsic::aarch64_sve_umin_x4:
5867 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5868 Node->getValueType(0),
5869 {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
5870 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
5871 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5873 case Intrinsic::aarch64_sve_fmin_x4:
5874 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5875 Node->getValueType(0),
5876 {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
5877 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
5878 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5880 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
5881 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5882 Node->getValueType(0),
5883 {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
5884 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
5885 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5887 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
5888 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5889 Node->getValueType(0),
5890 {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
5891 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
5892 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5894 case Intrinsic::aarch64_sve_fminnm_single_x2:
5895 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5896 Node->getValueType(0),
5897 {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
5898 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
5899 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5901 case Intrinsic::aarch64_sve_fminnm_single_x4:
5902 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5903 Node->getValueType(0),
5904 {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
5905 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
5906 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5908 case Intrinsic::aarch64_sve_fmaxnm_x2:
5909 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5910 Node->getValueType(0),
5911 {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
5912 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
5913 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5915 case Intrinsic::aarch64_sve_fmaxnm_x4:
5916 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5917 Node->getValueType(0),
5918 {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
5919 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
5920 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5922 case Intrinsic::aarch64_sve_fminnm_x2:
5923 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5924 Node->getValueType(0),
5925 {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
5926 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
5927 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5929 case Intrinsic::aarch64_sve_fminnm_x4:
5930 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5931 Node->getValueType(0),
5932 {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
5933 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
5934 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5936 case Intrinsic::aarch64_sve_fcvtzs_x2:
5937 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZS_2Z2Z_StoS);
5939 case Intrinsic::aarch64_sve_scvtf_x2:
5940 SelectCVTIntrinsic(
Node, 2, AArch64::SCVTF_2Z2Z_StoS);
5942 case Intrinsic::aarch64_sve_fcvtzu_x2:
5943 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZU_2Z2Z_StoS);
5945 case Intrinsic::aarch64_sve_ucvtf_x2:
5946 SelectCVTIntrinsic(
Node, 2, AArch64::UCVTF_2Z2Z_StoS);
5948 case Intrinsic::aarch64_sve_fcvtzs_x4:
5949 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZS_4Z4Z_StoS);
5951 case Intrinsic::aarch64_sve_scvtf_x4:
5952 SelectCVTIntrinsic(
Node, 4, AArch64::SCVTF_4Z4Z_StoS);
5954 case Intrinsic::aarch64_sve_fcvtzu_x4:
5955 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZU_4Z4Z_StoS);
5957 case Intrinsic::aarch64_sve_ucvtf_x4:
5958 SelectCVTIntrinsic(
Node, 4, AArch64::UCVTF_4Z4Z_StoS);
5960 case Intrinsic::aarch64_sve_fcvt_widen_x2:
5961 SelectUnaryMultiIntrinsic(
Node, 2,
false, AArch64::FCVT_2ZZ_H_S);
5963 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
5964 SelectUnaryMultiIntrinsic(
Node, 2,
false, AArch64::FCVTL_2ZZ_H_S);
5966 case Intrinsic::aarch64_sve_sclamp_single_x2:
5967 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5968 Node->getValueType(0),
5969 {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
5970 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
5971 SelectClamp(
Node, 2,
Op);
5973 case Intrinsic::aarch64_sve_uclamp_single_x2:
5974 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5975 Node->getValueType(0),
5976 {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
5977 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
5978 SelectClamp(
Node, 2,
Op);
5980 case Intrinsic::aarch64_sve_fclamp_single_x2:
5981 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5982 Node->getValueType(0),
5983 {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
5984 AArch64::FCLAMP_VG2_2Z2Z_D}))
5985 SelectClamp(
Node, 2,
Op);
5987 case Intrinsic::aarch64_sve_bfclamp_single_x2:
5988 SelectClamp(
Node, 2, AArch64::BFCLAMP_VG2_2ZZZ_H);
5990 case Intrinsic::aarch64_sve_sclamp_single_x4:
5991 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5992 Node->getValueType(0),
5993 {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
5994 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
5995 SelectClamp(
Node, 4,
Op);
5997 case Intrinsic::aarch64_sve_uclamp_single_x4:
5998 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5999 Node->getValueType(0),
6000 {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
6001 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
6002 SelectClamp(
Node, 4,
Op);
6004 case Intrinsic::aarch64_sve_fclamp_single_x4:
6005 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6006 Node->getValueType(0),
6007 {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
6008 AArch64::FCLAMP_VG4_4Z4Z_D}))
6009 SelectClamp(
Node, 4,
Op);
6011 case Intrinsic::aarch64_sve_bfclamp_single_x4:
6012 SelectClamp(
Node, 4, AArch64::BFCLAMP_VG4_4ZZZ_H);
6014 case Intrinsic::aarch64_sve_add_single_x2:
6015 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6016 Node->getValueType(0),
6017 {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
6018 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
6019 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
6021 case Intrinsic::aarch64_sve_add_single_x4:
6022 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6023 Node->getValueType(0),
6024 {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
6025 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
6026 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
6028 case Intrinsic::aarch64_sve_zip_x2:
6029 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6030 Node->getValueType(0),
6031 {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
6032 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
6033 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6035 case Intrinsic::aarch64_sve_zipq_x2:
6036 SelectUnaryMultiIntrinsic(
Node, 2,
false,
6037 AArch64::ZIP_VG2_2ZZZ_Q);
6039 case Intrinsic::aarch64_sve_zip_x4:
6040 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6041 Node->getValueType(0),
6042 {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
6043 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
6044 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6046 case Intrinsic::aarch64_sve_zipq_x4:
6047 SelectUnaryMultiIntrinsic(
Node, 4,
true,
6048 AArch64::ZIP_VG4_4Z4Z_Q);
6050 case Intrinsic::aarch64_sve_uzp_x2:
6051 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6052 Node->getValueType(0),
6053 {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
6054 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
6055 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6057 case Intrinsic::aarch64_sve_uzpq_x2:
6058 SelectUnaryMultiIntrinsic(
Node, 2,
false,
6059 AArch64::UZP_VG2_2ZZZ_Q);
6061 case Intrinsic::aarch64_sve_uzp_x4:
6062 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6063 Node->getValueType(0),
6064 {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
6065 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
6066 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6068 case Intrinsic::aarch64_sve_uzpq_x4:
6069 SelectUnaryMultiIntrinsic(
Node, 4,
true,
6070 AArch64::UZP_VG4_4Z4Z_Q);
6072 case Intrinsic::aarch64_sve_sel_x2:
6073 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6074 Node->getValueType(0),
6075 {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
6076 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
6077 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op,
true);
6079 case Intrinsic::aarch64_sve_sel_x4:
6080 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6081 Node->getValueType(0),
6082 {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
6083 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
6084 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op,
true);
6086 case Intrinsic::aarch64_sve_frinta_x2:
6087 SelectFrintFromVT(
Node, 2, AArch64::FRINTA_2Z2Z_S);
6089 case Intrinsic::aarch64_sve_frinta_x4:
6090 SelectFrintFromVT(
Node, 4, AArch64::FRINTA_4Z4Z_S);
6092 case Intrinsic::aarch64_sve_frintm_x2:
6093 SelectFrintFromVT(
Node, 2, AArch64::FRINTM_2Z2Z_S);
6095 case Intrinsic::aarch64_sve_frintm_x4:
6096 SelectFrintFromVT(
Node, 4, AArch64::FRINTM_4Z4Z_S);
6098 case Intrinsic::aarch64_sve_frintn_x2:
6099 SelectFrintFromVT(
Node, 2, AArch64::FRINTN_2Z2Z_S);
6101 case Intrinsic::aarch64_sve_frintn_x4:
6102 SelectFrintFromVT(
Node, 4, AArch64::FRINTN_4Z4Z_S);
6104 case Intrinsic::aarch64_sve_frintp_x2:
6105 SelectFrintFromVT(
Node, 2, AArch64::FRINTP_2Z2Z_S);
6107 case Intrinsic::aarch64_sve_frintp_x4:
6108 SelectFrintFromVT(
Node, 4, AArch64::FRINTP_4Z4Z_S);
6110 case Intrinsic::aarch64_sve_sunpk_x2:
6111 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6112 Node->getValueType(0),
6113 {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6114 AArch64::SUNPK_VG2_2ZZ_D}))
6115 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6117 case Intrinsic::aarch64_sve_uunpk_x2:
6118 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6119 Node->getValueType(0),
6120 {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6121 AArch64::UUNPK_VG2_2ZZ_D}))
6122 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6124 case Intrinsic::aarch64_sve_sunpk_x4:
6125 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6126 Node->getValueType(0),
6127 {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6128 AArch64::SUNPK_VG4_4Z2Z_D}))
6129 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6131 case Intrinsic::aarch64_sve_uunpk_x4:
6132 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6133 Node->getValueType(0),
6134 {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6135 AArch64::UUNPK_VG4_4Z2Z_D}))
6136 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6138 case Intrinsic::aarch64_sve_pext_x2: {
6139 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6140 Node->getValueType(0),
6141 {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6142 AArch64::PEXT_2PCI_D}))
6143 SelectPExtPair(
Node,
Op);
6150 unsigned IntNo =
Node->getConstantOperandVal(1);
6151 if (
Node->getNumOperands() >= 3)
6152 VT =
Node->getOperand(2)->getValueType(0);
6156 case Intrinsic::aarch64_neon_st1x2: {
6157 if (VT == MVT::v8i8) {
6158 SelectStore(
Node, 2, AArch64::ST1Twov8b);
6160 }
else if (VT == MVT::v16i8) {
6161 SelectStore(
Node, 2, AArch64::ST1Twov16b);
6163 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6164 VT == MVT::v4bf16) {
6165 SelectStore(
Node, 2, AArch64::ST1Twov4h);
6167 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6168 VT == MVT::v8bf16) {
6169 SelectStore(
Node, 2, AArch64::ST1Twov8h);
6171 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6172 SelectStore(
Node, 2, AArch64::ST1Twov2s);
6174 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6175 SelectStore(
Node, 2, AArch64::ST1Twov4s);
6177 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6178 SelectStore(
Node, 2, AArch64::ST1Twov2d);
6180 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6181 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6186 case Intrinsic::aarch64_neon_st1x3: {
6187 if (VT == MVT::v8i8) {
6188 SelectStore(
Node, 3, AArch64::ST1Threev8b);
6190 }
else if (VT == MVT::v16i8) {
6191 SelectStore(
Node, 3, AArch64::ST1Threev16b);
6193 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6194 VT == MVT::v4bf16) {
6195 SelectStore(
Node, 3, AArch64::ST1Threev4h);
6197 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6198 VT == MVT::v8bf16) {
6199 SelectStore(
Node, 3, AArch64::ST1Threev8h);
6201 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6202 SelectStore(
Node, 3, AArch64::ST1Threev2s);
6204 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6205 SelectStore(
Node, 3, AArch64::ST1Threev4s);
6207 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6208 SelectStore(
Node, 3, AArch64::ST1Threev2d);
6210 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6211 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6216 case Intrinsic::aarch64_neon_st1x4: {
6217 if (VT == MVT::v8i8) {
6218 SelectStore(
Node, 4, AArch64::ST1Fourv8b);
6220 }
else if (VT == MVT::v16i8) {
6221 SelectStore(
Node, 4, AArch64::ST1Fourv16b);
6223 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6224 VT == MVT::v4bf16) {
6225 SelectStore(
Node, 4, AArch64::ST1Fourv4h);
6227 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6228 VT == MVT::v8bf16) {
6229 SelectStore(
Node, 4, AArch64::ST1Fourv8h);
6231 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6232 SelectStore(
Node, 4, AArch64::ST1Fourv2s);
6234 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6235 SelectStore(
Node, 4, AArch64::ST1Fourv4s);
6237 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6238 SelectStore(
Node, 4, AArch64::ST1Fourv2d);
6240 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6241 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6246 case Intrinsic::aarch64_neon_st2: {
6247 if (VT == MVT::v8i8) {
6248 SelectStore(
Node, 2, AArch64::ST2Twov8b);
6250 }
else if (VT == MVT::v16i8) {
6251 SelectStore(
Node, 2, AArch64::ST2Twov16b);
6253 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6254 VT == MVT::v4bf16) {
6255 SelectStore(
Node, 2, AArch64::ST2Twov4h);
6257 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6258 VT == MVT::v8bf16) {
6259 SelectStore(
Node, 2, AArch64::ST2Twov8h);
6261 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6262 SelectStore(
Node, 2, AArch64::ST2Twov2s);
6264 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6265 SelectStore(
Node, 2, AArch64::ST2Twov4s);
6267 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6268 SelectStore(
Node, 2, AArch64::ST2Twov2d);
6270 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6271 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6276 case Intrinsic::aarch64_neon_st3: {
6277 if (VT == MVT::v8i8) {
6278 SelectStore(
Node, 3, AArch64::ST3Threev8b);
6280 }
else if (VT == MVT::v16i8) {
6281 SelectStore(
Node, 3, AArch64::ST3Threev16b);
6283 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6284 VT == MVT::v4bf16) {
6285 SelectStore(
Node, 3, AArch64::ST3Threev4h);
6287 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6288 VT == MVT::v8bf16) {
6289 SelectStore(
Node, 3, AArch64::ST3Threev8h);
6291 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6292 SelectStore(
Node, 3, AArch64::ST3Threev2s);
6294 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6295 SelectStore(
Node, 3, AArch64::ST3Threev4s);
6297 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6298 SelectStore(
Node, 3, AArch64::ST3Threev2d);
6300 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6301 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6306 case Intrinsic::aarch64_neon_st4: {
6307 if (VT == MVT::v8i8) {
6308 SelectStore(
Node, 4, AArch64::ST4Fourv8b);
6310 }
else if (VT == MVT::v16i8) {
6311 SelectStore(
Node, 4, AArch64::ST4Fourv16b);
6313 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6314 VT == MVT::v4bf16) {
6315 SelectStore(
Node, 4, AArch64::ST4Fourv4h);
6317 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6318 VT == MVT::v8bf16) {
6319 SelectStore(
Node, 4, AArch64::ST4Fourv8h);
6321 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6322 SelectStore(
Node, 4, AArch64::ST4Fourv2s);
6324 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6325 SelectStore(
Node, 4, AArch64::ST4Fourv4s);
6327 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6328 SelectStore(
Node, 4, AArch64::ST4Fourv2d);
6330 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6331 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6336 case Intrinsic::aarch64_neon_st2lane: {
6337 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6338 SelectStoreLane(
Node, 2, AArch64::ST2i8);
6340 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6341 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6342 SelectStoreLane(
Node, 2, AArch64::ST2i16);
6344 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6346 SelectStoreLane(
Node, 2, AArch64::ST2i32);
6348 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6350 SelectStoreLane(
Node, 2, AArch64::ST2i64);
6355 case Intrinsic::aarch64_neon_st3lane: {
6356 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6357 SelectStoreLane(
Node, 3, AArch64::ST3i8);
6359 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6360 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6361 SelectStoreLane(
Node, 3, AArch64::ST3i16);
6363 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6365 SelectStoreLane(
Node, 3, AArch64::ST3i32);
6367 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6369 SelectStoreLane(
Node, 3, AArch64::ST3i64);
6374 case Intrinsic::aarch64_neon_st4lane: {
6375 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6376 SelectStoreLane(
Node, 4, AArch64::ST4i8);
6378 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6379 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6380 SelectStoreLane(
Node, 4, AArch64::ST4i16);
6382 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6384 SelectStoreLane(
Node, 4, AArch64::ST4i32);
6386 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6388 SelectStoreLane(
Node, 4, AArch64::ST4i64);
6393 case Intrinsic::aarch64_sve_st2q: {
6394 SelectPredicatedStore(
Node, 2, 4, AArch64::ST2Q, AArch64::ST2Q_IMM);
6397 case Intrinsic::aarch64_sve_st3q: {
6398 SelectPredicatedStore(
Node, 3, 4, AArch64::ST3Q, AArch64::ST3Q_IMM);
6401 case Intrinsic::aarch64_sve_st4q: {
6402 SelectPredicatedStore(
Node, 4, 4, AArch64::ST4Q, AArch64::ST4Q_IMM);
6405 case Intrinsic::aarch64_sve_st2: {
6406 if (VT == MVT::nxv16i8) {
6407 SelectPredicatedStore(
Node, 2, 0, AArch64::ST2B, AArch64::ST2B_IMM);
6409 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6410 VT == MVT::nxv8bf16) {
6411 SelectPredicatedStore(
Node, 2, 1, AArch64::ST2H, AArch64::ST2H_IMM);
6413 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6414 SelectPredicatedStore(
Node, 2, 2, AArch64::ST2W, AArch64::ST2W_IMM);
6416 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6417 SelectPredicatedStore(
Node, 2, 3, AArch64::ST2D, AArch64::ST2D_IMM);
6422 case Intrinsic::aarch64_sve_st3: {
6423 if (VT == MVT::nxv16i8) {
6424 SelectPredicatedStore(
Node, 3, 0, AArch64::ST3B, AArch64::ST3B_IMM);
6426 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6427 VT == MVT::nxv8bf16) {
6428 SelectPredicatedStore(
Node, 3, 1, AArch64::ST3H, AArch64::ST3H_IMM);
6430 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6431 SelectPredicatedStore(
Node, 3, 2, AArch64::ST3W, AArch64::ST3W_IMM);
6433 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6434 SelectPredicatedStore(
Node, 3, 3, AArch64::ST3D, AArch64::ST3D_IMM);
6439 case Intrinsic::aarch64_sve_st4: {
6440 if (VT == MVT::nxv16i8) {
6441 SelectPredicatedStore(
Node, 4, 0, AArch64::ST4B, AArch64::ST4B_IMM);
6443 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6444 VT == MVT::nxv8bf16) {
6445 SelectPredicatedStore(
Node, 4, 1, AArch64::ST4H, AArch64::ST4H_IMM);
6447 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6448 SelectPredicatedStore(
Node, 4, 2, AArch64::ST4W, AArch64::ST4W_IMM);
6450 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6451 SelectPredicatedStore(
Node, 4, 3, AArch64::ST4D, AArch64::ST4D_IMM);
6460 if (VT == MVT::v8i8) {
6461 SelectPostLoad(
Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
6463 }
else if (VT == MVT::v16i8) {
6464 SelectPostLoad(
Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
6466 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6467 SelectPostLoad(
Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
6469 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6470 SelectPostLoad(
Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
6472 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6473 SelectPostLoad(
Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
6475 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6476 SelectPostLoad(
Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
6478 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6479 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6481 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6482 SelectPostLoad(
Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
6488 if (VT == MVT::v8i8) {
6489 SelectPostLoad(
Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
6491 }
else if (VT == MVT::v16i8) {
6492 SelectPostLoad(
Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
6494 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6495 SelectPostLoad(
Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
6497 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6498 SelectPostLoad(
Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
6500 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6501 SelectPostLoad(
Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
6503 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6504 SelectPostLoad(
Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
6506 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6507 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6509 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6510 SelectPostLoad(
Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
6516 if (VT == MVT::v8i8) {
6517 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
6519 }
else if (VT == MVT::v16i8) {
6520 SelectPostLoad(
Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
6522 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6523 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
6525 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6526 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
6528 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6529 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
6531 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6532 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
6534 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6535 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6537 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6538 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
6544 if (VT == MVT::v8i8) {
6545 SelectPostLoad(
Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
6547 }
else if (VT == MVT::v16i8) {
6548 SelectPostLoad(
Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
6550 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6551 SelectPostLoad(
Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
6553 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6554 SelectPostLoad(
Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
6556 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6557 SelectPostLoad(
Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
6559 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6560 SelectPostLoad(
Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
6562 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6563 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6565 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6566 SelectPostLoad(
Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
6572 if (VT == MVT::v8i8) {
6573 SelectPostLoad(
Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
6575 }
else if (VT == MVT::v16i8) {
6576 SelectPostLoad(
Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
6578 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6579 SelectPostLoad(
Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
6581 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6582 SelectPostLoad(
Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
6584 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6585 SelectPostLoad(
Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
6587 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6588 SelectPostLoad(
Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
6590 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6591 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6593 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6594 SelectPostLoad(
Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
6600 if (VT == MVT::v8i8) {
6601 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
6603 }
else if (VT == MVT::v16i8) {
6604 SelectPostLoad(
Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
6606 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6607 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
6609 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6610 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
6612 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6613 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
6615 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6616 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
6618 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6619 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6621 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6622 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
6628 if (VT == MVT::v8i8) {
6629 SelectPostLoad(
Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
6631 }
else if (VT == MVT::v16i8) {
6632 SelectPostLoad(
Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
6634 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6635 SelectPostLoad(
Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
6637 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6638 SelectPostLoad(
Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
6640 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6641 SelectPostLoad(
Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
6643 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6644 SelectPostLoad(
Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
6646 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6647 SelectPostLoad(
Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
6649 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6650 SelectPostLoad(
Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
6656 if (VT == MVT::v8i8) {
6657 SelectPostLoad(
Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
6659 }
else if (VT == MVT::v16i8) {
6660 SelectPostLoad(
Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
6662 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6663 SelectPostLoad(
Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
6665 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6666 SelectPostLoad(
Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
6668 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6669 SelectPostLoad(
Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
6671 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6672 SelectPostLoad(
Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
6674 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6675 SelectPostLoad(
Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
6677 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6678 SelectPostLoad(
Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
6684 if (VT == MVT::v8i8) {
6685 SelectPostLoad(
Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
6687 }
else if (VT == MVT::v16i8) {
6688 SelectPostLoad(
Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
6690 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6691 SelectPostLoad(
Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
6693 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6694 SelectPostLoad(
Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
6696 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6697 SelectPostLoad(
Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
6699 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6700 SelectPostLoad(
Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
6702 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6703 SelectPostLoad(
Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
6705 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6706 SelectPostLoad(
Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
6712 if (VT == MVT::v8i8) {
6713 SelectPostLoad(
Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
6715 }
else if (VT == MVT::v16i8) {
6716 SelectPostLoad(
Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
6718 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6719 SelectPostLoad(
Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
6721 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6722 SelectPostLoad(
Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
6724 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6725 SelectPostLoad(
Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
6727 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6728 SelectPostLoad(
Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
6730 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6731 SelectPostLoad(
Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
6733 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6734 SelectPostLoad(
Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
6740 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6741 SelectPostLoadLane(
Node, 1, AArch64::LD1i8_POST);
6743 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6744 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6745 SelectPostLoadLane(
Node, 1, AArch64::LD1i16_POST);
6747 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6749 SelectPostLoadLane(
Node, 1, AArch64::LD1i32_POST);
6751 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6753 SelectPostLoadLane(
Node, 1, AArch64::LD1i64_POST);
6759 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6760 SelectPostLoadLane(
Node, 2, AArch64::LD2i8_POST);
6762 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6763 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6764 SelectPostLoadLane(
Node, 2, AArch64::LD2i16_POST);
6766 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6768 SelectPostLoadLane(
Node, 2, AArch64::LD2i32_POST);
6770 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6772 SelectPostLoadLane(
Node, 2, AArch64::LD2i64_POST);
6778 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6779 SelectPostLoadLane(
Node, 3, AArch64::LD3i8_POST);
6781 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6782 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6783 SelectPostLoadLane(
Node, 3, AArch64::LD3i16_POST);
6785 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6787 SelectPostLoadLane(
Node, 3, AArch64::LD3i32_POST);
6789 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6791 SelectPostLoadLane(
Node, 3, AArch64::LD3i64_POST);
6797 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6798 SelectPostLoadLane(
Node, 4, AArch64::LD4i8_POST);
6800 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6801 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6802 SelectPostLoadLane(
Node, 4, AArch64::LD4i16_POST);
6804 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6806 SelectPostLoadLane(
Node, 4, AArch64::LD4i32_POST);
6808 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6810 SelectPostLoadLane(
Node, 4, AArch64::LD4i64_POST);
6816 VT =
Node->getOperand(1).getValueType();
6817 if (VT == MVT::v8i8) {
6818 SelectPostStore(
Node, 2, AArch64::ST2Twov8b_POST);
6820 }
else if (VT == MVT::v16i8) {
6821 SelectPostStore(
Node, 2, AArch64::ST2Twov16b_POST);
6823 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6824 SelectPostStore(
Node, 2, AArch64::ST2Twov4h_POST);
6826 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6827 SelectPostStore(
Node, 2, AArch64::ST2Twov8h_POST);
6829 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6830 SelectPostStore(
Node, 2, AArch64::ST2Twov2s_POST);
6832 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6833 SelectPostStore(
Node, 2, AArch64::ST2Twov4s_POST);
6835 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6836 SelectPostStore(
Node, 2, AArch64::ST2Twov2d_POST);
6838 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6839 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6845 VT =
Node->getOperand(1).getValueType();
6846 if (VT == MVT::v8i8) {
6847 SelectPostStore(
Node, 3, AArch64::ST3Threev8b_POST);
6849 }
else if (VT == MVT::v16i8) {
6850 SelectPostStore(
Node, 3, AArch64::ST3Threev16b_POST);
6852 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6853 SelectPostStore(
Node, 3, AArch64::ST3Threev4h_POST);
6855 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6856 SelectPostStore(
Node, 3, AArch64::ST3Threev8h_POST);
6858 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6859 SelectPostStore(
Node, 3, AArch64::ST3Threev2s_POST);
6861 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6862 SelectPostStore(
Node, 3, AArch64::ST3Threev4s_POST);
6864 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6865 SelectPostStore(
Node, 3, AArch64::ST3Threev2d_POST);
6867 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6868 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6874 VT =
Node->getOperand(1).getValueType();
6875 if (VT == MVT::v8i8) {
6876 SelectPostStore(
Node, 4, AArch64::ST4Fourv8b_POST);
6878 }
else if (VT == MVT::v16i8) {
6879 SelectPostStore(
Node, 4, AArch64::ST4Fourv16b_POST);
6881 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6882 SelectPostStore(
Node, 4, AArch64::ST4Fourv4h_POST);
6884 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6885 SelectPostStore(
Node, 4, AArch64::ST4Fourv8h_POST);
6887 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6888 SelectPostStore(
Node, 4, AArch64::ST4Fourv2s_POST);
6890 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6891 SelectPostStore(
Node, 4, AArch64::ST4Fourv4s_POST);
6893 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6894 SelectPostStore(
Node, 4, AArch64::ST4Fourv2d_POST);
6896 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6897 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6903 VT =
Node->getOperand(1).getValueType();
6904 if (VT == MVT::v8i8) {
6905 SelectPostStore(
Node, 2, AArch64::ST1Twov8b_POST);
6907 }
else if (VT == MVT::v16i8) {
6908 SelectPostStore(
Node, 2, AArch64::ST1Twov16b_POST);
6910 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6911 SelectPostStore(
Node, 2, AArch64::ST1Twov4h_POST);
6913 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6914 SelectPostStore(
Node, 2, AArch64::ST1Twov8h_POST);
6916 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6917 SelectPostStore(
Node, 2, AArch64::ST1Twov2s_POST);
6919 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6920 SelectPostStore(
Node, 2, AArch64::ST1Twov4s_POST);
6922 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6923 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6925 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6926 SelectPostStore(
Node, 2, AArch64::ST1Twov2d_POST);
6932 VT =
Node->getOperand(1).getValueType();
6933 if (VT == MVT::v8i8) {
6934 SelectPostStore(
Node, 3, AArch64::ST1Threev8b_POST);
6936 }
else if (VT == MVT::v16i8) {
6937 SelectPostStore(
Node, 3, AArch64::ST1Threev16b_POST);
6939 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6940 SelectPostStore(
Node, 3, AArch64::ST1Threev4h_POST);
6942 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
6943 SelectPostStore(
Node, 3, AArch64::ST1Threev8h_POST);
6945 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6946 SelectPostStore(
Node, 3, AArch64::ST1Threev2s_POST);
6948 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6949 SelectPostStore(
Node, 3, AArch64::ST1Threev4s_POST);
6951 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6952 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6954 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6955 SelectPostStore(
Node, 3, AArch64::ST1Threev2d_POST);
6961 VT =
Node->getOperand(1).getValueType();
6962 if (VT == MVT::v8i8) {
6963 SelectPostStore(
Node, 4, AArch64::ST1Fourv8b_POST);
6965 }
else if (VT == MVT::v16i8) {
6966 SelectPostStore(
Node, 4, AArch64::ST1Fourv16b_POST);
6968 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6969 SelectPostStore(
Node, 4, AArch64::ST1Fourv4h_POST);
6971 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6972 SelectPostStore(
Node, 4, AArch64::ST1Fourv8h_POST);
6974 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6975 SelectPostStore(
Node, 4, AArch64::ST1Fourv2s_POST);
6977 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6978 SelectPostStore(
Node, 4, AArch64::ST1Fourv4s_POST);
6980 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6981 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6983 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6984 SelectPostStore(
Node, 4, AArch64::ST1Fourv2d_POST);
6990 VT =
Node->getOperand(1).getValueType();
6991 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6992 SelectPostStoreLane(
Node, 2, AArch64::ST2i8_POST);
6994 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6995 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6996 SelectPostStoreLane(
Node, 2, AArch64::ST2i16_POST);
6998 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7000 SelectPostStoreLane(
Node, 2, AArch64::ST2i32_POST);
7002 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7004 SelectPostStoreLane(
Node, 2, AArch64::ST2i64_POST);
7010 VT =
Node->getOperand(1).getValueType();
7011 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7012 SelectPostStoreLane(
Node, 3, AArch64::ST3i8_POST);
7014 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7015 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7016 SelectPostStoreLane(
Node, 3, AArch64::ST3i16_POST);
7018 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7020 SelectPostStoreLane(
Node, 3, AArch64::ST3i32_POST);
7022 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7024 SelectPostStoreLane(
Node, 3, AArch64::ST3i64_POST);
7030 VT =
Node->getOperand(1).getValueType();
7031 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7032 SelectPostStoreLane(
Node, 4, AArch64::ST4i8_POST);
7034 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7035 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7036 SelectPostStoreLane(
Node, 4, AArch64::ST4i16_POST);
7038 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7040 SelectPostStoreLane(
Node, 4, AArch64::ST4i32_POST);
7042 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7044 SelectPostStoreLane(
Node, 4, AArch64::ST4i64_POST);
7050 if (VT == MVT::nxv16i8) {
7051 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B);
7053 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7054 VT == MVT::nxv8bf16) {
7055 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H);
7057 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7058 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W);
7060 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7061 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D);
7067 if (VT == MVT::nxv16i8) {
7068 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B);
7070 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7071 VT == MVT::nxv8bf16) {
7072 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H);
7074 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7075 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W);
7077 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7078 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D);
7084 if (VT == MVT::nxv16i8) {
7085 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B);
7087 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7088 VT == MVT::nxv8bf16) {
7089 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H);
7091 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7092 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W);
7094 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7095 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D);
7110 return new AArch64DAGToDAGISelLegacy(
TM, OptLevel);
7122 assert(NumVec > 0 && NumVec < 5 &&
"Invalid number of vectors.");
7126 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7127 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7141 if (isa<MemSDNode>(Root))
7142 return cast<MemSDNode>(Root)->getMemoryVT();
7144 if (isa<MemIntrinsicSDNode>(Root))
7145 return cast<MemIntrinsicSDNode>(Root)->getMemoryVT();
7147 const unsigned Opcode = Root->
getOpcode();
7155 return cast<VTSDNode>(Root->
getOperand(3))->getVT();
7157 return cast<VTSDNode>(Root->
getOperand(4))->getVT();
7177 case Intrinsic::aarch64_sme_ldr:
7178 case Intrinsic::aarch64_sme_str:
7179 return MVT::nxv16i8;
7180 case Intrinsic::aarch64_sve_prf:
7185 case Intrinsic::aarch64_sve_ld2_sret:
7186 case Intrinsic::aarch64_sve_ld2q_sret:
7189 case Intrinsic::aarch64_sve_st2q:
7192 case Intrinsic::aarch64_sve_ld3_sret:
7193 case Intrinsic::aarch64_sve_ld3q_sret:
7196 case Intrinsic::aarch64_sve_st3q:
7199 case Intrinsic::aarch64_sve_ld4_sret:
7200 case Intrinsic::aarch64_sve_ld4q_sret:
7203 case Intrinsic::aarch64_sve_st4q:
7206 case Intrinsic::aarch64_sve_ld1udq:
7207 case Intrinsic::aarch64_sve_st1dq:
7208 return EVT(MVT::nxv1i64);
7209 case Intrinsic::aarch64_sve_ld1uwq:
7210 case Intrinsic::aarch64_sve_st1wq:
7211 return EVT(MVT::nxv1i32);
7218template <
int64_t Min,
int64_t Max>
7219bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(
SDNode *Root,
SDValue N,
7227 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
7232 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
7251 int64_t MulImm = cast<ConstantSDNode>(VScale.
getOperand(0))->getSExtValue();
7253 if ((MulImm % MemWidthBytes) != 0)
7256 int64_t
Offset = MulImm / MemWidthBytes;
7257 if (Offset < Min || Offset > Max)
7260 Base =
N.getOperand(0);
7262 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
7269 OffImm = CurDAG->getTargetConstant(
Offset,
SDLoc(
N), MVT::i64);
7275bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(
SDValue N,
unsigned Scale,
7293 if (
auto C = dyn_cast<ConstantSDNode>(RHS)) {
7294 int64_t ImmOff =
C->getSExtValue();
7295 unsigned Size = 1 << Scale;
7304 Offset = CurDAG->getTargetConstant(ImmOff >> Scale,
DL, MVT::i64);
7306 SDNode *
MI = CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
7316 if (
auto *
C = dyn_cast<ConstantSDNode>(ShiftRHS))
7317 if (
C->getZExtValue() == Scale) {
7326bool AArch64DAGToDAGISel::SelectAllActivePredicate(
SDValue N) {
7333bool AArch64DAGToDAGISel::SelectAnyPredicate(
SDValue N) {
7334 EVT VT =
N.getValueType();
7338bool AArch64DAGToDAGISel::SelectSMETileSlice(
SDValue N,
unsigned MaxSize,
7343 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
7344 int64_t ImmOff =
C->getSExtValue();
7345 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0))) {
7346 Base =
N.getOperand(0);
7347 Offset = CurDAG->getTargetConstant(ImmOff / Scale,
SDLoc(
N), MVT::i64);
7354 Offset = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
static int getIntOperandFromRegisterString(StringRef RegString)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
Does DstMask form a complementary pair with the mask provided by BitsToBeInserted,...
static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N)
Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32,...
static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op, SDValue &Src, int &DstLSB, int &Width)
static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &DstLSB, int &Width)
Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL,...
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG)
static std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG)
static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
static bool isShiftedMask(uint64_t Mask, EVT VT)
bool SelectSMETile(unsigned &BaseReg, unsigned TileNum)
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root)
Return the EVT of the data associated to a memory operation in Root.
static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static bool isWorthFoldingADDlow(SDValue N)
If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD i...
static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N)
getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef< unsigned > Opcodes)
This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bi...
static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT, unsigned NumVec)
When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent sca...
static bool isPreferredADD(int64_t ImmOff)
static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
Create a machine node performing a notional SHL of Op by ShlAmount.
static bool isWorthFoldingSHL(SDValue V)
Determine whether it is worth it to fold SHL into the addressing mode.
static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1, SDValue Src, SDValue Dst, SelectionDAG *CurDAG, const bool BiggerPattern)
static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits, unsigned Depth)
static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth=0)
static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected)
static AArch64_AM::ShiftExtendType getExtendTypeForNode(SDValue N, bool IsLoadStore=false)
getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
static bool isIntImmediate(const SDNode *N, uint64_t &Imm)
isIntImmediate - This method tests to see if the node is a constant operand.
static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG, SDValue &ShiftedOperand, uint64_t &EncodedShiftImm)
static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range, unsigned Size)
Check if the immediate offset is valid as a scaled immediate.
static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
amdgpu AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
unsigned const TargetRegisterInfo * TRI
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool isLittleEndian() const
bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const
bool getExactInverse(APFloat *inv) const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
void flipAllBits()
Toggle every bit to its opposite value.
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
An arbitrary precision integer that knows its signedness.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
const Constant * getConstVal() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
int64_t getOffset() const
const GlobalValue * getGlobal() const
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint8_t getStackID(int ObjectIdx) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
static constexpr unsigned MaxRecursionDepth
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
unsigned getID() const
Return the register class ID number.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A Use represents the edge between a Value definition and its users.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
uint32_t parseGenericRegister(StringRef Name)
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static bool processLogicalImmediate(uint64_t Imm, unsigned RegSize, uint64_t &Encoding)
processLogicalImmediate - Determine if an immediate value can be encoded as the immediate operand of ...
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static constexpr unsigned SVEBitsPerBlock
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
@ Undef
Value of the register doesn't matter.
Not(const Pred &P) -> Not< Pred >
Reg
All possible values of the reg field in the ModR/M byte.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
FunctionPass * createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOptLevel OptLevel)
createAArch64ISelDag - This pass converts a legalized DAG into a AArch64-specific DAG,...
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
constexpr unsigned BitWidth
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
unsigned getBitWidth() const
Get the bit width of this value.