22#include "llvm/IR/IntrinsicsAArch64.h"
31#define DEBUG_TYPE "aarch64-isel"
32#define PASS_NAME "AArch64 Instruction Selection"
47 AArch64DAGToDAGISel() =
delete;
64 std::vector<SDValue> &OutOps)
override;
66 template <
signed Low,
signed High,
signed Scale>
74 return SelectShiftedRegister(
N,
false, Reg, Shift);
77 return SelectShiftedRegister(
N,
true, Reg, Shift);
80 return SelectAddrModeIndexed7S(
N, 1,
Base, OffImm);
83 return SelectAddrModeIndexed7S(
N, 2,
Base, OffImm);
86 return SelectAddrModeIndexed7S(
N, 4,
Base, OffImm);
89 return SelectAddrModeIndexed7S(
N, 8,
Base, OffImm);
92 return SelectAddrModeIndexed7S(
N, 16,
Base, OffImm);
95 return SelectAddrModeIndexedBitWidth(
N,
true, 9, 16,
Base, OffImm);
98 return SelectAddrModeIndexedBitWidth(
N,
false, 6, 16,
Base, OffImm);
101 return SelectAddrModeIndexed(
N, 1,
Base, OffImm);
104 return SelectAddrModeIndexed(
N, 2,
Base, OffImm);
107 return SelectAddrModeIndexed(
N, 4,
Base, OffImm);
110 return SelectAddrModeIndexed(
N, 8,
Base, OffImm);
113 return SelectAddrModeIndexed(
N, 16,
Base, OffImm);
116 return SelectAddrModeUnscaled(
N, 1,
Base, OffImm);
119 return SelectAddrModeUnscaled(
N, 2,
Base, OffImm);
122 return SelectAddrModeUnscaled(
N, 4,
Base, OffImm);
125 return SelectAddrModeUnscaled(
N, 8,
Base, OffImm);
128 return SelectAddrModeUnscaled(
N, 16,
Base, OffImm);
130 template <
unsigned Size,
unsigned Max>
134 bool Found = SelectAddrModeIndexed(
N,
Size,
Base, OffImm);
136 if (
auto *CI = dyn_cast<ConstantSDNode>(OffImm)) {
137 int64_t
C = CI->getSExtValue();
145 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
152 return SelectAddrModeWRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
158 return SelectAddrModeXRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
163 N =
N->getOperand(0);
165 !isa<ConstantSDNode>(
N->getOperand(1)))
167 EVT VT =
N->getValueType(0);
168 EVT LVT =
N->getOperand(0).getValueType();
169 unsigned Index =
N->getConstantOperandVal(1);
173 Res =
N->getOperand(0);
181 EVT VT =
Op.getValueType();
182 unsigned ShtAmt =
N->getConstantOperandVal(1);
189 Op.getOperand(1).getConstantOperandVal(0)
190 <<
Op.getOperand(1).getConstantOperandVal(1));
192 isa<ConstantSDNode>(
Op.getOperand(1).getOperand(0)))
194 Op.getOperand(1).getConstantOperandVal(0));
198 if (Imm != 1ULL << (ShtAmt - 1))
201 Res1 =
Op.getOperand(0);
202 Res2 = CurDAG->getTargetConstant(ShtAmt,
SDLoc(
N), MVT::i32);
206 bool SelectDupZeroOrUndef(
SDValue N) {
207 switch(
N->getOpcode()) {
212 auto Opnd0 =
N->getOperand(0);
227 switch(
N->getOpcode()) {
230 auto Opnd0 =
N->getOperand(0);
242 template<MVT::SimpleValueType VT>
244 return SelectSVEAddSubImm(
N, VT, Imm, Shift);
247 template <MVT::SimpleValueType VT,
bool Negate>
249 return SelectSVEAddSubSSatImm(
N, VT, Imm, Shift, Negate);
252 template <MVT::SimpleValueType VT>
254 return SelectSVECpyDupImm(
N, VT, Imm, Shift);
257 template <MVT::SimpleValueType VT,
bool Invert = false>
259 return SelectSVELogicalImm(
N, VT, Imm, Invert);
262 template <MVT::SimpleValueType VT>
264 return SelectSVEArithImm(
N, VT, Imm);
267 template <
unsigned Low,
unsigned High,
bool AllowSaturation = false>
269 return SelectSVEShiftImm(
N,
Low,
High, AllowSaturation, Imm);
276 EVT EltVT =
N->getValueType(0).getVectorElementType();
277 return SelectSVEShiftImm(
N->getOperand(0), 1,
283 template<
signed Min,
signed Max,
signed Scale,
bool Shift>
285 if (!isa<ConstantSDNode>(
N))
288 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
290 MulImm = 1LL << MulImm;
292 if ((MulImm % std::abs(Scale)) != 0)
296 if ((MulImm >= Min) && (MulImm <= Max)) {
297 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
304 template <
signed Max,
signed Scale>
306 if (!isa<ConstantSDNode>(
N))
309 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
311 if (MulImm >= 0 && MulImm <= Max) {
313 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
320 template <
unsigned BaseReg,
unsigned Max>
322 if (
auto *CI = dyn_cast<ConstantSDNode>(
N)) {
328 Imm = CurDAG->getRegister(BaseReg +
C, MVT::Other);
351 const unsigned SubRegs[]);
353 void SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
355 bool tryIndexedLoad(
SDNode *
N);
357 void SelectPtrauthAuth(
SDNode *
N);
358 void SelectPtrauthResign(
SDNode *
N);
360 bool trySelectStackSlotTagP(
SDNode *
N);
363 void SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
365 void SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
367 void SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
368 void SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
369 void SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
370 unsigned Opc_rr,
unsigned Opc_ri,
371 bool IsIntr =
false);
372 void SelectContiguousMultiVectorLoad(
SDNode *
N,
unsigned NumVecs,
373 unsigned Scale,
unsigned Opc_ri,
375 void SelectDestructiveMultiIntrinsic(
SDNode *
N,
unsigned NumVecs,
376 bool IsZmMulti,
unsigned Opcode,
377 bool HasPred =
false);
378 void SelectPExtPair(
SDNode *
N,
unsigned Opc);
379 void SelectWhilePair(
SDNode *
N,
unsigned Opc);
380 void SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
381 void SelectClamp(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
382 void SelectUnaryMultiIntrinsic(
SDNode *
N,
unsigned NumOutVecs,
383 bool IsTupleInput,
unsigned Opc);
384 void SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
386 template <
unsigned MaxIdx,
unsigned Scale>
387 void SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
unsigned BaseReg,
389 void SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
390 unsigned Op,
unsigned MaxIdx,
unsigned Scale,
391 unsigned BaseReg = 0);
394 template <
int64_t Min,
int64_t Max>
398 template <
unsigned Scale>
400 return SelectSVERegRegAddrMode(
N, Scale,
Base,
Offset);
403 void SelectMultiVectorLuti(
SDNode *
Node,
unsigned NumOutVecs,
unsigned Opc,
406 template <
unsigned MaxIdx,
unsigned Scale>
411 void SelectStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
412 void SelectPostStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
413 void SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
414 void SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
415 void SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
416 unsigned Opc_rr,
unsigned Opc_ri);
417 std::tuple<unsigned, SDValue, SDValue>
418 findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
unsigned Opc_ri,
422 bool tryBitfieldExtractOp(
SDNode *
N);
423 bool tryBitfieldExtractOpFromSExt(
SDNode *
N);
424 bool tryBitfieldInsertOp(
SDNode *
N);
425 bool tryBitfieldInsertInZeroOp(
SDNode *
N);
426 bool tryShiftAmountMod(
SDNode *
N);
428 bool tryReadRegister(
SDNode *
N);
429 bool tryWriteRegister(
SDNode *
N);
431 bool trySelectCastFixedLengthToScalableVector(
SDNode *
N);
432 bool trySelectCastScalableToFixedLengthVector(
SDNode *
N);
437#include "AArch64GenDAGISel.inc"
445 return SelectAddrModeIndexedBitWidth(
N,
true, 7,
Size,
Base, OffImm);
447 bool SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
unsigned BW,
460 bool isWorthFoldingALU(
SDValue V,
bool LSL =
false)
const;
461 bool isWorthFoldingAddr(
SDValue V,
unsigned Size)
const;
462 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
465 template<
unsigned RegW
idth>
467 return SelectCVTFixedPosOperand(
N, FixedPos, RegWidth);
470 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
472 template<
unsigned RegW
idth>
474 return SelectCVTFixedPosRecipOperand(
N, FixedPos, RegWidth);
480 bool SelectCMP_SWAP(
SDNode *
N);
490 bool AllowSaturation,
SDValue &Imm);
498 bool SelectAllActivePredicate(
SDValue N);
508 ID, std::make_unique<AArch64DAGToDAGISel>(tm, OptLevel)) {}
512char AArch64DAGToDAGISelLegacy::ID = 0;
520 Imm =
C->getZExtValue();
537 return N->getOpcode() == Opc &&
548 return Imm == ImmExpected;
552bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
554 std::vector<SDValue> &OutOps) {
555 switch(ConstraintID) {
558 case InlineAsm::ConstraintCode::m:
559 case InlineAsm::ConstraintCode::o:
560 case InlineAsm::ConstraintCode::Q:
566 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(), dl, MVT::i64);
568 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
569 dl,
Op.getValueType(),
571 OutOps.push_back(NewOp);
587 if (!isa<ConstantSDNode>(
N.getNode()))
590 uint64_t Immed =
N.getNode()->getAsZExtVal();
593 if (Immed >> 12 == 0) {
595 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
603 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
604 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
617 if (!isa<ConstantSDNode>(
N.getNode()))
621 uint64_t Immed =
N.getNode()->getAsZExtVal();
629 if (
N.getValueType() == MVT::i32)
632 Immed = ~Immed + 1ULL;
633 if (Immed & 0xFFFFFFFFFF000000ULL)
636 Immed &= 0xFFFFFFULL;
637 return SelectArithImmed(CurDAG->getConstant(Immed,
SDLoc(
N), MVT::i32), Val,
644 switch (
N.getOpcode()) {
663 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
666 unsigned ShiftVal = CSD->getZExtValue();
675 if (!isa<MemSDNode>(*UI))
677 if (!isa<MemSDNode>(*UII))
684bool AArch64DAGToDAGISel::isWorthFoldingAddr(
SDValue V,
unsigned Size)
const {
687 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
692 if (Subtarget->hasAddrLSLSlow14() && (
Size == 2 ||
Size == 16))
714bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(
SDValue N,
SDValue &Reg,
716 EVT VT =
N.getValueType();
717 if (VT != MVT::i32 && VT != MVT::i64)
720 if (
N->getOpcode() !=
ISD::AND || !
N->hasOneUse())
726 unsigned LHSOpcode =
LHS->getOpcode();
740 unsigned LowZBits, MaskLen;
744 unsigned BitWidth =
N.getValueSizeInBits();
751 if (LowZBits <= ShiftAmtC || (
BitWidth != LowZBits + MaskLen))
754 NewShiftC = LowZBits - ShiftAmtC;
755 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
761 NewShiftC = LowZBits + ShiftAmtC;
774 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
776 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
780 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC,
DL, VT);
782 Reg =
SDValue(CurDAG->getMachineNode(NewShiftOp,
DL, VT,
LHS->getOperand(0),
783 NewShiftAmt, BitWidthMinus1),
786 Shift = CurDAG->getTargetConstant(ShVal,
DL, MVT::i32);
798 SrcVT = cast<VTSDNode>(
N.getOperand(1))->getVT();
800 SrcVT =
N.getOperand(0).getValueType();
802 if (!IsLoadStore && SrcVT == MVT::i8)
804 else if (!IsLoadStore && SrcVT == MVT::i16)
806 else if (SrcVT == MVT::i32)
808 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
813 EVT SrcVT =
N.getOperand(0).getValueType();
814 if (!IsLoadStore && SrcVT == MVT::i8)
816 else if (!IsLoadStore && SrcVT == MVT::i16)
818 else if (SrcVT == MVT::i32)
820 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
848bool AArch64DAGToDAGISel::isWorthFoldingALU(
SDValue V,
bool LSL)
const {
851 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
856 if (LSL && Subtarget->hasALULSLFast() &&
V.getOpcode() ==
ISD::SHL &&
857 V.getConstantOperandVal(1) <= 4 &&
870bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
872 if (SelectShiftedRegisterFromAnd(
N, Reg, Shift))
882 unsigned BitSize =
N.getValueSizeInBits();
883 unsigned Val =
RHS->getZExtValue() & (BitSize - 1);
886 Reg =
N.getOperand(0);
887 Shift = CurDAG->getTargetConstant(ShVal,
SDLoc(
N), MVT::i32);
888 return isWorthFoldingALU(
N,
true);
899 if (
N.getValueType() == MVT::i32)
907template<
signed Low,
signed High,
signed Scale>
909 if (!isa<ConstantSDNode>(
N))
912 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
913 if ((MulImm % std::abs(Scale)) == 0) {
914 int64_t RDVLImm = MulImm / Scale;
915 if ((RDVLImm >=
Low) && (RDVLImm <=
High)) {
916 Imm = CurDAG->getTargetConstant(RDVLImm,
SDLoc(
N), MVT::i32);
926bool AArch64DAGToDAGISel::SelectArithExtendedRegister(
SDValue N,
SDValue &Reg,
928 unsigned ShiftVal = 0;
943 Reg =
N.getOperand(0).getOperand(0);
949 Reg =
N.getOperand(0);
954 unsigned Opc =
N.getOpcode();
955 return Opc !=
ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
972 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
974 return isWorthFoldingALU(
N);
979bool AArch64DAGToDAGISel::SelectArithUXTXRegister(
SDValue N,
SDValue &Reg,
981 unsigned ShiftVal = 0;
995 Reg =
N.getOperand(0);
996 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
998 return isWorthFoldingALU(
N);
1007 for (
auto *
Use :
N->uses()) {
1034bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
1035 unsigned BW,
unsigned Size,
1042 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1044 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1050 if (CurDAG->isBaseWithConstantOffset(
N)) {
1051 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1053 int64_t RHSC =
RHS->getSExtValue();
1055 int64_t
Range = 0x1LL << (BW - 1);
1057 if ((RHSC & (
Size - 1)) == 0 && RHSC >= -(
Range << Scale) &&
1058 RHSC < (
Range << Scale)) {
1059 Base =
N.getOperand(0);
1061 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1064 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1073 if ((RHSC & (
Size - 1)) == 0 && RHSC < (
Range << Scale)) {
1074 Base =
N.getOperand(0);
1076 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1079 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1090 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1097bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
1103 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1105 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1111 dyn_cast<GlobalAddressSDNode>(
N.getOperand(1).getNode());
1112 Base =
N.getOperand(0);
1113 OffImm =
N.getOperand(1);
1122 if (CurDAG->isBaseWithConstantOffset(
N)) {
1123 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1124 int64_t RHSC = (int64_t)
RHS->getZExtValue();
1127 Base =
N.getOperand(0);
1129 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1132 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1140 if (SelectAddrModeUnscaled(
N,
Size,
Base, OffImm))
1148 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1157bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
1160 if (!CurDAG->isBaseWithConstantOffset(
N))
1162 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1163 int64_t RHSC =
RHS->getSExtValue();
1164 if (RHSC >= -256 && RHSC < 256) {
1165 Base =
N.getOperand(0);
1167 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1169 Base = CurDAG->getTargetFrameIndex(
1172 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i64);
1182 CurDAG->
getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1189bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
1209 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1215 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1218 return isWorthFoldingAddr(
N,
Size);
1221bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
1233 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
1241 if (!isa<MemSDNode>(*UI))
1246 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1249 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1250 SelectExtendedSHL(RHS,
Size,
true,
Offset, SignExtend)) {
1252 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1257 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1258 SelectExtendedSHL(LHS,
Size,
true,
Offset, SignExtend)) {
1260 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1265 DoShift = CurDAG->getTargetConstant(
false, dl, MVT::i32);
1269 if (IsExtendedRegisterWorthFolding &&
1276 if (isWorthFoldingAddr(LHS,
Size))
1281 if (IsExtendedRegisterWorthFolding &&
1288 if (isWorthFoldingAddr(RHS,
Size))
1300 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1303 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1305 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1306 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1310bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
1325 if (!isa<MemSDNode>(*UI))
1340 if (isa<ConstantSDNode>(RHS)) {
1341 int64_t ImmOff = (int64_t)
RHS->getAsZExtVal();
1351 CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
1354 N = CurDAG->getNode(
ISD::ADD,
DL, MVT::i64, LHS, MOVIV);
1358 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1361 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1362 SelectExtendedSHL(RHS,
Size,
false,
Offset, SignExtend)) {
1364 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1369 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1370 SelectExtendedSHL(LHS,
Size,
false,
Offset, SignExtend)) {
1372 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1379 SignExtend = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1380 DoShift = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1386 static const unsigned RegClassIDs[] = {
1387 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1388 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1389 AArch64::dsub2, AArch64::dsub3};
1395 static const unsigned RegClassIDs[] = {
1396 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1397 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1398 AArch64::qsub2, AArch64::qsub3};
1404 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1405 AArch64::ZPR3RegClassID,
1406 AArch64::ZPR4RegClassID};
1407 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1408 AArch64::zsub2, AArch64::zsub3};
1418 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1419 AArch64::ZPR4Mul4RegClassID};
1420 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1421 AArch64::zsub2, AArch64::zsub3};
1426 const unsigned RegClassIDs[],
1427 const unsigned SubRegs[]) {
1430 if (Regs.
size() == 1)
1441 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2],
DL, MVT::i32));
1444 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1446 Ops.
push_back(CurDAG->getTargetConstant(SubRegs[i],
DL, MVT::i32));
1450 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped, Ops);
1454void AArch64DAGToDAGISel::SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1457 EVT VT =
N->getValueType(0);
1459 unsigned ExtOff = isExt;
1462 unsigned Vec0Off = ExtOff + 1;
1464 N->op_begin() + Vec0Off + NumVecs);
1471 Ops.
push_back(
N->getOperand(NumVecs + ExtOff + 1));
1472 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1475static std::tuple<SDValue, SDValue>
1495 auto *ConstDiscN = dyn_cast<ConstantSDNode>(ConstDisc);
1496 if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue()))
1501 AddrDisc = DAG->
getRegister(AArch64::XZR, MVT::i64);
1503 return std::make_tuple(
1508void AArch64DAGToDAGISel::SelectPtrauthAuth(
SDNode *
N) {
1513 SDValue AUTDisc =
N->getOperand(3);
1515 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1516 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
1518 SDValue AUTAddrDisc, AUTConstDisc;
1519 std::tie(AUTConstDisc, AUTAddrDisc) =
1522 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
1523 AArch64::X16, Val,
SDValue());
1524 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.
getValue(1)};
1526 SDNode *AUT = CurDAG->getMachineNode(AArch64::AUT,
DL, MVT::i64, Ops);
1527 ReplaceNode(
N, AUT);
1531void AArch64DAGToDAGISel::SelectPtrauthResign(
SDNode *
N) {
1536 SDValue AUTDisc =
N->getOperand(3);
1538 SDValue PACDisc =
N->getOperand(5);
1540 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1541 unsigned PACKeyC = cast<ConstantSDNode>(PACKey)->getZExtValue();
1543 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
1544 PACKey = CurDAG->getTargetConstant(PACKeyC,
DL, MVT::i64);
1546 SDValue AUTAddrDisc, AUTConstDisc;
1547 std::tie(AUTConstDisc, AUTAddrDisc) =
1550 SDValue PACAddrDisc, PACConstDisc;
1551 std::tie(PACConstDisc, PACAddrDisc) =
1554 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
1555 AArch64::X16, Val,
SDValue());
1557 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1558 PACConstDisc, PACAddrDisc, X16Copy.
getValue(1)};
1560 SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC,
DL, MVT::i64, Ops);
1561 ReplaceNode(
N, AUTPAC);
1565bool AArch64DAGToDAGISel::tryIndexedLoad(
SDNode *
N) {
1567 if (
LD->isUnindexed())
1569 EVT VT =
LD->getMemoryVT();
1570 EVT DstVT =
N->getValueType(0);
1577 unsigned Opcode = 0;
1580 bool InsertTo64 =
false;
1582 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1583 else if (VT == MVT::i32) {
1585 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1587 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1589 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1595 }
else if (VT == MVT::i16) {
1597 if (DstVT == MVT::i64)
1598 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1600 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1602 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1603 InsertTo64 = DstVT == MVT::i64;
1608 }
else if (VT == MVT::i8) {
1610 if (DstVT == MVT::i64)
1611 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1613 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1615 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1616 InsertTo64 = DstVT == MVT::i64;
1621 }
else if (VT == MVT::f16) {
1622 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1623 }
else if (VT == MVT::bf16) {
1624 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1625 }
else if (VT == MVT::f32) {
1626 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1628 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1630 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1638 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1640 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1645 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {
MemOp});
1650 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1652 SDValue(CurDAG->getMachineNode(
1653 AArch64::SUBREG_TO_REG, dl, MVT::i64,
1654 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1659 ReplaceUses(
SDValue(
N, 0), LoadedVal);
1662 CurDAG->RemoveDeadNode(
N);
1666void AArch64DAGToDAGISel::SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1667 unsigned SubRegIdx) {
1669 EVT VT =
N->getValueType(0);
1675 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1677 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1679 for (
unsigned i = 0; i < NumVecs; ++i)
1681 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1687 if (
auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(
N)) {
1689 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
1692 CurDAG->RemoveDeadNode(
N);
1695void AArch64DAGToDAGISel::SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
1696 unsigned Opc,
unsigned SubRegIdx) {
1698 EVT VT =
N->getValueType(0);
1705 const EVT ResTys[] = {MVT::i64,
1706 MVT::Untyped, MVT::Other};
1708 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1716 ReplaceUses(
SDValue(
N, 0), SuperReg);
1718 for (
unsigned i = 0; i < NumVecs; ++i)
1720 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1724 CurDAG->RemoveDeadNode(
N);
1730std::tuple<unsigned, SDValue, SDValue>
1731AArch64DAGToDAGISel::findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
1737 SDValue NewOffset = OldOffset;
1739 const bool IsRegImm = SelectAddrModeIndexedSVE<-8, 7>(
1740 N, OldBase, NewBase, NewOffset);
1744 const bool IsRegReg =
1745 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
1748 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
1761template <SelectTypeKind Kind>
1773 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
1778 if (EltVT != MVT::i1)
1782 if (EltVT == MVT::bf16)
1784 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
1814void AArch64DAGToDAGISel::SelectPExtPair(
SDNode *
N,
unsigned Opc) {
1817 if (
Imm->getZExtValue() > 1)
1821 EVT VT =
N->getValueType(0);
1822 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1823 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1826 for (
unsigned I = 0;
I < 2; ++
I)
1827 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1828 AArch64::psub0 +
I,
DL, VT, SuperReg));
1830 CurDAG->RemoveDeadNode(
N);
1833void AArch64DAGToDAGISel::SelectWhilePair(
SDNode *
N,
unsigned Opc) {
1835 EVT VT =
N->getValueType(0);
1837 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1839 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1842 for (
unsigned I = 0;
I < 2; ++
I)
1843 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1844 AArch64::psub0 +
I,
DL, VT, SuperReg));
1846 CurDAG->RemoveDeadNode(
N);
1849void AArch64DAGToDAGISel::SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
1851 EVT VT =
N->getValueType(0);
1853 SDValue Ops = createZTuple(Regs);
1857 for (
unsigned i = 0; i < NumVecs; ++i)
1858 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1859 AArch64::zsub0 + i,
DL, VT, SuperReg));
1861 CurDAG->RemoveDeadNode(
N);
1864void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(
SDNode *
N,
1869 assert(Opcode != 0 &&
"Unexpected opcode");
1872 EVT VT =
N->getValueType(0);
1873 unsigned FirstVecIdx = HasPred ? 2 : 1;
1875 auto GetMultiVecOperand = [=](
unsigned StartIdx) {
1877 return createZMulTuple(Regs);
1880 SDValue Zdn = GetMultiVecOperand(FirstVecIdx);
1884 Zm = GetMultiVecOperand(NumVecs + FirstVecIdx);
1886 Zm =
N->getOperand(NumVecs + FirstVecIdx);
1890 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped,
1891 N->getOperand(1), Zdn, Zm);
1893 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped, Zdn, Zm);
1895 for (
unsigned i = 0; i < NumVecs; ++i)
1896 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1897 AArch64::zsub0 + i,
DL, VT, SuperReg));
1899 CurDAG->RemoveDeadNode(
N);
1902void AArch64DAGToDAGISel::SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
1903 unsigned Scale,
unsigned Opc_ri,
1904 unsigned Opc_rr,
bool IsIntr) {
1905 assert(Scale < 5 &&
"Invalid scaling value.");
1907 EVT VT =
N->getValueType(0);
1913 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
1914 N, Opc_rr, Opc_ri,
N->getOperand(IsIntr ? 3 : 2),
1915 CurDAG->getTargetConstant(0,
DL, MVT::i64), Scale);
1917 SDValue Ops[] = {
N->getOperand(IsIntr ? 2 : 1),
1921 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1923 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1925 for (
unsigned i = 0; i < NumVecs; ++i)
1926 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1927 AArch64::zsub0 + i,
DL, VT, SuperReg));
1930 unsigned ChainIdx = NumVecs;
1932 CurDAG->RemoveDeadNode(
N);
1935void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(
SDNode *
N,
1940 assert(Scale < 4 &&
"Invalid scaling value.");
1942 EVT VT =
N->getValueType(0);
1950 findAddrModeSVELoadStore(
N, Opc_rr, Opc_ri,
Base,
Offset, Scale);
1956 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1958 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1960 for (
unsigned i = 0; i < NumVecs; ++i)
1961 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1962 AArch64::zsub0 + i,
DL, VT, SuperReg));
1965 unsigned ChainIdx = NumVecs;
1967 CurDAG->RemoveDeadNode(
N);
1970void AArch64DAGToDAGISel::SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
1972 if (
N->getValueType(0) != MVT::nxv4f32)
1974 SelectUnaryMultiIntrinsic(
N, NumVecs,
true, Opcode);
1977void AArch64DAGToDAGISel::SelectMultiVectorLuti(
SDNode *
Node,
1978 unsigned NumOutVecs,
1981 if (
Imm->getZExtValue() > MaxImm)
1985 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
1989 EVT VT =
Node->getValueType(0);
1992 CurDAG->getMachineNode(Opc,
DL, {MVT::Untyped, MVT::Other}, Ops);
1995 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
1996 ReplaceUses(
SDValue(
Node,
I), CurDAG->getTargetExtractSubreg(
1997 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2000 unsigned ChainIdx = NumOutVecs;
2002 CurDAG->RemoveDeadNode(
Node);
2005void AArch64DAGToDAGISel::SelectClamp(
SDNode *
N,
unsigned NumVecs,
2008 EVT VT =
N->getValueType(0);
2011 SDValue Zd = createZMulTuple(Regs);
2012 SDValue Zn =
N->getOperand(1 + NumVecs);
2013 SDValue Zm =
N->getOperand(2 + NumVecs);
2019 for (
unsigned i = 0; i < NumVecs; ++i)
2020 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
2021 AArch64::zsub0 + i,
DL, VT, SuperReg));
2023 CurDAG->RemoveDeadNode(
N);
2053template <
unsigned MaxIdx,
unsigned Scale>
2054void AArch64DAGToDAGISel::SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
2055 unsigned BaseReg,
unsigned Op) {
2056 unsigned TileNum = 0;
2057 if (BaseReg != AArch64::ZA)
2058 TileNum =
N->getConstantOperandVal(2);
2064 if (BaseReg == AArch64::ZA)
2065 SliceBase =
N->getOperand(2);
2067 SliceBase =
N->getOperand(3);
2069 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2075 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
2077 EVT VT =
N->getValueType(0);
2078 for (
unsigned I = 0;
I < NumVecs; ++
I)
2080 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2083 unsigned ChainIdx = NumVecs;
2085 CurDAG->RemoveDeadNode(
N);
2088void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
2089 unsigned Op,
unsigned MaxIdx,
2090 unsigned Scale,
unsigned BaseReg) {
2094 SDValue SliceBase =
N->getOperand(2);
2095 if (BaseReg != AArch64::ZA)
2096 SliceBase =
N->getOperand(3);
2099 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2106 if (BaseReg != AArch64::ZA )
2111 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
2113 EVT VT =
N->getValueType(0);
2114 for (
unsigned I = 0;
I < NumVecs; ++
I)
2116 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2120 unsigned ChainIdx = NumVecs;
2122 CurDAG->RemoveDeadNode(
N);
2125void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(
SDNode *
N,
2126 unsigned NumOutVecs,
2130 EVT VT =
N->getValueType(0);
2131 unsigned NumInVecs =
N->getNumOperands() - 1;
2135 assert((NumInVecs == 2 || NumInVecs == 4) &&
2136 "Don't know how to handle multi-register input!");
2141 for (
unsigned I = 0;
I < NumInVecs;
I++)
2145 SDNode *Res = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
2148 for (
unsigned I = 0;
I < NumOutVecs;
I++)
2149 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2150 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2151 CurDAG->RemoveDeadNode(
N);
2154void AArch64DAGToDAGISel::SelectStore(
SDNode *
N,
unsigned NumVecs,
2157 EVT VT =
N->getOperand(2)->getValueType(0);
2164 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
N->getOperand(0)};
2165 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2169 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2174void AArch64DAGToDAGISel::SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
2175 unsigned Scale,
unsigned Opc_rr,
2181 SDValue RegSeq = createZTuple(Regs);
2186 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
2187 N, Opc_rr, Opc_ri,
N->getOperand(NumVecs + 3),
2188 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2190 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
2194 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2206 if (
auto FINode = dyn_cast<FrameIndexSDNode>(
N)) {
2207 int FI = FINode->getIndex();
2209 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
2216void AArch64DAGToDAGISel::SelectPostStore(
SDNode *
N,
unsigned NumVecs,
2219 EVT VT =
N->getOperand(2)->getValueType(0);
2220 const EVT ResTys[] = {MVT::i64,
2229 N->getOperand(NumVecs + 1),
2230 N->getOperand(NumVecs + 2),
2232 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2272void AArch64DAGToDAGISel::SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
2275 EVT VT =
N->getValueType(0);
2283 WidenVector(*CurDAG));
2287 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2289 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2291 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2292 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2293 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2297 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2298 AArch64::qsub2, AArch64::qsub3 };
2299 for (
unsigned i = 0; i < NumVecs; ++i) {
2300 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2307 CurDAG->RemoveDeadNode(
N);
2310void AArch64DAGToDAGISel::SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
2313 EVT VT =
N->getValueType(0);
2321 WidenVector(*CurDAG));
2325 const EVT ResTys[] = {MVT::i64,
2328 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2331 CurDAG->getTargetConstant(LaneNo, dl,
2333 N->getOperand(NumVecs + 2),
2334 N->getOperand(NumVecs + 3),
2336 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2348 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2349 AArch64::qsub2, AArch64::qsub3 };
2350 for (
unsigned i = 0; i < NumVecs; ++i) {
2351 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
2361 CurDAG->RemoveDeadNode(
N);
2364void AArch64DAGToDAGISel::SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
2367 EVT VT =
N->getOperand(2)->getValueType(0);
2375 WidenVector(*CurDAG));
2379 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2381 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2382 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2383 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
2387 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2392void AArch64DAGToDAGISel::SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
2395 EVT VT =
N->getOperand(2)->getValueType(0);
2403 WidenVector(*CurDAG));
2407 const EVT ResTys[] = {MVT::i64,
2410 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2412 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2413 N->getOperand(NumVecs + 2),
2414 N->getOperand(NumVecs + 3),
2416 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2420 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2427 unsigned &LSB,
unsigned &MSB,
2428 unsigned NumberOfIgnoredLowBits,
2429 bool BiggerPattern) {
2431 "N must be a AND operation to call this function");
2433 EVT VT =
N->getValueType(0);
2438 assert((VT == MVT::i32 || VT == MVT::i64) &&
2439 "Type checking must have been done before calling this function");
2453 const SDNode *Op0 =
N->getOperand(0).getNode();
2457 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
2460 if (AndImm & (AndImm + 1))
2463 bool ClampMSB =
false;
2483 ClampMSB = (VT == MVT::i32);
2484 }
else if (BiggerPattern) {
2490 Opd0 =
N->getOperand(0);
2496 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.
getSizeInBits())) {
2499 <<
": Found large shift immediate, this should not happen\n"));
2505 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(AndImm)
2506 : llvm::countr_one<uint64_t>(AndImm)) -
2513 MSB = MSB > 31 ? 31 : MSB;
2515 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2520 SDValue &Opd0,
unsigned &Immr,
2524 EVT VT =
N->getValueType(0);
2526 assert((VT == MVT::i32 || VT == MVT::i64) &&
2527 "Type checking must have been done before calling this function");
2531 Op =
Op->getOperand(0);
2532 VT =
Op->getValueType(0);
2541 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
2545 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2546 Opd0 =
Op.getOperand(0);
2548 Imms = ShiftImm + Width - 1;
2576 Opd0 =
N->getOperand(0).getOperand(0);
2586 Opc =
N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2593 unsigned &Immr,
unsigned &Imms,
2594 bool BiggerPattern) {
2596 "N must be a SHR/SRA operation to call this function");
2598 EVT VT =
N->getValueType(0);
2603 assert((VT == MVT::i32 || VT == MVT::i64) &&
2604 "Type checking must have been done before calling this function");
2614 Opd0 =
N->getOperand(0).getOperand(0);
2615 }
else if (VT == MVT::i32 &&
N->getOpcode() ==
ISD::SRL &&
2621 Opd0 =
N->getOperand(0).getOperand(0);
2624 assert(VT == MVT::i64 &&
"the promoted type should be i64");
2625 }
else if (BiggerPattern) {
2629 Opd0 =
N->getOperand(0);
2638 <<
": Found large shift immediate, this should not happen\n"));
2647 "bad amount in shift node!");
2648 int immr = SrlImm - ShlImm;
2653 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
2655 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
2659bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(
SDNode *
N) {
2662 EVT VT =
N->getValueType(0);
2663 EVT NarrowVT =
N->getOperand(0)->getValueType(0);
2664 if (VT != MVT::i64 || NarrowVT != MVT::i32)
2675 unsigned Immr = ShiftImm;
2677 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2678 CurDAG->getTargetConstant(Imms, dl, VT)};
2679 CurDAG->SelectNodeTo(
N, AArch64::SBFMXri, VT, Ops);
2684 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
2685 unsigned NumberOfIgnoredLowBits = 0,
2686 bool BiggerPattern =
false) {
2687 if (
N->getValueType(0) != MVT::i32 &&
N->getValueType(0) != MVT::i64)
2690 switch (
N->getOpcode()) {
2692 if (!
N->isMachineOpcode())
2697 NumberOfIgnoredLowBits, BiggerPattern);
2706 unsigned NOpc =
N->getMachineOpcode();
2710 case AArch64::SBFMWri:
2711 case AArch64::UBFMWri:
2712 case AArch64::SBFMXri:
2713 case AArch64::UBFMXri:
2715 Opd0 =
N->getOperand(0);
2716 Immr =
N->getConstantOperandVal(1);
2717 Imms =
N->getConstantOperandVal(2);
2724bool AArch64DAGToDAGISel::tryBitfieldExtractOp(
SDNode *
N) {
2725 unsigned Opc, Immr, Imms;
2730 EVT VT =
N->getValueType(0);
2735 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
2736 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
2737 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
2739 SDNode *
BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
2740 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
2746 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2747 CurDAG->getTargetConstant(Imms, dl, VT)};
2748 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
2757 unsigned NumberOfIgnoredHighBits,
EVT VT) {
2758 assert((VT == MVT::i32 || VT == MVT::i64) &&
2759 "i32 or i64 mask type expected!");
2765 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
2766 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
2789 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2799 APInt OpUsefulBits(UsefulBits);
2803 OpUsefulBits <<= MSB - Imm + 1;
2808 OpUsefulBits <<= Imm;
2810 OpUsefulBits <<= MSB + 1;
2813 OpUsefulBits <<= OpUsefulBits.
getBitWidth() - Imm;
2819 UsefulBits &= OpUsefulBits;
2825 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2827 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2835 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2836 APInt Mask(UsefulBits);
2837 Mask.clearAllBits();
2845 Mask.lshrInPlace(ShiftAmt);
2851 Mask.lshrInPlace(ShiftAmt);
2863 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2865 cast<const ConstantSDNode>(
Op.getOperand(3).getNode())->getZExtValue();
2867 APInt OpUsefulBits(UsefulBits);
2881 OpUsefulBits <<= Width;
2884 if (
Op.getOperand(1) == Orig) {
2886 Mask = ResultUsefulBits & OpUsefulBits;
2890 if (
Op.getOperand(0) == Orig)
2892 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2898 OpUsefulBits <<= Width;
2900 OpUsefulBits <<= LSB;
2902 if (
Op.getOperand(1) == Orig) {
2904 Mask = ResultUsefulBits & OpUsefulBits;
2905 Mask.lshrInPlace(LSB);
2908 if (
Op.getOperand(0) == Orig)
2909 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2926 case AArch64::ANDSWri:
2927 case AArch64::ANDSXri:
2928 case AArch64::ANDWri:
2929 case AArch64::ANDXri:
2933 case AArch64::UBFMWri:
2934 case AArch64::UBFMXri:
2937 case AArch64::ORRWrs:
2938 case AArch64::ORRXrs:
2943 case AArch64::BFMWri:
2944 case AArch64::BFMXri:
2947 case AArch64::STRBBui:
2948 case AArch64::STURBBi:
2954 case AArch64::STRHHui:
2955 case AArch64::STURHHi:
2968 unsigned Bitwidth =
Op.getScalarValueSizeInBits();
2970 UsefulBits =
APInt(Bitwidth, 0);
2979 UsersUsefulBits |= UsefulBitsForUse;
2984 UsefulBits &= UsersUsefulBits;
2994 EVT VT =
Op.getValueType();
2997 unsigned UBFMOpc =
BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3000 if (ShlAmount > 0) {
3003 UBFMOpc, dl, VT,
Op,
3008 assert(ShlAmount < 0 &&
"expected right shift");
3009 int ShrAmount = -ShlAmount;
3035 bool BiggerPattern,
SDValue &Src,
3036 int &DstLSB,
int &Width) {
3037 EVT VT =
Op.getValueType();
3046 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
3050 switch (
Op.getOpcode()) {
3055 NonZeroBits, Src, DstLSB, Width);
3058 NonZeroBits, Src, DstLSB, Width);
3071 EVT VT =
Op.getValueType();
3072 assert((VT == MVT::i32 || VT == MVT::i64) &&
3073 "Caller guarantees VT is one of i32 or i64");
3086 assert((~AndImm & NonZeroBits) == 0 &&
3087 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3116 if (!BiggerPattern && !AndOp0.
hasOneUse())
3135 <<
"Found large Width in bit-field-positioning -- this indicates no "
3136 "proper combining / constant folding was performed\n");
3145 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3160 "Op.getNode() should be a SHL node to call this function");
3162 "Op.getNode() should shift ShlImm to call this function");
3169 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3193 EVT VT =
Op.getValueType();
3194 assert((VT == MVT::i32 || VT == MVT::i64) &&
3195 "Caller guarantees that type is i32 or i64");
3202 if (!BiggerPattern && !
Op.hasOneUse())
3211 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3219 assert(VT == MVT::i32 || VT == MVT::i64);
3230 EVT VT =
N->getValueType(0);
3231 if (VT != MVT::i32 && VT != MVT::i64)
3249 if (!
And.hasOneUse() ||
3259 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3266 if ((OrImm & NotKnownZero) != 0) {
3278 unsigned ImmS = Width - 1;
3284 bool IsBFI = LSB != 0;
3289 unsigned OrChunks = 0, BFIChunks = 0;
3290 for (
unsigned Shift = 0; Shift <
BitWidth; Shift += 16) {
3291 if (((OrImm >> Shift) & 0xFFFF) != 0)
3293 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3296 if (BFIChunks > OrChunks)
3302 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3310 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3319 if (!Dst.hasOneUse())
3322 EVT VT = Dst.getValueType();
3323 assert((VT == MVT::i32 || VT == MVT::i64) &&
3324 "Caller should guarantee that VT is one of i32 or i64");
3332 SDValue DstOp0 = Dst.getOperand(0);
3352 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3353 unsigned MaskWidth =
3356 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3362 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1,
DL, VT));
3363 ShiftedOperand =
SDValue(UBFMNode, 0);
3373 ShiftedOperand = Dst.getOperand(0);
3380 ShiftedOperand = Dst.getOperand(0);
3392 const bool BiggerPattern) {
3393 EVT VT =
N->getValueType(0);
3394 assert(
N->getOpcode() ==
ISD::OR &&
"Expect N to be an OR node");
3395 assert(((
N->getOperand(0) == OrOpd0 &&
N->getOperand(1) == OrOpd1) ||
3396 (
N->getOperand(1) == OrOpd0 &&
N->getOperand(0) == OrOpd1)) &&
3397 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3398 assert((VT == MVT::i32 || VT == MVT::i64) &&
3399 "Expect result type to be i32 or i64 since N is combinable to BFM");
3406 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3409 if (BiggerPattern) {
3423 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3432 assert((!BiggerPattern) &&
"BiggerPattern should be handled above");
3494 EVT VT =
N->getValueType(0);
3495 if (VT != MVT::i32 && VT != MVT::i64)
3503 unsigned NumberOfIgnoredLowBits = UsefulBits.
countr_zero();
3504 unsigned NumberOfIgnoredHighBits = UsefulBits.
countl_zero();
3524 for (
int I = 0;
I < 4; ++
I) {
3527 unsigned ImmR, ImmS;
3528 bool BiggerPattern =
I / 2;
3529 SDValue OrOpd0Val =
N->getOperand(
I % 2);
3531 SDValue OrOpd1Val =
N->getOperand((
I + 1) % 2);
3537 NumberOfIgnoredLowBits, BiggerPattern)) {
3540 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3541 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3546 Width = ImmS - ImmR + 1;
3557 Src, DstLSB, Width)) {
3565 assert((VT == MVT::i32 || VT == MVT::i64) &&
"unexpected OR operand");
3575 APInt BitsToBeInserted =
3578 if ((BitsToBeInserted & ~Known.
Zero) != 0)
3602 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3635 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3637 if (Src->hasOneUse() &&
3640 Src = Src->getOperand(0);
3650 unsigned ImmS = Width - 1;
3656 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3664bool AArch64DAGToDAGISel::tryBitfieldInsertOp(
SDNode *
N) {
3673 CurDAG->SelectNodeTo(
N, TargetOpcode::IMPLICIT_DEF,
N->getValueType(0));
3686bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(
SDNode *
N) {
3690 EVT VT =
N->getValueType(0);
3691 if (VT != MVT::i32 && VT != MVT::i64)
3697 Op0, DstLSB, Width))
3703 unsigned ImmS = Width - 1;
3706 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR,
DL, VT),
3707 CurDAG->getTargetConstant(ImmS,
DL, VT)};
3708 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3709 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3715bool AArch64DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
3716 EVT VT =
N->getValueType(0);
3719 switch (
N->getOpcode()) {
3721 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
3724 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
3727 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
3730 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
3738 if (VT == MVT::i32) {
3741 }
else if (VT == MVT::i64) {
3747 SDValue ShiftAmt =
N->getOperand(1);
3767 (Add0Imm %
Size == 0)) {
3773 if (SubVT == MVT::i32) {
3774 NegOpc = AArch64::SUBWrr;
3775 ZeroReg = AArch64::WZR;
3777 assert(SubVT == MVT::i64);
3778 NegOpc = AArch64::SUBXrr;
3779 ZeroReg = AArch64::XZR;
3782 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3784 CurDAG->getMachineNode(NegOpc,
DL, SubVT, Zero, Add1);
3785 NewShiftAmt =
SDValue(Neg, 0);
3793 if (SubVT == MVT::i32) {
3794 NotOpc = AArch64::ORNWrr;
3795 ZeroReg = AArch64::WZR;
3797 assert(SubVT == MVT::i64);
3798 NotOpc = AArch64::ORNXrr;
3799 ZeroReg = AArch64::XZR;
3802 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3804 CurDAG->getMachineNode(NotOpc,
DL, SubVT, Zero, Add1);
3805 NewShiftAmt =
SDValue(Not, 0);
3826 else if (VT == MVT::i64 && NewShiftAmt->
getValueType(0) == MVT::i32) {
3827 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32,
DL, MVT::i32);
3829 AArch64::SUBREG_TO_REG,
DL, VT,
3830 CurDAG->getTargetConstant(0,
DL, MVT::i64), NewShiftAmt,
SubReg);
3831 NewShiftAmt =
SDValue(Ext, 0);
3834 SDValue Ops[] = {
N->getOperand(0), NewShiftAmt};
3835 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3842 bool isReciprocal) {
3845 FVal = CN->getValueAPF();
3846 else if (
LoadSDNode *LN = dyn_cast<LoadSDNode>(
N)) {
3849 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
3853 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
3854 FVal = cast<ConstantFP>(CN->
getConstVal())->getValueAPF();
3877 if (!IsExact || !IntVal.isPowerOf2())
3879 unsigned FBits = IntVal.logBase2();
3883 if (FBits == 0 || FBits > RegWidth)
return false;
3889bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
3890 unsigned RegWidth) {
3895bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(
SDValue N,
3897 unsigned RegWidth) {
3907 RegString.
split(Fields,
':');
3909 if (Fields.
size() == 1)
3913 &&
"Invalid number of fields in read register string");
3916 bool AllIntFields =
true;
3920 AllIntFields &= !
Field.getAsInteger(10, IntField);
3925 "Unexpected non-integer value in special register string.");
3930 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
3931 (Ops[3] << 3) | (Ops[4]);
3938bool AArch64DAGToDAGISel::tryReadRegister(
SDNode *
N) {
3939 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3940 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3945 unsigned Opcode64Bit = AArch64::MRS;
3950 const auto *TheReg =
3952 if (TheReg && TheReg->Readable &&
3953 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3954 Imm = TheReg->Encoding;
3960 if (!ReadIs128Bit && RegString->getString() ==
"pc") {
3961 Opcode64Bit = AArch64::ADR;
3969 SDValue InChain =
N->getOperand(0);
3970 SDValue SysRegImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
3971 if (!ReadIs128Bit) {
3972 CurDAG->SelectNodeTo(
N, Opcode64Bit, MVT::i64, MVT::Other ,
3973 {SysRegImm, InChain});
3977 {MVT::Untyped , MVT::Other },
3978 {SysRegImm, InChain});
3982 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64,
DL, MVT::i64,
3984 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64,
DL, MVT::i64,
3990 ReplaceUses(
SDValue(
N, 2), OutChain);
3999bool AArch64DAGToDAGISel::tryWriteRegister(
SDNode *
N) {
4000 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
4001 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
4006 if (!WriteIs128Bit) {
4012 auto trySelectPState = [&](
auto PMapper,
unsigned State) {
4014 assert(isa<ConstantSDNode>(
N->getOperand(2)) &&
4015 "Expected a constant integer expression.");
4016 unsigned Reg = PMapper->Encoding;
4017 uint64_t Immed =
N->getConstantOperandVal(2);
4018 CurDAG->SelectNodeTo(
4019 N, State, MVT::Other, CurDAG->getTargetConstant(Reg,
DL, MVT::i32),
4020 CurDAG->getTargetConstant(Immed,
DL, MVT::i16),
N->getOperand(0));
4026 if (trySelectPState(
4027 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
4028 AArch64::MSRpstateImm4))
4030 if (trySelectPState(
4031 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
4032 AArch64::MSRpstateImm1))
4042 if (TheReg && TheReg->Writeable &&
4043 TheReg->haveFeatures(Subtarget->getFeatureBits()))
4044 Imm = TheReg->Encoding;
4052 SDValue InChain =
N->getOperand(0);
4053 if (!WriteIs128Bit) {
4054 CurDAG->SelectNodeTo(
N, AArch64::MSR, MVT::Other,
4055 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
4056 N->getOperand(2), InChain);
4060 SDNode *Pair = CurDAG->getMachineNode(
4061 TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped ,
4062 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(),
DL,
4065 CurDAG->getTargetConstant(AArch64::sube64,
DL, MVT::i32),
4067 CurDAG->getTargetConstant(AArch64::subo64,
DL, MVT::i32)});
4069 CurDAG->SelectNodeTo(
N, AArch64::MSRR, MVT::Other,
4070 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
4078bool AArch64DAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
4080 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
4083 if (Subtarget->hasLSE())
return false;
4085 if (MemTy == MVT::i8)
4086 Opcode = AArch64::CMP_SWAP_8;
4087 else if (MemTy == MVT::i16)
4088 Opcode = AArch64::CMP_SWAP_16;
4089 else if (MemTy == MVT::i32)
4090 Opcode = AArch64::CMP_SWAP_32;
4091 else if (MemTy == MVT::i64)
4092 Opcode = AArch64::CMP_SWAP_64;
4096 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4097 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
4099 SDNode *CmpSwap = CurDAG->getMachineNode(
4101 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
4104 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
4108 CurDAG->RemoveDeadNode(
N);
4115 if (!isa<ConstantSDNode>(
N))
4127 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4128 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4135 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4136 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4140 if (Val <= 65280 && Val % 256 == 0) {
4141 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4142 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4153bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(
SDValue N,
MVT VT,
4156 if (!isa<ConstantSDNode>(
N))
4160 int64_t Val = cast<ConstantSDNode>(
N)
4177 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4178 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4185 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4186 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4190 if (Val <= 65280 && Val % 256 == 0) {
4191 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4192 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4205 if (!isa<ConstantSDNode>(
N))
4209 int64_t Val = cast<ConstantSDNode>(
N)
4217 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4218 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4224 if (Val >= -128 && Val <= 127) {
4225 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4226 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4230 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) {
4231 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4232 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF,
DL, MVT::i32);
4243bool AArch64DAGToDAGISel::SelectSVESignedArithImm(
SDValue N,
SDValue &Imm) {
4244 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4245 int64_t ImmVal = CNode->getSExtValue();
4247 if (ImmVal >= -128 && ImmVal < 128) {
4248 Imm = CurDAG->getTargetConstant(ImmVal,
DL, MVT::i32);
4256 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4257 uint64_t ImmVal = CNode->getZExtValue();
4267 ImmVal &= 0xFFFFFFFF;
4276 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4285 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4286 uint64_t ImmVal = CNode->getZExtValue();
4296 ImmVal |= ImmVal << 8;
4297 ImmVal |= ImmVal << 16;
4298 ImmVal |= ImmVal << 32;
4302 ImmVal |= ImmVal << 16;
4303 ImmVal |= ImmVal << 32;
4306 ImmVal &= 0xFFFFFFFF;
4307 ImmVal |= ImmVal << 32;
4317 Imm = CurDAG->getTargetConstant(encoding,
DL, MVT::i64);
4332 if (
auto *CN = dyn_cast<ConstantSDNode>(
N)) {
4333 uint64_t ImmVal = CN->getZExtValue();
4340 if (ImmVal >
High) {
4341 if (!AllowSaturation)
4346 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4353bool AArch64DAGToDAGISel::trySelectStackSlotTagP(
SDNode *
N) {
4357 if (!(isa<FrameIndexSDNode>(
N->getOperand(1)))) {
4369 int FI = cast<FrameIndexSDNode>(
N->getOperand(1))->getIndex();
4370 SDValue FiOp = CurDAG->getTargetFrameIndex(
4372 int TagOffset =
N->getConstantOperandVal(3);
4374 SDNode *Out = CurDAG->getMachineNode(
4375 AArch64::TAGPstack,
DL, MVT::i64,
4376 {FiOp, CurDAG->getTargetConstant(0,
DL, MVT::i64),
N->getOperand(2),
4377 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4378 ReplaceNode(
N, Out);
4382void AArch64DAGToDAGISel::SelectTagP(
SDNode *
N) {
4383 assert(isa<ConstantSDNode>(
N->getOperand(3)) &&
4384 "llvm.aarch64.tagp third argument must be an immediate");
4385 if (trySelectStackSlotTagP(
N))
4392 int TagOffset =
N->getConstantOperandVal(3);
4393 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP,
DL, MVT::i64,
4394 {
N->getOperand(1),
N->getOperand(2)});
4395 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr,
DL, MVT::i64,
4396 {
SDValue(N1, 0),
N->getOperand(2)});
4397 SDNode *N3 = CurDAG->getMachineNode(
4398 AArch64::ADDG,
DL, MVT::i64,
4399 {
SDValue(N2, 0), CurDAG->getTargetConstant(0,
DL, MVT::i64),
4400 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4404bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(
SDNode *
N) {
4408 if (
N->getConstantOperandVal(2) != 0)
4410 if (!
N->getOperand(0).isUndef())
4414 EVT VT =
N->getValueType(0);
4415 EVT InVT =
N->getOperand(1).getValueType();
4426 "Expected to insert into a packed scalable vector!");
4429 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4430 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4431 N->getOperand(1), RC));
4435bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(
SDNode *
N) {
4439 if (
N->getConstantOperandVal(1) != 0)
4443 EVT VT =
N->getValueType(0);
4444 EVT InVT =
N->getOperand(0).getValueType();
4455 "Expected to extract from a packed scalable vector!");
4458 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4459 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4460 N->getOperand(0), RC));
4464bool AArch64DAGToDAGISel::trySelectXAR(
SDNode *
N) {
4469 EVT VT =
N->getValueType(0);
4481 (Subtarget->hasSVE2() ||
4482 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4491 if (!TLI->isAllActivePredicate(*CurDAG, N0.
getOperand(0)) ||
4492 !TLI->isAllActivePredicate(*CurDAG, N1.
getOperand(0)))
4499 APInt ShlAmt, ShrAmt;
4512 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
4513 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
4514 AArch64::XAR_ZZZI_D})) {
4515 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
4521 if (!Subtarget->hasSHA3())
4543 if (ShAmt + HsAmt != 64)
4547 CurDAG->SelectNodeTo(
N, AArch64::XAR, N0.
getValueType(), Ops);
4552void AArch64DAGToDAGISel::Select(
SDNode *
Node) {
4554 if (
Node->isMachineOpcode()) {
4556 Node->setNodeId(-1);
4561 EVT VT =
Node->getValueType(0);
4563 switch (
Node->getOpcode()) {
4568 if (SelectCMP_SWAP(
Node))
4574 if (tryReadRegister(
Node))
4580 if (tryWriteRegister(
Node))
4587 if (tryIndexedLoad(
Node))
4596 if (tryBitfieldExtractOp(
Node))
4598 if (tryBitfieldInsertInZeroOp(
Node))
4603 if (tryShiftAmountMod(
Node))
4608 if (tryBitfieldExtractOpFromSExt(
Node))
4613 if (tryBitfieldInsertOp(
Node))
4615 if (trySelectXAR(
Node))
4620 if (trySelectCastScalableToFixedLengthVector(
Node))
4626 if (trySelectCastFixedLengthToScalableVector(
Node))
4635 if (ConstNode->
isZero()) {
4636 if (VT == MVT::i32) {
4638 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::WZR, MVT::i32);
4639 ReplaceNode(
Node,
New.getNode());
4641 }
else if (VT == MVT::i64) {
4643 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::XZR, MVT::i64);
4644 ReplaceNode(
Node,
New.getNode());
4653 int FI = cast<FrameIndexSDNode>(
Node)->getIndex();
4656 SDValue TFI = CurDAG->getTargetFrameIndex(
4659 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0,
DL, MVT::i32),
4660 CurDAG->getTargetConstant(Shifter,
DL, MVT::i32) };
4661 CurDAG->SelectNodeTo(
Node, AArch64::ADDXri, MVT::i64, Ops);
4665 unsigned IntNo =
Node->getConstantOperandVal(1);
4669 case Intrinsic::aarch64_gcsss: {
4673 SDValue Zero = CurDAG->getCopyFromReg(Chain,
DL, AArch64::XZR, MVT::i64);
4675 CurDAG->getMachineNode(AArch64::GCSSS1,
DL, MVT::Other, Val, Chain);
4676 SDNode *SS2 = CurDAG->getMachineNode(AArch64::GCSSS2,
DL, MVT::i64,
4677 MVT::Other, Zero,
SDValue(SS1, 0));
4678 ReplaceNode(
Node, SS2);
4681 case Intrinsic::aarch64_ldaxp:
4682 case Intrinsic::aarch64_ldxp: {
4684 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
4689 SDNode *Ld = CurDAG->getMachineNode(
Op,
DL, MVT::i64, MVT::i64,
4690 MVT::Other, MemAddr, Chain);
4694 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4695 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4696 ReplaceNode(
Node, Ld);
4699 case Intrinsic::aarch64_stlxp:
4700 case Intrinsic::aarch64_stxp: {
4702 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
4710 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
4712 SDNode *St = CurDAG->getMachineNode(
Op,
DL, MVT::i32, MVT::Other, Ops);
4715 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4716 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4718 ReplaceNode(
Node, St);
4721 case Intrinsic::aarch64_neon_ld1x2:
4722 if (VT == MVT::v8i8) {
4723 SelectLoad(
Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
4725 }
else if (VT == MVT::v16i8) {
4726 SelectLoad(
Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
4728 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4729 SelectLoad(
Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
4731 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4732 SelectLoad(
Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
4734 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4735 SelectLoad(
Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
4737 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4738 SelectLoad(
Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
4740 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4741 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4743 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4744 SelectLoad(
Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
4748 case Intrinsic::aarch64_neon_ld1x3:
4749 if (VT == MVT::v8i8) {
4750 SelectLoad(
Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
4752 }
else if (VT == MVT::v16i8) {
4753 SelectLoad(
Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
4755 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4756 SelectLoad(
Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
4758 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4759 SelectLoad(
Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
4761 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4762 SelectLoad(
Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
4764 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4765 SelectLoad(
Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
4767 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4768 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4770 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4771 SelectLoad(
Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
4775 case Intrinsic::aarch64_neon_ld1x4:
4776 if (VT == MVT::v8i8) {
4777 SelectLoad(
Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
4779 }
else if (VT == MVT::v16i8) {
4780 SelectLoad(
Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
4782 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4783 SelectLoad(
Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
4785 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4786 SelectLoad(
Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
4788 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4789 SelectLoad(
Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
4791 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4792 SelectLoad(
Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
4794 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4795 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4797 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4798 SelectLoad(
Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
4802 case Intrinsic::aarch64_neon_ld2:
4803 if (VT == MVT::v8i8) {
4804 SelectLoad(
Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
4806 }
else if (VT == MVT::v16i8) {
4807 SelectLoad(
Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
4809 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4810 SelectLoad(
Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
4812 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4813 SelectLoad(
Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
4815 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4816 SelectLoad(
Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
4818 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4819 SelectLoad(
Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
4821 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4822 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4824 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4825 SelectLoad(
Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
4829 case Intrinsic::aarch64_neon_ld3:
4830 if (VT == MVT::v8i8) {
4831 SelectLoad(
Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
4833 }
else if (VT == MVT::v16i8) {
4834 SelectLoad(
Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
4836 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4837 SelectLoad(
Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
4839 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4840 SelectLoad(
Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
4842 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4843 SelectLoad(
Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
4845 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4846 SelectLoad(
Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
4848 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4849 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4851 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4852 SelectLoad(
Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
4856 case Intrinsic::aarch64_neon_ld4:
4857 if (VT == MVT::v8i8) {
4858 SelectLoad(
Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
4860 }
else if (VT == MVT::v16i8) {
4861 SelectLoad(
Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
4863 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4864 SelectLoad(
Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
4866 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4867 SelectLoad(
Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
4869 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4870 SelectLoad(
Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
4872 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4873 SelectLoad(
Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
4875 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4876 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4878 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4879 SelectLoad(
Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
4883 case Intrinsic::aarch64_neon_ld2r:
4884 if (VT == MVT::v8i8) {
4885 SelectLoad(
Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
4887 }
else if (VT == MVT::v16i8) {
4888 SelectLoad(
Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
4890 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4891 SelectLoad(
Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
4893 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4894 SelectLoad(
Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
4896 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4897 SelectLoad(
Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
4899 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4900 SelectLoad(
Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
4902 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4903 SelectLoad(
Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
4905 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4906 SelectLoad(
Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
4910 case Intrinsic::aarch64_neon_ld3r:
4911 if (VT == MVT::v8i8) {
4912 SelectLoad(
Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
4914 }
else if (VT == MVT::v16i8) {
4915 SelectLoad(
Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
4917 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4918 SelectLoad(
Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
4920 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4921 SelectLoad(
Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
4923 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4924 SelectLoad(
Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
4926 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4927 SelectLoad(
Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
4929 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4930 SelectLoad(
Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
4932 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4933 SelectLoad(
Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
4937 case Intrinsic::aarch64_neon_ld4r:
4938 if (VT == MVT::v8i8) {
4939 SelectLoad(
Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
4941 }
else if (VT == MVT::v16i8) {
4942 SelectLoad(
Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
4944 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4945 SelectLoad(
Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
4947 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4948 SelectLoad(
Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
4950 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4951 SelectLoad(
Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
4953 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4954 SelectLoad(
Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
4956 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4957 SelectLoad(
Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
4959 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4960 SelectLoad(
Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
4964 case Intrinsic::aarch64_neon_ld2lane:
4965 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4966 SelectLoadLane(
Node, 2, AArch64::LD2i8);
4968 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4969 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4970 SelectLoadLane(
Node, 2, AArch64::LD2i16);
4972 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4974 SelectLoadLane(
Node, 2, AArch64::LD2i32);
4976 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4978 SelectLoadLane(
Node, 2, AArch64::LD2i64);
4982 case Intrinsic::aarch64_neon_ld3lane:
4983 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4984 SelectLoadLane(
Node, 3, AArch64::LD3i8);
4986 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4987 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4988 SelectLoadLane(
Node, 3, AArch64::LD3i16);
4990 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4992 SelectLoadLane(
Node, 3, AArch64::LD3i32);
4994 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4996 SelectLoadLane(
Node, 3, AArch64::LD3i64);
5000 case Intrinsic::aarch64_neon_ld4lane:
5001 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5002 SelectLoadLane(
Node, 4, AArch64::LD4i8);
5004 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5005 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5006 SelectLoadLane(
Node, 4, AArch64::LD4i16);
5008 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5010 SelectLoadLane(
Node, 4, AArch64::LD4i32);
5012 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5014 SelectLoadLane(
Node, 4, AArch64::LD4i64);
5018 case Intrinsic::aarch64_ld64b:
5019 SelectLoad(
Node, 8, AArch64::LD64B, AArch64::x8sub_0);
5021 case Intrinsic::aarch64_sve_ld2q_sret: {
5022 SelectPredicatedLoad(
Node, 2, 4, AArch64::LD2Q_IMM, AArch64::LD2Q,
true);
5025 case Intrinsic::aarch64_sve_ld3q_sret: {
5026 SelectPredicatedLoad(
Node, 3, 4, AArch64::LD3Q_IMM, AArch64::LD3Q,
true);
5029 case Intrinsic::aarch64_sve_ld4q_sret: {
5030 SelectPredicatedLoad(
Node, 4, 4, AArch64::LD4Q_IMM, AArch64::LD4Q,
true);
5033 case Intrinsic::aarch64_sve_ld2_sret: {
5034 if (VT == MVT::nxv16i8) {
5035 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B,
5038 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5039 VT == MVT::nxv8bf16) {
5040 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H,
5043 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5044 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W,
5047 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5048 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D,
5054 case Intrinsic::aarch64_sve_ld1_pn_x2: {
5055 if (VT == MVT::nxv16i8) {
5056 if (Subtarget->hasSME2())
5057 SelectContiguousMultiVectorLoad(
5058 Node, 2, 0, AArch64::LD1B_2Z_IMM_PSEUDO, AArch64::LD1B_2Z_PSEUDO);
5059 else if (Subtarget->hasSVE2p1())
5060 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LD1B_2Z_IMM,
5065 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5066 VT == MVT::nxv8bf16) {
5067 if (Subtarget->hasSME2())
5068 SelectContiguousMultiVectorLoad(
5069 Node, 2, 1, AArch64::LD1H_2Z_IMM_PSEUDO, AArch64::LD1H_2Z_PSEUDO);
5070 else if (Subtarget->hasSVE2p1())
5071 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LD1H_2Z_IMM,
5076 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5077 if (Subtarget->hasSME2())
5078 SelectContiguousMultiVectorLoad(
5079 Node, 2, 2, AArch64::LD1W_2Z_IMM_PSEUDO, AArch64::LD1W_2Z_PSEUDO);
5080 else if (Subtarget->hasSVE2p1())
5081 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LD1W_2Z_IMM,
5086 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5087 if (Subtarget->hasSME2())
5088 SelectContiguousMultiVectorLoad(
5089 Node, 2, 3, AArch64::LD1D_2Z_IMM_PSEUDO, AArch64::LD1D_2Z_PSEUDO);
5090 else if (Subtarget->hasSVE2p1())
5091 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LD1D_2Z_IMM,
5099 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5100 if (VT == MVT::nxv16i8) {
5101 if (Subtarget->hasSME2())
5102 SelectContiguousMultiVectorLoad(
5103 Node, 4, 0, AArch64::LD1B_4Z_IMM_PSEUDO, AArch64::LD1B_4Z_PSEUDO);
5104 else if (Subtarget->hasSVE2p1())
5105 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LD1B_4Z_IMM,
5110 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5111 VT == MVT::nxv8bf16) {
5112 if (Subtarget->hasSME2())
5113 SelectContiguousMultiVectorLoad(
5114 Node, 4, 1, AArch64::LD1H_4Z_IMM_PSEUDO, AArch64::LD1H_4Z_PSEUDO);
5115 else if (Subtarget->hasSVE2p1())
5116 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LD1H_4Z_IMM,
5121 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5122 if (Subtarget->hasSME2())
5123 SelectContiguousMultiVectorLoad(
5124 Node, 4, 2, AArch64::LD1W_4Z_IMM_PSEUDO, AArch64::LD1W_4Z_PSEUDO);
5125 else if (Subtarget->hasSVE2p1())
5126 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LD1W_4Z_IMM,
5131 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5132 if (Subtarget->hasSME2())
5133 SelectContiguousMultiVectorLoad(
5134 Node, 4, 3, AArch64::LD1D_4Z_IMM_PSEUDO, AArch64::LD1D_4Z_PSEUDO);
5135 else if (Subtarget->hasSVE2p1())
5136 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LD1D_4Z_IMM,
5144 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5145 if (VT == MVT::nxv16i8) {
5146 if (Subtarget->hasSME2())
5147 SelectContiguousMultiVectorLoad(
Node, 2, 0,
5148 AArch64::LDNT1B_2Z_IMM_PSEUDO,
5149 AArch64::LDNT1B_2Z_PSEUDO);
5150 else if (Subtarget->hasSVE2p1())
5151 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LDNT1B_2Z_IMM,
5152 AArch64::LDNT1B_2Z);
5156 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5157 VT == MVT::nxv8bf16) {
5158 if (Subtarget->hasSME2())
5159 SelectContiguousMultiVectorLoad(
Node, 2, 1,
5160 AArch64::LDNT1H_2Z_IMM_PSEUDO,
5161 AArch64::LDNT1H_2Z_PSEUDO);
5162 else if (Subtarget->hasSVE2p1())
5163 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LDNT1H_2Z_IMM,
5164 AArch64::LDNT1H_2Z);
5168 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5169 if (Subtarget->hasSME2())
5170 SelectContiguousMultiVectorLoad(
Node, 2, 2,
5171 AArch64::LDNT1W_2Z_IMM_PSEUDO,
5172 AArch64::LDNT1W_2Z_PSEUDO);
5173 else if (Subtarget->hasSVE2p1())
5174 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LDNT1W_2Z_IMM,
5175 AArch64::LDNT1W_2Z);
5179 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5180 if (Subtarget->hasSME2())
5181 SelectContiguousMultiVectorLoad(
Node, 2, 3,
5182 AArch64::LDNT1D_2Z_IMM_PSEUDO,
5183 AArch64::LDNT1D_2Z_PSEUDO);
5184 else if (Subtarget->hasSVE2p1())
5185 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LDNT1D_2Z_IMM,
5186 AArch64::LDNT1D_2Z);
5193 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5194 if (VT == MVT::nxv16i8) {
5195 if (Subtarget->hasSME2())
5196 SelectContiguousMultiVectorLoad(
Node, 4, 0,
5197 AArch64::LDNT1B_4Z_IMM_PSEUDO,
5198 AArch64::LDNT1B_4Z_PSEUDO);
5199 else if (Subtarget->hasSVE2p1())
5200 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LDNT1B_4Z_IMM,
5201 AArch64::LDNT1B_4Z);
5205 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5206 VT == MVT::nxv8bf16) {
5207 if (Subtarget->hasSME2())
5208 SelectContiguousMultiVectorLoad(
Node, 4, 1,
5209 AArch64::LDNT1H_4Z_IMM_PSEUDO,
5210 AArch64::LDNT1H_4Z_PSEUDO);
5211 else if (Subtarget->hasSVE2p1())
5212 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LDNT1H_4Z_IMM,
5213 AArch64::LDNT1H_4Z);
5217 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5218 if (Subtarget->hasSME2())
5219 SelectContiguousMultiVectorLoad(
Node, 4, 2,
5220 AArch64::LDNT1W_4Z_IMM_PSEUDO,
5221 AArch64::LDNT1W_4Z_PSEUDO);
5222 else if (Subtarget->hasSVE2p1())
5223 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LDNT1W_4Z_IMM,
5224 AArch64::LDNT1W_4Z);
5228 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5229 if (Subtarget->hasSME2())
5230 SelectContiguousMultiVectorLoad(
Node, 4, 3,
5231 AArch64::LDNT1D_4Z_IMM_PSEUDO,
5232 AArch64::LDNT1D_4Z_PSEUDO);
5233 else if (Subtarget->hasSVE2p1())
5234 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LDNT1D_4Z_IMM,
5235 AArch64::LDNT1D_4Z);
5242 case Intrinsic::aarch64_sve_ld3_sret: {
5243 if (VT == MVT::nxv16i8) {
5244 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B,
5247 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5248 VT == MVT::nxv8bf16) {
5249 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H,
5252 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5253 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W,
5256 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5257 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D,
5263 case Intrinsic::aarch64_sve_ld4_sret: {
5264 if (VT == MVT::nxv16i8) {
5265 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B,
5268 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5269 VT == MVT::nxv8bf16) {
5270 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H,
5273 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5274 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W,
5277 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5278 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D,
5284 case Intrinsic::aarch64_sme_read_hor_vg2: {
5285 if (VT == MVT::nxv16i8) {
5286 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5287 AArch64::MOVA_2ZMXI_H_B);
5289 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5290 VT == MVT::nxv8bf16) {
5291 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5292 AArch64::MOVA_2ZMXI_H_H);
5294 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5295 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5296 AArch64::MOVA_2ZMXI_H_S);
5298 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5299 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5300 AArch64::MOVA_2ZMXI_H_D);
5305 case Intrinsic::aarch64_sme_read_ver_vg2: {
5306 if (VT == MVT::nxv16i8) {
5307 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5308 AArch64::MOVA_2ZMXI_V_B);
5310 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5311 VT == MVT::nxv8bf16) {
5312 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5313 AArch64::MOVA_2ZMXI_V_H);
5315 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5316 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5317 AArch64::MOVA_2ZMXI_V_S);
5319 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5320 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5321 AArch64::MOVA_2ZMXI_V_D);
5326 case Intrinsic::aarch64_sme_read_hor_vg4: {
5327 if (VT == MVT::nxv16i8) {
5328 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5329 AArch64::MOVA_4ZMXI_H_B);
5331 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5332 VT == MVT::nxv8bf16) {
5333 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5334 AArch64::MOVA_4ZMXI_H_H);
5336 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5337 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAS0,
5338 AArch64::MOVA_4ZMXI_H_S);
5340 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5341 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAD0,
5342 AArch64::MOVA_4ZMXI_H_D);
5347 case Intrinsic::aarch64_sme_read_ver_vg4: {
5348 if (VT == MVT::nxv16i8) {
5349 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5350 AArch64::MOVA_4ZMXI_V_B);
5352 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5353 VT == MVT::nxv8bf16) {
5354 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5355 AArch64::MOVA_4ZMXI_V_H);
5357 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5358 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAS0,
5359 AArch64::MOVA_4ZMXI_V_S);
5361 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5362 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAD0,
5363 AArch64::MOVA_4ZMXI_V_D);
5368 case Intrinsic::aarch64_sme_read_vg1x2: {
5369 SelectMultiVectorMove<7, 1>(
Node, 2, AArch64::ZA,
5370 AArch64::MOVA_VG2_2ZMXI);
5373 case Intrinsic::aarch64_sme_read_vg1x4: {
5374 SelectMultiVectorMove<7, 1>(
Node, 4, AArch64::ZA,
5375 AArch64::MOVA_VG4_4ZMXI);
5378 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5379 if (VT == MVT::nxv16i8) {
5380 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_B_PSEUDO, 14, 2);
5382 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5383 VT == MVT::nxv8bf16) {
5384 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_H_PSEUDO, 6, 2);
5386 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5387 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_S_PSEUDO, 2, 2);
5389 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5390 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_H_D_PSEUDO, 0, 2);
5395 case Intrinsic::aarch64_sme_readz_vert_x2: {
5396 if (VT == MVT::nxv16i8) {
5397 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_B_PSEUDO, 14, 2);
5399 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5400 VT == MVT::nxv8bf16) {
5401 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_H_PSEUDO, 6, 2);
5403 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5404 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_S_PSEUDO, 2, 2);
5406 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5407 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_2ZMI_V_D_PSEUDO, 0, 2);
5412 case Intrinsic::aarch64_sme_readz_horiz_x4: {
5413 if (VT == MVT::nxv16i8) {
5414 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_B_PSEUDO, 12, 4);
5416 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5417 VT == MVT::nxv8bf16) {
5418 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_H_PSEUDO, 4, 4);
5420 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5421 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_S_PSEUDO, 0, 4);
5423 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5424 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_H_D_PSEUDO, 0, 4);
5429 case Intrinsic::aarch64_sme_readz_vert_x4: {
5430 if (VT == MVT::nxv16i8) {
5431 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_B_PSEUDO, 12, 4);
5433 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5434 VT == MVT::nxv8bf16) {
5435 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_H_PSEUDO, 4, 4);
5437 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5438 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_S_PSEUDO, 0, 4);
5440 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5441 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_4ZMI_V_D_PSEUDO, 0, 4);
5446 case Intrinsic::aarch64_sme_readz_x2: {
5447 SelectMultiVectorMoveZ(
Node, 2, AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, 7, 1,
5451 case Intrinsic::aarch64_sme_readz_x4: {
5452 SelectMultiVectorMoveZ(
Node, 4, AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, 7, 1,
5456 case Intrinsic::swift_async_context_addr: {
5459 SDValue CopyFP = CurDAG->getCopyFromReg(Chain,
DL, AArch64::FP, MVT::i64);
5461 CurDAG->getMachineNode(AArch64::SUBXri,
DL, MVT::i64, CopyFP,
5462 CurDAG->getTargetConstant(8,
DL, MVT::i32),
5463 CurDAG->getTargetConstant(0,
DL, MVT::i32)),
5467 CurDAG->RemoveDeadNode(
Node);
5469 auto &MF = CurDAG->getMachineFunction();
5470 MF.getFrameInfo().setFrameAddressIsTaken(
true);
5474 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
5475 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5476 Node->getValueType(0),
5477 {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
5478 AArch64::LUTI2_4ZTZI_S}))
5480 SelectMultiVectorLuti(
Node, 4, Opc, 3);
5483 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
5484 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5485 Node->getValueType(0),
5486 {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
5488 SelectMultiVectorLuti(
Node, 4, Opc, 1);
5491 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
5492 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5493 Node->getValueType(0),
5494 {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
5495 AArch64::LUTI2_2ZTZI_S}))
5497 SelectMultiVectorLuti(
Node, 2, Opc, 7);
5500 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
5501 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5502 Node->getValueType(0),
5503 {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
5504 AArch64::LUTI4_2ZTZI_S}))
5506 SelectMultiVectorLuti(
Node, 2, Opc, 3);
5512 unsigned IntNo =
Node->getConstantOperandVal(0);
5516 case Intrinsic::aarch64_tagp:
5520 case Intrinsic::ptrauth_auth:
5521 SelectPtrauthAuth(
Node);
5524 case Intrinsic::ptrauth_resign:
5525 SelectPtrauthResign(
Node);
5528 case Intrinsic::aarch64_neon_tbl2:
5529 SelectTable(
Node, 2,
5530 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
5533 case Intrinsic::aarch64_neon_tbl3:
5534 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
5535 : AArch64::TBLv16i8Three,
5538 case Intrinsic::aarch64_neon_tbl4:
5539 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
5540 : AArch64::TBLv16i8Four,
5543 case Intrinsic::aarch64_neon_tbx2:
5544 SelectTable(
Node, 2,
5545 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
5548 case Intrinsic::aarch64_neon_tbx3:
5549 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
5550 : AArch64::TBXv16i8Three,
5553 case Intrinsic::aarch64_neon_tbx4:
5554 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
5555 : AArch64::TBXv16i8Four,
5558 case Intrinsic::aarch64_sve_srshl_single_x2:
5559 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5560 Node->getValueType(0),
5561 {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
5562 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
5563 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5565 case Intrinsic::aarch64_sve_srshl_single_x4:
5566 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5567 Node->getValueType(0),
5568 {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
5569 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
5570 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5572 case Intrinsic::aarch64_sve_urshl_single_x2:
5573 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5574 Node->getValueType(0),
5575 {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
5576 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
5577 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5579 case Intrinsic::aarch64_sve_urshl_single_x4:
5580 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5581 Node->getValueType(0),
5582 {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
5583 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
5584 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5586 case Intrinsic::aarch64_sve_srshl_x2:
5587 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5588 Node->getValueType(0),
5589 {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
5590 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
5591 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5593 case Intrinsic::aarch64_sve_srshl_x4:
5594 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5595 Node->getValueType(0),
5596 {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
5597 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
5598 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5600 case Intrinsic::aarch64_sve_urshl_x2:
5601 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5602 Node->getValueType(0),
5603 {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
5604 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
5605 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5607 case Intrinsic::aarch64_sve_urshl_x4:
5608 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5609 Node->getValueType(0),
5610 {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
5611 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
5612 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5614 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
5615 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5616 Node->getValueType(0),
5617 {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
5618 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
5619 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5621 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
5622 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5623 Node->getValueType(0),
5624 {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
5625 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
5626 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5628 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
5629 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5630 Node->getValueType(0),
5631 {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
5632 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
5633 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5635 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
5636 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5637 Node->getValueType(0),
5638 {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
5639 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
5640 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5642 case Intrinsic::aarch64_sve_whilege_x2:
5643 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5644 Node->getValueType(0),
5645 {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
5646 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
5647 SelectWhilePair(
Node,
Op);
5649 case Intrinsic::aarch64_sve_whilegt_x2:
5650 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5651 Node->getValueType(0),
5652 {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
5653 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
5654 SelectWhilePair(
Node,
Op);
5656 case Intrinsic::aarch64_sve_whilehi_x2:
5657 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5658 Node->getValueType(0),
5659 {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
5660 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
5661 SelectWhilePair(
Node,
Op);
5663 case Intrinsic::aarch64_sve_whilehs_x2:
5664 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5665 Node->getValueType(0),
5666 {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
5667 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
5668 SelectWhilePair(
Node,
Op);
5670 case Intrinsic::aarch64_sve_whilele_x2:
5671 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5672 Node->getValueType(0),
5673 {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
5674 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
5675 SelectWhilePair(
Node,
Op);
5677 case Intrinsic::aarch64_sve_whilelo_x2:
5678 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5679 Node->getValueType(0),
5680 {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
5681 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
5682 SelectWhilePair(
Node,
Op);
5684 case Intrinsic::aarch64_sve_whilels_x2:
5685 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5686 Node->getValueType(0),
5687 {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
5688 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
5689 SelectWhilePair(
Node,
Op);
5691 case Intrinsic::aarch64_sve_whilelt_x2:
5692 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5693 Node->getValueType(0),
5694 {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
5695 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
5696 SelectWhilePair(
Node,
Op);
5698 case Intrinsic::aarch64_sve_smax_single_x2:
5699 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5700 Node->getValueType(0),
5701 {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
5702 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
5703 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5705 case Intrinsic::aarch64_sve_umax_single_x2:
5706 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5707 Node->getValueType(0),
5708 {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
5709 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
5710 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5712 case Intrinsic::aarch64_sve_fmax_single_x2:
5713 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5714 Node->getValueType(0),
5715 {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
5716 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
5717 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5719 case Intrinsic::aarch64_sve_smax_single_x4:
5720 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5721 Node->getValueType(0),
5722 {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
5723 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
5724 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5726 case Intrinsic::aarch64_sve_umax_single_x4:
5727 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5728 Node->getValueType(0),
5729 {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
5730 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
5731 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5733 case Intrinsic::aarch64_sve_fmax_single_x4:
5734 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5735 Node->getValueType(0),
5736 {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
5737 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
5738 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5740 case Intrinsic::aarch64_sve_smin_single_x2:
5741 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5742 Node->getValueType(0),
5743 {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
5744 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
5745 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5747 case Intrinsic::aarch64_sve_umin_single_x2:
5748 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5749 Node->getValueType(0),
5750 {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
5751 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
5752 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5754 case Intrinsic::aarch64_sve_fmin_single_x2:
5755 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5756 Node->getValueType(0),
5757 {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
5758 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
5759 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5761 case Intrinsic::aarch64_sve_smin_single_x4:
5762 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5763 Node->getValueType(0),
5764 {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
5765 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
5766 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5768 case Intrinsic::aarch64_sve_umin_single_x4:
5769 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5770 Node->getValueType(0),
5771 {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
5772 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
5773 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5775 case Intrinsic::aarch64_sve_fmin_single_x4:
5776 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5777 Node->getValueType(0),
5778 {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
5779 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
5780 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5782 case Intrinsic::aarch64_sve_smax_x2:
5783 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5784 Node->getValueType(0),
5785 {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
5786 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
5787 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5789 case Intrinsic::aarch64_sve_umax_x2:
5790 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5791 Node->getValueType(0),
5792 {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
5793 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
5794 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5796 case Intrinsic::aarch64_sve_fmax_x2:
5797 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5798 Node->getValueType(0),
5799 {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
5800 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
5801 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5803 case Intrinsic::aarch64_sve_smax_x4:
5804 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5805 Node->getValueType(0),
5806 {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
5807 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
5808 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5810 case Intrinsic::aarch64_sve_umax_x4:
5811 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5812 Node->getValueType(0),
5813 {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
5814 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
5815 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5817 case Intrinsic::aarch64_sve_fmax_x4:
5818 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5819 Node->getValueType(0),
5820 {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
5821 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
5822 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5824 case Intrinsic::aarch64_sve_smin_x2:
5825 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5826 Node->getValueType(0),
5827 {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
5828 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
5829 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5831 case Intrinsic::aarch64_sve_umin_x2:
5832 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5833 Node->getValueType(0),
5834 {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
5835 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
5836 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5838 case Intrinsic::aarch64_sve_fmin_x2:
5839 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5840 Node->getValueType(0),
5841 {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
5842 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
5843 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5845 case Intrinsic::aarch64_sve_smin_x4:
5846 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5847 Node->getValueType(0),
5848 {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
5849 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
5850 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5852 case Intrinsic::aarch64_sve_umin_x4:
5853 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5854 Node->getValueType(0),
5855 {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
5856 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
5857 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5859 case Intrinsic::aarch64_sve_fmin_x4:
5860 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5861 Node->getValueType(0),
5862 {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
5863 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
5864 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5866 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
5867 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5868 Node->getValueType(0),
5869 {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
5870 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
5871 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5873 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
5874 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5875 Node->getValueType(0),
5876 {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
5877 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
5878 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5880 case Intrinsic::aarch64_sve_fminnm_single_x2:
5881 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5882 Node->getValueType(0),
5883 {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
5884 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
5885 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5887 case Intrinsic::aarch64_sve_fminnm_single_x4:
5888 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5889 Node->getValueType(0),
5890 {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
5891 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
5892 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5894 case Intrinsic::aarch64_sve_fmaxnm_x2:
5895 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5896 Node->getValueType(0),
5897 {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
5898 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
5899 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5901 case Intrinsic::aarch64_sve_fmaxnm_x4:
5902 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5903 Node->getValueType(0),
5904 {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
5905 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
5906 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5908 case Intrinsic::aarch64_sve_fminnm_x2:
5909 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5910 Node->getValueType(0),
5911 {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
5912 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
5913 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5915 case Intrinsic::aarch64_sve_fminnm_x4:
5916 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5917 Node->getValueType(0),
5918 {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
5919 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
5920 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5922 case Intrinsic::aarch64_sve_fcvtzs_x2:
5923 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZS_2Z2Z_StoS);
5925 case Intrinsic::aarch64_sve_scvtf_x2:
5926 SelectCVTIntrinsic(
Node, 2, AArch64::SCVTF_2Z2Z_StoS);
5928 case Intrinsic::aarch64_sve_fcvtzu_x2:
5929 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZU_2Z2Z_StoS);
5931 case Intrinsic::aarch64_sve_ucvtf_x2:
5932 SelectCVTIntrinsic(
Node, 2, AArch64::UCVTF_2Z2Z_StoS);
5934 case Intrinsic::aarch64_sve_fcvtzs_x4:
5935 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZS_4Z4Z_StoS);
5937 case Intrinsic::aarch64_sve_scvtf_x4:
5938 SelectCVTIntrinsic(
Node, 4, AArch64::SCVTF_4Z4Z_StoS);
5940 case Intrinsic::aarch64_sve_fcvtzu_x4:
5941 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZU_4Z4Z_StoS);
5943 case Intrinsic::aarch64_sve_ucvtf_x4:
5944 SelectCVTIntrinsic(
Node, 4, AArch64::UCVTF_4Z4Z_StoS);
5946 case Intrinsic::aarch64_sve_fcvt_widen_x2:
5947 SelectUnaryMultiIntrinsic(
Node, 2,
false, AArch64::FCVT_2ZZ_H_S);
5949 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
5950 SelectUnaryMultiIntrinsic(
Node, 2,
false, AArch64::FCVTL_2ZZ_H_S);
5952 case Intrinsic::aarch64_sve_sclamp_single_x2:
5953 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5954 Node->getValueType(0),
5955 {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
5956 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
5957 SelectClamp(
Node, 2,
Op);
5959 case Intrinsic::aarch64_sve_uclamp_single_x2:
5960 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5961 Node->getValueType(0),
5962 {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
5963 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
5964 SelectClamp(
Node, 2,
Op);
5966 case Intrinsic::aarch64_sve_fclamp_single_x2:
5967 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5968 Node->getValueType(0),
5969 {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
5970 AArch64::FCLAMP_VG2_2Z2Z_D}))
5971 SelectClamp(
Node, 2,
Op);
5973 case Intrinsic::aarch64_sve_bfclamp_single_x2:
5974 SelectClamp(
Node, 2, AArch64::BFCLAMP_VG2_2ZZZ_H);
5976 case Intrinsic::aarch64_sve_sclamp_single_x4:
5977 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5978 Node->getValueType(0),
5979 {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
5980 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
5981 SelectClamp(
Node, 4,
Op);
5983 case Intrinsic::aarch64_sve_uclamp_single_x4:
5984 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5985 Node->getValueType(0),
5986 {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
5987 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
5988 SelectClamp(
Node, 4,
Op);
5990 case Intrinsic::aarch64_sve_fclamp_single_x4:
5991 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5992 Node->getValueType(0),
5993 {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
5994 AArch64::FCLAMP_VG4_4Z4Z_D}))
5995 SelectClamp(
Node, 4,
Op);
5997 case Intrinsic::aarch64_sve_bfclamp_single_x4:
5998 SelectClamp(
Node, 4, AArch64::BFCLAMP_VG4_4ZZZ_H);
6000 case Intrinsic::aarch64_sve_add_single_x2:
6001 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6002 Node->getValueType(0),
6003 {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
6004 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
6005 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
6007 case Intrinsic::aarch64_sve_add_single_x4:
6008 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6009 Node->getValueType(0),
6010 {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
6011 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
6012 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
6014 case Intrinsic::aarch64_sve_zip_x2:
6015 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6016 Node->getValueType(0),
6017 {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
6018 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
6019 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6021 case Intrinsic::aarch64_sve_zipq_x2:
6022 SelectUnaryMultiIntrinsic(
Node, 2,
false,
6023 AArch64::ZIP_VG2_2ZZZ_Q);
6025 case Intrinsic::aarch64_sve_zip_x4:
6026 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6027 Node->getValueType(0),
6028 {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
6029 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
6030 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6032 case Intrinsic::aarch64_sve_zipq_x4:
6033 SelectUnaryMultiIntrinsic(
Node, 4,
true,
6034 AArch64::ZIP_VG4_4Z4Z_Q);
6036 case Intrinsic::aarch64_sve_uzp_x2:
6037 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6038 Node->getValueType(0),
6039 {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
6040 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
6041 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6043 case Intrinsic::aarch64_sve_uzpq_x2:
6044 SelectUnaryMultiIntrinsic(
Node, 2,
false,
6045 AArch64::UZP_VG2_2ZZZ_Q);
6047 case Intrinsic::aarch64_sve_uzp_x4:
6048 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6049 Node->getValueType(0),
6050 {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
6051 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
6052 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6054 case Intrinsic::aarch64_sve_uzpq_x4:
6055 SelectUnaryMultiIntrinsic(
Node, 4,
true,
6056 AArch64::UZP_VG4_4Z4Z_Q);
6058 case Intrinsic::aarch64_sve_sel_x2:
6059 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6060 Node->getValueType(0),
6061 {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
6062 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
6063 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op,
true);
6065 case Intrinsic::aarch64_sve_sel_x4:
6066 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6067 Node->getValueType(0),
6068 {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
6069 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
6070 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op,
true);
6072 case Intrinsic::aarch64_sve_frinta_x2:
6073 SelectFrintFromVT(
Node, 2, AArch64::FRINTA_2Z2Z_S);
6075 case Intrinsic::aarch64_sve_frinta_x4:
6076 SelectFrintFromVT(
Node, 4, AArch64::FRINTA_4Z4Z_S);
6078 case Intrinsic::aarch64_sve_frintm_x2:
6079 SelectFrintFromVT(
Node, 2, AArch64::FRINTM_2Z2Z_S);
6081 case Intrinsic::aarch64_sve_frintm_x4:
6082 SelectFrintFromVT(
Node, 4, AArch64::FRINTM_4Z4Z_S);
6084 case Intrinsic::aarch64_sve_frintn_x2:
6085 SelectFrintFromVT(
Node, 2, AArch64::FRINTN_2Z2Z_S);
6087 case Intrinsic::aarch64_sve_frintn_x4:
6088 SelectFrintFromVT(
Node, 4, AArch64::FRINTN_4Z4Z_S);
6090 case Intrinsic::aarch64_sve_frintp_x2:
6091 SelectFrintFromVT(
Node, 2, AArch64::FRINTP_2Z2Z_S);
6093 case Intrinsic::aarch64_sve_frintp_x4:
6094 SelectFrintFromVT(
Node, 4, AArch64::FRINTP_4Z4Z_S);
6096 case Intrinsic::aarch64_sve_sunpk_x2:
6097 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6098 Node->getValueType(0),
6099 {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6100 AArch64::SUNPK_VG2_2ZZ_D}))
6101 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6103 case Intrinsic::aarch64_sve_uunpk_x2:
6104 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6105 Node->getValueType(0),
6106 {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6107 AArch64::UUNPK_VG2_2ZZ_D}))
6108 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
6110 case Intrinsic::aarch64_sve_sunpk_x4:
6111 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6112 Node->getValueType(0),
6113 {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6114 AArch64::SUNPK_VG4_4Z2Z_D}))
6115 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6117 case Intrinsic::aarch64_sve_uunpk_x4:
6118 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6119 Node->getValueType(0),
6120 {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6121 AArch64::UUNPK_VG4_4Z2Z_D}))
6122 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
6124 case Intrinsic::aarch64_sve_pext_x2: {
6125 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6126 Node->getValueType(0),
6127 {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6128 AArch64::PEXT_2PCI_D}))
6129 SelectPExtPair(
Node,
Op);
6136 unsigned IntNo =
Node->getConstantOperandVal(1);
6137 if (
Node->getNumOperands() >= 3)
6138 VT =
Node->getOperand(2)->getValueType(0);
6142 case Intrinsic::aarch64_neon_st1x2: {
6143 if (VT == MVT::v8i8) {
6144 SelectStore(
Node, 2, AArch64::ST1Twov8b);
6146 }
else if (VT == MVT::v16i8) {
6147 SelectStore(
Node, 2, AArch64::ST1Twov16b);
6149 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6150 VT == MVT::v4bf16) {
6151 SelectStore(
Node, 2, AArch64::ST1Twov4h);
6153 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6154 VT == MVT::v8bf16) {
6155 SelectStore(
Node, 2, AArch64::ST1Twov8h);
6157 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6158 SelectStore(
Node, 2, AArch64::ST1Twov2s);
6160 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6161 SelectStore(
Node, 2, AArch64::ST1Twov4s);
6163 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6164 SelectStore(
Node, 2, AArch64::ST1Twov2d);
6166 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6167 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6172 case Intrinsic::aarch64_neon_st1x3: {
6173 if (VT == MVT::v8i8) {
6174 SelectStore(
Node, 3, AArch64::ST1Threev8b);
6176 }
else if (VT == MVT::v16i8) {
6177 SelectStore(
Node, 3, AArch64::ST1Threev16b);
6179 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6180 VT == MVT::v4bf16) {
6181 SelectStore(
Node, 3, AArch64::ST1Threev4h);
6183 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6184 VT == MVT::v8bf16) {
6185 SelectStore(
Node, 3, AArch64::ST1Threev8h);
6187 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6188 SelectStore(
Node, 3, AArch64::ST1Threev2s);
6190 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6191 SelectStore(
Node, 3, AArch64::ST1Threev4s);
6193 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6194 SelectStore(
Node, 3, AArch64::ST1Threev2d);
6196 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6197 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6202 case Intrinsic::aarch64_neon_st1x4: {
6203 if (VT == MVT::v8i8) {
6204 SelectStore(
Node, 4, AArch64::ST1Fourv8b);
6206 }
else if (VT == MVT::v16i8) {
6207 SelectStore(
Node, 4, AArch64::ST1Fourv16b);
6209 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6210 VT == MVT::v4bf16) {
6211 SelectStore(
Node, 4, AArch64::ST1Fourv4h);
6213 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6214 VT == MVT::v8bf16) {
6215 SelectStore(
Node, 4, AArch64::ST1Fourv8h);
6217 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6218 SelectStore(
Node, 4, AArch64::ST1Fourv2s);
6220 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6221 SelectStore(
Node, 4, AArch64::ST1Fourv4s);
6223 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6224 SelectStore(
Node, 4, AArch64::ST1Fourv2d);
6226 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6227 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6232 case Intrinsic::aarch64_neon_st2: {
6233 if (VT == MVT::v8i8) {
6234 SelectStore(
Node, 2, AArch64::ST2Twov8b);
6236 }
else if (VT == MVT::v16i8) {
6237 SelectStore(
Node, 2, AArch64::ST2Twov16b);
6239 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6240 VT == MVT::v4bf16) {
6241 SelectStore(
Node, 2, AArch64::ST2Twov4h);
6243 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6244 VT == MVT::v8bf16) {
6245 SelectStore(
Node, 2, AArch64::ST2Twov8h);
6247 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6248 SelectStore(
Node, 2, AArch64::ST2Twov2s);
6250 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6251 SelectStore(
Node, 2, AArch64::ST2Twov4s);
6253 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6254 SelectStore(
Node, 2, AArch64::ST2Twov2d);
6256 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6257 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6262 case Intrinsic::aarch64_neon_st3: {
6263 if (VT == MVT::v8i8) {
6264 SelectStore(
Node, 3, AArch64::ST3Threev8b);
6266 }
else if (VT == MVT::v16i8) {
6267 SelectStore(
Node, 3, AArch64::ST3Threev16b);
6269 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6270 VT == MVT::v4bf16) {
6271 SelectStore(
Node, 3, AArch64::ST3Threev4h);
6273 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6274 VT == MVT::v8bf16) {
6275 SelectStore(
Node, 3, AArch64::ST3Threev8h);
6277 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6278 SelectStore(
Node, 3, AArch64::ST3Threev2s);
6280 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6281 SelectStore(
Node, 3, AArch64::ST3Threev4s);
6283 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6284 SelectStore(
Node, 3, AArch64::ST3Threev2d);
6286 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6287 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6292 case Intrinsic::aarch64_neon_st4: {
6293 if (VT == MVT::v8i8) {
6294 SelectStore(
Node, 4, AArch64::ST4Fourv8b);
6296 }
else if (VT == MVT::v16i8) {
6297 SelectStore(
Node, 4, AArch64::ST4Fourv16b);
6299 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6300 VT == MVT::v4bf16) {
6301 SelectStore(
Node, 4, AArch64::ST4Fourv4h);
6303 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6304 VT == MVT::v8bf16) {
6305 SelectStore(
Node, 4, AArch64::ST4Fourv8h);
6307 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6308 SelectStore(
Node, 4, AArch64::ST4Fourv2s);
6310 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6311 SelectStore(
Node, 4, AArch64::ST4Fourv4s);
6313 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6314 SelectStore(
Node, 4, AArch64::ST4Fourv2d);
6316 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6317 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6322 case Intrinsic::aarch64_neon_st2lane: {
6323 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6324 SelectStoreLane(
Node, 2, AArch64::ST2i8);
6326 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6327 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6328 SelectStoreLane(
Node, 2, AArch64::ST2i16);
6330 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6332 SelectStoreLane(
Node, 2, AArch64::ST2i32);
6334 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6336 SelectStoreLane(
Node, 2, AArch64::ST2i64);
6341 case Intrinsic::aarch64_neon_st3lane: {
6342 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6343 SelectStoreLane(
Node, 3, AArch64::ST3i8);
6345 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6346 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6347 SelectStoreLane(
Node, 3, AArch64::ST3i16);
6349 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6351 SelectStoreLane(
Node, 3, AArch64::ST3i32);
6353 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6355 SelectStoreLane(
Node, 3, AArch64::ST3i64);
6360 case Intrinsic::aarch64_neon_st4lane: {
6361 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6362 SelectStoreLane(
Node, 4, AArch64::ST4i8);
6364 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6365 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6366 SelectStoreLane(
Node, 4, AArch64::ST4i16);
6368 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6370 SelectStoreLane(
Node, 4, AArch64::ST4i32);
6372 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6374 SelectStoreLane(
Node, 4, AArch64::ST4i64);
6379 case Intrinsic::aarch64_sve_st2q: {
6380 SelectPredicatedStore(
Node, 2, 4, AArch64::ST2Q, AArch64::ST2Q_IMM);
6383 case Intrinsic::aarch64_sve_st3q: {
6384 SelectPredicatedStore(
Node, 3, 4, AArch64::ST3Q, AArch64::ST3Q_IMM);
6387 case Intrinsic::aarch64_sve_st4q: {
6388 SelectPredicatedStore(
Node, 4, 4, AArch64::ST4Q, AArch64::ST4Q_IMM);
6391 case Intrinsic::aarch64_sve_st2: {
6392 if (VT == MVT::nxv16i8) {
6393 SelectPredicatedStore(
Node, 2, 0, AArch64::ST2B, AArch64::ST2B_IMM);
6395 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6396 VT == MVT::nxv8bf16) {
6397 SelectPredicatedStore(
Node, 2, 1, AArch64::ST2H, AArch64::ST2H_IMM);
6399 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6400 SelectPredicatedStore(
Node, 2, 2, AArch64::ST2W, AArch64::ST2W_IMM);
6402 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6403 SelectPredicatedStore(
Node, 2, 3, AArch64::ST2D, AArch64::ST2D_IMM);
6408 case Intrinsic::aarch64_sve_st3: {
6409 if (VT == MVT::nxv16i8) {
6410 SelectPredicatedStore(
Node, 3, 0, AArch64::ST3B, AArch64::ST3B_IMM);
6412 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6413 VT == MVT::nxv8bf16) {
6414 SelectPredicatedStore(
Node, 3, 1, AArch64::ST3H, AArch64::ST3H_IMM);
6416 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6417 SelectPredicatedStore(
Node, 3, 2, AArch64::ST3W, AArch64::ST3W_IMM);
6419 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6420 SelectPredicatedStore(
Node, 3, 3, AArch64::ST3D, AArch64::ST3D_IMM);
6425 case Intrinsic::aarch64_sve_st4: {
6426 if (VT == MVT::nxv16i8) {
6427 SelectPredicatedStore(
Node, 4, 0, AArch64::ST4B, AArch64::ST4B_IMM);
6429 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6430 VT == MVT::nxv8bf16) {
6431 SelectPredicatedStore(
Node, 4, 1, AArch64::ST4H, AArch64::ST4H_IMM);
6433 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6434 SelectPredicatedStore(
Node, 4, 2, AArch64::ST4W, AArch64::ST4W_IMM);
6436 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6437 SelectPredicatedStore(
Node, 4, 3, AArch64::ST4D, AArch64::ST4D_IMM);
6446 if (VT == MVT::v8i8) {
6447 SelectPostLoad(
Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
6449 }
else if (VT == MVT::v16i8) {
6450 SelectPostLoad(
Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
6452 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6453 SelectPostLoad(
Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
6455 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6456 SelectPostLoad(
Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
6458 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6459 SelectPostLoad(
Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
6461 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6462 SelectPostLoad(
Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
6464 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6465 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6467 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6468 SelectPostLoad(
Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
6474 if (VT == MVT::v8i8) {
6475 SelectPostLoad(
Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
6477 }
else if (VT == MVT::v16i8) {
6478 SelectPostLoad(
Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
6480 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6481 SelectPostLoad(
Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
6483 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6484 SelectPostLoad(
Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
6486 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6487 SelectPostLoad(
Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
6489 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6490 SelectPostLoad(
Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
6492 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6493 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6495 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6496 SelectPostLoad(
Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
6502 if (VT == MVT::v8i8) {
6503 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
6505 }
else if (VT == MVT::v16i8) {
6506 SelectPostLoad(
Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
6508 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6509 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
6511 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6512 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
6514 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6515 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
6517 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6518 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
6520 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6521 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6523 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6524 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
6530 if (VT == MVT::v8i8) {
6531 SelectPostLoad(
Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
6533 }
else if (VT == MVT::v16i8) {
6534 SelectPostLoad(
Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
6536 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6537 SelectPostLoad(
Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
6539 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6540 SelectPostLoad(
Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
6542 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6543 SelectPostLoad(
Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
6545 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6546 SelectPostLoad(
Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
6548 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6549 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6551 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6552 SelectPostLoad(
Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
6558 if (VT == MVT::v8i8) {
6559 SelectPostLoad(
Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
6561 }
else if (VT == MVT::v16i8) {
6562 SelectPostLoad(
Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
6564 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6565 SelectPostLoad(
Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
6567 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6568 SelectPostLoad(
Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
6570 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6571 SelectPostLoad(
Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
6573 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6574 SelectPostLoad(
Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
6576 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6577 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6579 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6580 SelectPostLoad(
Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
6586 if (VT == MVT::v8i8) {
6587 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
6589 }
else if (VT == MVT::v16i8) {
6590 SelectPostLoad(
Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
6592 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6593 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
6595 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6596 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
6598 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6599 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
6601 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6602 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
6604 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6605 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6607 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6608 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
6614 if (VT == MVT::v8i8) {
6615 SelectPostLoad(
Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
6617 }
else if (VT == MVT::v16i8) {
6618 SelectPostLoad(
Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
6620 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6621 SelectPostLoad(
Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
6623 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6624 SelectPostLoad(
Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
6626 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6627 SelectPostLoad(
Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
6629 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6630 SelectPostLoad(
Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
6632 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6633 SelectPostLoad(
Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
6635 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6636 SelectPostLoad(
Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
6642 if (VT == MVT::v8i8) {
6643 SelectPostLoad(
Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
6645 }
else if (VT == MVT::v16i8) {
6646 SelectPostLoad(
Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
6648 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6649 SelectPostLoad(
Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
6651 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6652 SelectPostLoad(
Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
6654 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6655 SelectPostLoad(
Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
6657 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6658 SelectPostLoad(
Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
6660 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6661 SelectPostLoad(
Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
6663 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6664 SelectPostLoad(
Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
6670 if (VT == MVT::v8i8) {
6671 SelectPostLoad(
Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
6673 }
else if (VT == MVT::v16i8) {
6674 SelectPostLoad(
Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
6676 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6677 SelectPostLoad(
Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
6679 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6680 SelectPostLoad(
Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
6682 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6683 SelectPostLoad(
Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
6685 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6686 SelectPostLoad(
Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
6688 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6689 SelectPostLoad(
Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
6691 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6692 SelectPostLoad(
Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
6698 if (VT == MVT::v8i8) {
6699 SelectPostLoad(
Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
6701 }
else if (VT == MVT::v16i8) {
6702 SelectPostLoad(
Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
6704 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6705 SelectPostLoad(
Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
6707 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6708 SelectPostLoad(
Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
6710 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6711 SelectPostLoad(
Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
6713 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6714 SelectPostLoad(
Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
6716 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6717 SelectPostLoad(
Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
6719 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6720 SelectPostLoad(
Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
6726 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6727 SelectPostLoadLane(
Node, 1, AArch64::LD1i8_POST);
6729 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6730 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6731 SelectPostLoadLane(
Node, 1, AArch64::LD1i16_POST);
6733 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6735 SelectPostLoadLane(
Node, 1, AArch64::LD1i32_POST);
6737 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6739 SelectPostLoadLane(
Node, 1, AArch64::LD1i64_POST);
6745 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6746 SelectPostLoadLane(
Node, 2, AArch64::LD2i8_POST);
6748 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6749 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6750 SelectPostLoadLane(
Node, 2, AArch64::LD2i16_POST);
6752 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6754 SelectPostLoadLane(
Node, 2, AArch64::LD2i32_POST);
6756 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6758 SelectPostLoadLane(
Node, 2, AArch64::LD2i64_POST);
6764 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6765 SelectPostLoadLane(
Node, 3, AArch64::LD3i8_POST);
6767 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6768 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6769 SelectPostLoadLane(
Node, 3, AArch64::LD3i16_POST);
6771 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6773 SelectPostLoadLane(
Node, 3, AArch64::LD3i32_POST);
6775 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6777 SelectPostLoadLane(
Node, 3, AArch64::LD3i64_POST);
6783 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6784 SelectPostLoadLane(
Node, 4, AArch64::LD4i8_POST);
6786 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6787 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6788 SelectPostLoadLane(
Node, 4, AArch64::LD4i16_POST);
6790 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6792 SelectPostLoadLane(
Node, 4, AArch64::LD4i32_POST);
6794 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6796 SelectPostLoadLane(
Node, 4, AArch64::LD4i64_POST);
6802 VT =
Node->getOperand(1).getValueType();
6803 if (VT == MVT::v8i8) {
6804 SelectPostStore(
Node, 2, AArch64::ST2Twov8b_POST);
6806 }
else if (VT == MVT::v16i8) {
6807 SelectPostStore(
Node, 2, AArch64::ST2Twov16b_POST);
6809 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6810 SelectPostStore(
Node, 2, AArch64::ST2Twov4h_POST);
6812 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6813 SelectPostStore(
Node, 2, AArch64::ST2Twov8h_POST);
6815 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6816 SelectPostStore(
Node, 2, AArch64::ST2Twov2s_POST);
6818 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6819 SelectPostStore(
Node, 2, AArch64::ST2Twov4s_POST);
6821 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6822 SelectPostStore(
Node, 2, AArch64::ST2Twov2d_POST);
6824 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6825 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6831 VT =
Node->getOperand(1).getValueType();
6832 if (VT == MVT::v8i8) {
6833 SelectPostStore(
Node, 3, AArch64::ST3Threev8b_POST);
6835 }
else if (VT == MVT::v16i8) {
6836 SelectPostStore(
Node, 3, AArch64::ST3Threev16b_POST);
6838 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6839 SelectPostStore(
Node, 3, AArch64::ST3Threev4h_POST);
6841 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6842 SelectPostStore(
Node, 3, AArch64::ST3Threev8h_POST);
6844 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6845 SelectPostStore(
Node, 3, AArch64::ST3Threev2s_POST);
6847 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6848 SelectPostStore(
Node, 3, AArch64::ST3Threev4s_POST);
6850 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6851 SelectPostStore(
Node, 3, AArch64::ST3Threev2d_POST);
6853 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6854 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6860 VT =
Node->getOperand(1).getValueType();
6861 if (VT == MVT::v8i8) {
6862 SelectPostStore(
Node, 4, AArch64::ST4Fourv8b_POST);
6864 }
else if (VT == MVT::v16i8) {
6865 SelectPostStore(
Node, 4, AArch64::ST4Fourv16b_POST);
6867 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6868 SelectPostStore(
Node, 4, AArch64::ST4Fourv4h_POST);
6870 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6871 SelectPostStore(
Node, 4, AArch64::ST4Fourv8h_POST);
6873 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6874 SelectPostStore(
Node, 4, AArch64::ST4Fourv2s_POST);
6876 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6877 SelectPostStore(
Node, 4, AArch64::ST4Fourv4s_POST);
6879 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6880 SelectPostStore(
Node, 4, AArch64::ST4Fourv2d_POST);
6882 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6883 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6889 VT =
Node->getOperand(1).getValueType();
6890 if (VT == MVT::v8i8) {
6891 SelectPostStore(
Node, 2, AArch64::ST1Twov8b_POST);
6893 }
else if (VT == MVT::v16i8) {
6894 SelectPostStore(
Node, 2, AArch64::ST1Twov16b_POST);
6896 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6897 SelectPostStore(
Node, 2, AArch64::ST1Twov4h_POST);
6899 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6900 SelectPostStore(
Node, 2, AArch64::ST1Twov8h_POST);
6902 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6903 SelectPostStore(
Node, 2, AArch64::ST1Twov2s_POST);
6905 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6906 SelectPostStore(
Node, 2, AArch64::ST1Twov4s_POST);
6908 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6909 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6911 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6912 SelectPostStore(
Node, 2, AArch64::ST1Twov2d_POST);
6918 VT =
Node->getOperand(1).getValueType();
6919 if (VT == MVT::v8i8) {
6920 SelectPostStore(
Node, 3, AArch64::ST1Threev8b_POST);
6922 }
else if (VT == MVT::v16i8) {
6923 SelectPostStore(
Node, 3, AArch64::ST1Threev16b_POST);
6925 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6926 SelectPostStore(
Node, 3, AArch64::ST1Threev4h_POST);
6928 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
6929 SelectPostStore(
Node, 3, AArch64::ST1Threev8h_POST);
6931 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6932 SelectPostStore(
Node, 3, AArch64::ST1Threev2s_POST);
6934 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6935 SelectPostStore(
Node, 3, AArch64::ST1Threev4s_POST);
6937 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6938 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6940 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6941 SelectPostStore(
Node, 3, AArch64::ST1Threev2d_POST);
6947 VT =
Node->getOperand(1).getValueType();
6948 if (VT == MVT::v8i8) {
6949 SelectPostStore(
Node, 4, AArch64::ST1Fourv8b_POST);
6951 }
else if (VT == MVT::v16i8) {
6952 SelectPostStore(
Node, 4, AArch64::ST1Fourv16b_POST);
6954 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6955 SelectPostStore(
Node, 4, AArch64::ST1Fourv4h_POST);
6957 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6958 SelectPostStore(
Node, 4, AArch64::ST1Fourv8h_POST);
6960 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6961 SelectPostStore(
Node, 4, AArch64::ST1Fourv2s_POST);
6963 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6964 SelectPostStore(
Node, 4, AArch64::ST1Fourv4s_POST);
6966 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6967 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6969 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6970 SelectPostStore(
Node, 4, AArch64::ST1Fourv2d_POST);
6976 VT =
Node->getOperand(1).getValueType();
6977 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6978 SelectPostStoreLane(
Node, 2, AArch64::ST2i8_POST);
6980 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6981 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6982 SelectPostStoreLane(
Node, 2, AArch64::ST2i16_POST);
6984 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6986 SelectPostStoreLane(
Node, 2, AArch64::ST2i32_POST);
6988 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6990 SelectPostStoreLane(
Node, 2, AArch64::ST2i64_POST);
6996 VT =
Node->getOperand(1).getValueType();
6997 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6998 SelectPostStoreLane(
Node, 3, AArch64::ST3i8_POST);
7000 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7001 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7002 SelectPostStoreLane(
Node, 3, AArch64::ST3i16_POST);
7004 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7006 SelectPostStoreLane(
Node, 3, AArch64::ST3i32_POST);
7008 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7010 SelectPostStoreLane(
Node, 3, AArch64::ST3i64_POST);
7016 VT =
Node->getOperand(1).getValueType();
7017 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7018 SelectPostStoreLane(
Node, 4, AArch64::ST4i8_POST);
7020 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7021 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7022 SelectPostStoreLane(
Node, 4, AArch64::ST4i16_POST);
7024 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7026 SelectPostStoreLane(
Node, 4, AArch64::ST4i32_POST);
7028 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7030 SelectPostStoreLane(
Node, 4, AArch64::ST4i64_POST);
7036 if (VT == MVT::nxv16i8) {
7037 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B);
7039 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7040 VT == MVT::nxv8bf16) {
7041 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H);
7043 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7044 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W);
7046 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7047 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D);
7053 if (VT == MVT::nxv16i8) {
7054 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B);
7056 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7057 VT == MVT::nxv8bf16) {
7058 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H);
7060 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7061 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W);
7063 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7064 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D);
7070 if (VT == MVT::nxv16i8) {
7071 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B);
7073 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7074 VT == MVT::nxv8bf16) {
7075 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H);
7077 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7078 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W);
7080 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7081 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D);
7096 return new AArch64DAGToDAGISelLegacy(
TM, OptLevel);
7108 assert(NumVec > 0 && NumVec < 5 &&
"Invalid number of vectors.");
7112 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7113 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7127 if (isa<MemSDNode>(Root))
7128 return cast<MemSDNode>(Root)->getMemoryVT();
7130 if (isa<MemIntrinsicSDNode>(Root))
7131 return cast<MemIntrinsicSDNode>(Root)->getMemoryVT();
7133 const unsigned Opcode = Root->
getOpcode();
7141 return cast<VTSDNode>(Root->
getOperand(3))->getVT();
7143 return cast<VTSDNode>(Root->
getOperand(4))->getVT();
7163 case Intrinsic::aarch64_sme_ldr:
7164 case Intrinsic::aarch64_sme_str:
7165 return MVT::nxv16i8;
7166 case Intrinsic::aarch64_sve_prf:
7171 case Intrinsic::aarch64_sve_ld2_sret:
7172 case Intrinsic::aarch64_sve_ld2q_sret:
7175 case Intrinsic::aarch64_sve_st2q:
7178 case Intrinsic::aarch64_sve_ld3_sret:
7179 case Intrinsic::aarch64_sve_ld3q_sret:
7182 case Intrinsic::aarch64_sve_st3q:
7185 case Intrinsic::aarch64_sve_ld4_sret:
7186 case Intrinsic::aarch64_sve_ld4q_sret:
7189 case Intrinsic::aarch64_sve_st4q:
7192 case Intrinsic::aarch64_sve_ld1udq:
7193 case Intrinsic::aarch64_sve_st1dq:
7194 return EVT(MVT::nxv1i64);
7195 case Intrinsic::aarch64_sve_ld1uwq:
7196 case Intrinsic::aarch64_sve_st1wq:
7197 return EVT(MVT::nxv1i32);
7204template <
int64_t Min,
int64_t Max>
7205bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(
SDNode *Root,
SDValue N,
7213 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
7218 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
7237 int64_t MulImm = cast<ConstantSDNode>(VScale.
getOperand(0))->getSExtValue();
7239 if ((MulImm % MemWidthBytes) != 0)
7242 int64_t
Offset = MulImm / MemWidthBytes;
7243 if (Offset < Min || Offset > Max)
7246 Base =
N.getOperand(0);
7248 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
7255 OffImm = CurDAG->getTargetConstant(
Offset,
SDLoc(
N), MVT::i64);
7261bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(
SDValue N,
unsigned Scale,
7279 if (
auto C = dyn_cast<ConstantSDNode>(RHS)) {
7280 int64_t ImmOff =
C->getSExtValue();
7281 unsigned Size = 1 << Scale;
7290 Offset = CurDAG->getTargetConstant(ImmOff >> Scale,
DL, MVT::i64);
7292 SDNode *
MI = CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
7302 if (
auto *
C = dyn_cast<ConstantSDNode>(ShiftRHS))
7303 if (
C->getZExtValue() == Scale) {
7312bool AArch64DAGToDAGISel::SelectAllActivePredicate(
SDValue N) {
7319bool AArch64DAGToDAGISel::SelectAnyPredicate(
SDValue N) {
7320 EVT VT =
N.getValueType();
7324bool AArch64DAGToDAGISel::SelectSMETileSlice(
SDValue N,
unsigned MaxSize,
7329 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
7330 int64_t ImmOff =
C->getSExtValue();
7331 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0))) {
7332 Base =
N.getOperand(0);
7333 Offset = CurDAG->getTargetConstant(ImmOff / Scale,
SDLoc(
N), MVT::i64);
7340 Offset = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
static int getIntOperandFromRegisterString(StringRef RegString)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
Does DstMask form a complementary pair with the mask provided by BitsToBeInserted,...
static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N)
Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32,...
static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op, SDValue &Src, int &DstLSB, int &Width)
static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &DstLSB, int &Width)
Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL,...
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG)
static std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG)
static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
static bool isShiftedMask(uint64_t Mask, EVT VT)
bool SelectSMETile(unsigned &BaseReg, unsigned TileNum)
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root)
Return the EVT of the data associated to a memory operation in Root.
static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static bool isWorthFoldingADDlow(SDValue N)
If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD i...
static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N)
getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef< unsigned > Opcodes)
This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bi...
static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT, unsigned NumVec)
When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent sca...
static bool isPreferredADD(int64_t ImmOff)
static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
Create a machine node performing a notional SHL of Op by ShlAmount.
static bool isWorthFoldingSHL(SDValue V)
Determine whether it is worth it to fold SHL into the addressing mode.
static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1, SDValue Src, SDValue Dst, SelectionDAG *CurDAG, const bool BiggerPattern)
static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits, unsigned Depth)
static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth=0)
static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected)
static AArch64_AM::ShiftExtendType getExtendTypeForNode(SDValue N, bool IsLoadStore=false)
getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
static bool isIntImmediate(const SDNode *N, uint64_t &Imm)
isIntImmediate - This method tests to see if the node is a constant operand.
static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG, SDValue &ShiftedOperand, uint64_t &EncodedShiftImm)
static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range, unsigned Size)
Check if the immediate offset is valid as a scaled immediate.
static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
amdgpu AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
unsigned const TargetRegisterInfo * TRI
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool isLittleEndian() const
bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const
bool getExactInverse(APFloat *inv) const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
void flipAllBits()
Toggle every bit to its opposite value.
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
An arbitrary precision integer that knows its signedness.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
const Constant * getConstVal() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
int64_t getOffset() const
const GlobalValue * getGlobal() const
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint8_t getStackID(int ObjectIdx) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
static constexpr unsigned MaxRecursionDepth
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
unsigned getID() const
Return the register class ID number.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A Use represents the edge between a Value definition and its users.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
uint32_t parseGenericRegister(StringRef Name)
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static bool processLogicalImmediate(uint64_t Imm, unsigned RegSize, uint64_t &Encoding)
processLogicalImmediate - Determine if an immediate value can be encoded as the immediate operand of ...
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static constexpr unsigned SVEBitsPerBlock
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
@ Undef
Value of the register doesn't matter.
Not(const Pred &P) -> Not< Pred >
Reg
All possible values of the reg field in the ModR/M byte.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
FunctionPass * createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOptLevel OptLevel)
createAArch64ISelDag - This pass converts a legalized DAG into a AArch64-specific DAG,...
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
constexpr unsigned BitWidth
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
unsigned getBitWidth() const
Get the bit width of this value.