22#include "llvm/IR/IntrinsicsAArch64.h"
31#define DEBUG_TYPE "aarch64-isel"
32#define PASS_NAME "AArch64 Instruction Selection"
49 AArch64DAGToDAGISel() =
delete;
66 std::vector<SDValue> &OutOps)
override;
68 template <
signed Low,
signed High,
signed Scale>
76 return SelectShiftedRegister(
N,
false, Reg, Shift);
79 return SelectShiftedRegister(
N,
true, Reg, Shift);
82 return SelectAddrModeIndexed7S(
N, 1,
Base, OffImm);
85 return SelectAddrModeIndexed7S(
N, 2,
Base, OffImm);
88 return SelectAddrModeIndexed7S(
N, 4,
Base, OffImm);
91 return SelectAddrModeIndexed7S(
N, 8,
Base, OffImm);
94 return SelectAddrModeIndexed7S(
N, 16,
Base, OffImm);
97 return SelectAddrModeIndexedBitWidth(
N,
true, 9, 16,
Base, OffImm);
100 return SelectAddrModeIndexedBitWidth(
N,
false, 6, 16,
Base, OffImm);
103 return SelectAddrModeIndexed(
N, 1,
Base, OffImm);
106 return SelectAddrModeIndexed(
N, 2,
Base, OffImm);
109 return SelectAddrModeIndexed(
N, 4,
Base, OffImm);
112 return SelectAddrModeIndexed(
N, 8,
Base, OffImm);
115 return SelectAddrModeIndexed(
N, 16,
Base, OffImm);
118 return SelectAddrModeUnscaled(
N, 1,
Base, OffImm);
121 return SelectAddrModeUnscaled(
N, 2,
Base, OffImm);
124 return SelectAddrModeUnscaled(
N, 4,
Base, OffImm);
127 return SelectAddrModeUnscaled(
N, 8,
Base, OffImm);
130 return SelectAddrModeUnscaled(
N, 16,
Base, OffImm);
132 template <
unsigned Size,
unsigned Max>
136 bool Found = SelectAddrModeIndexed(
N,
Size,
Base, OffImm);
138 if (
auto *CI = dyn_cast<ConstantSDNode>(OffImm)) {
139 int64_t
C = CI->getSExtValue();
147 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
154 return SelectAddrModeWRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
160 return SelectAddrModeXRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
165 N =
N->getOperand(0);
167 !isa<ConstantSDNode>(
N->getOperand(1)))
169 EVT VT =
N->getValueType(0);
170 EVT LVT =
N->getOperand(0).getValueType();
171 unsigned Index =
N->getConstantOperandVal(1);
175 Res =
N->getOperand(0);
183 EVT VT =
Op.getValueType();
184 unsigned ShtAmt =
N->getConstantOperandVal(1);
191 Op.getOperand(1).getConstantOperandVal(0)
192 <<
Op.getOperand(1).getConstantOperandVal(1));
194 isa<ConstantSDNode>(
Op.getOperand(1).getOperand(0)))
196 Op.getOperand(1).getConstantOperandVal(0));
200 if (Imm != 1ULL << (ShtAmt - 1))
203 Res1 =
Op.getOperand(0);
204 Res2 = CurDAG->getTargetConstant(ShtAmt,
SDLoc(
N), MVT::i32);
208 bool SelectDupZeroOrUndef(
SDValue N) {
209 switch(
N->getOpcode()) {
214 auto Opnd0 =
N->getOperand(0);
229 switch(
N->getOpcode()) {
232 auto Opnd0 =
N->getOperand(0);
244 bool SelectDupNegativeZero(
SDValue N) {
245 switch(
N->getOpcode()) {
249 return Const && Const->isZero() && Const->isNegative();
256 template<MVT::SimpleValueType VT>
258 return SelectSVEAddSubImm(
N, VT, Imm, Shift);
261 template <MVT::SimpleValueType VT,
bool Negate>
263 return SelectSVEAddSubSSatImm(
N, VT, Imm, Shift, Negate);
266 template <MVT::SimpleValueType VT>
268 return SelectSVECpyDupImm(
N, VT, Imm, Shift);
271 template <MVT::SimpleValueType VT,
bool Invert = false>
273 return SelectSVELogicalImm(
N, VT, Imm, Invert);
276 template <MVT::SimpleValueType VT>
278 return SelectSVEArithImm(
N, VT, Imm);
281 template <
unsigned Low,
unsigned High,
bool AllowSaturation = false>
283 return SelectSVEShiftImm(
N,
Low,
High, AllowSaturation, Imm);
290 EVT EltVT =
N->getValueType(0).getVectorElementType();
291 return SelectSVEShiftImm(
N->getOperand(0), 1,
297 template<
signed Min,
signed Max,
signed Scale,
bool Shift>
299 if (!isa<ConstantSDNode>(
N))
302 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
304 MulImm = 1LL << MulImm;
306 if ((MulImm % std::abs(Scale)) != 0)
310 if ((MulImm >= Min) && (MulImm <= Max)) {
311 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
318 template <
signed Max,
signed Scale>
320 if (!isa<ConstantSDNode>(
N))
323 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
325 if (MulImm >= 0 && MulImm <= Max) {
327 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
334 template <
unsigned BaseReg,
unsigned Max>
336 if (
auto *CI = dyn_cast<ConstantSDNode>(
N)) {
342 Imm = CurDAG->getRegister(BaseReg +
C, MVT::Other);
365 const unsigned SubRegs[]);
367 void SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
369 bool tryIndexedLoad(
SDNode *
N);
371 bool trySelectStackSlotTagP(
SDNode *
N);
374 void SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
376 void SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
378 void SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
379 void SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
380 void SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
381 unsigned Opc_rr,
unsigned Opc_ri,
382 bool IsIntr =
false);
383 void SelectContiguousMultiVectorLoad(
SDNode *
N,
unsigned NumVecs,
384 unsigned Scale,
unsigned Opc_ri,
386 void SelectDestructiveMultiIntrinsic(
SDNode *
N,
unsigned NumVecs,
387 bool IsZmMulti,
unsigned Opcode,
388 bool HasPred =
false);
389 void SelectPExtPair(
SDNode *
N,
unsigned Opc);
390 void SelectWhilePair(
SDNode *
N,
unsigned Opc);
391 void SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
392 void SelectClamp(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
393 void SelectUnaryMultiIntrinsic(
SDNode *
N,
unsigned NumOutVecs,
394 bool IsTupleInput,
unsigned Opc);
395 void SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
397 template <
unsigned MaxIdx,
unsigned Scale>
398 void SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
unsigned BaseReg,
403 template <
int64_t Min,
int64_t Max>
407 template <
unsigned Scale>
409 return SelectSVERegRegAddrMode(
N, Scale,
Base,
Offset);
412 void SelectMultiVectorLuti(
SDNode *
Node,
unsigned NumOutVecs,
unsigned Opc,
415 template <
unsigned MaxIdx,
unsigned Scale>
420 void SelectStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
421 void SelectPostStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
422 void SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
423 void SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
424 void SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
425 unsigned Opc_rr,
unsigned Opc_ri);
426 std::tuple<unsigned, SDValue, SDValue>
427 findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
unsigned Opc_ri,
431 bool tryBitfieldExtractOp(
SDNode *
N);
432 bool tryBitfieldExtractOpFromSExt(
SDNode *
N);
433 bool tryBitfieldInsertOp(
SDNode *
N);
434 bool tryBitfieldInsertInZeroOp(
SDNode *
N);
435 bool tryShiftAmountMod(
SDNode *
N);
437 bool tryReadRegister(
SDNode *
N);
438 bool tryWriteRegister(
SDNode *
N);
440 bool trySelectCastFixedLengthToScalableVector(
SDNode *
N);
441 bool trySelectCastScalableToFixedLengthVector(
SDNode *
N);
446#include "AArch64GenDAGISel.inc"
454 return SelectAddrModeIndexedBitWidth(
N,
true, 7,
Size,
Base, OffImm);
456 bool SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
unsigned BW,
469 bool isWorthFoldingALU(
SDValue V,
bool LSL =
false)
const;
470 bool isWorthFoldingAddr(
SDValue V,
unsigned Size)
const;
471 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
474 template<
unsigned RegW
idth>
476 return SelectCVTFixedPosOperand(
N, FixedPos, RegWidth);
479 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
481 template<
unsigned RegW
idth>
483 return SelectCVTFixedPosRecipOperand(
N, FixedPos, RegWidth);
489 bool SelectCMP_SWAP(
SDNode *
N);
499 bool AllowSaturation,
SDValue &Imm);
507 bool SelectAllActivePredicate(
SDValue N);
512char AArch64DAGToDAGISel::ID = 0;
520 Imm =
C->getZExtValue();
537 return N->getOpcode() == Opc &&
548 return Imm == ImmExpected;
552bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
554 std::vector<SDValue> &OutOps) {
555 switch(ConstraintID) {
558 case InlineAsm::ConstraintCode::m:
559 case InlineAsm::ConstraintCode::o:
560 case InlineAsm::ConstraintCode::Q:
566 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(), dl, MVT::i64);
568 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
569 dl,
Op.getValueType(),
571 OutOps.push_back(NewOp);
587 if (!isa<ConstantSDNode>(
N.getNode()))
590 uint64_t Immed =
N.getNode()->getAsZExtVal();
593 if (Immed >> 12 == 0) {
595 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
603 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
604 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
617 if (!isa<ConstantSDNode>(
N.getNode()))
621 uint64_t Immed =
N.getNode()->getAsZExtVal();
629 if (
N.getValueType() == MVT::i32)
632 Immed = ~Immed + 1ULL;
633 if (Immed & 0xFFFFFFFFFF000000ULL)
636 Immed &= 0xFFFFFFULL;
637 return SelectArithImmed(CurDAG->getConstant(Immed,
SDLoc(
N), MVT::i32), Val,
644 switch (
N.getOpcode()) {
663 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
666 unsigned ShiftVal = CSD->getZExtValue();
675 if (!isa<MemSDNode>(*UI))
677 if (!isa<MemSDNode>(*UII))
684bool AArch64DAGToDAGISel::isWorthFoldingAddr(
SDValue V,
unsigned Size)
const {
687 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
692 if (Subtarget->hasAddrLSLSlow14() && (
Size == 2 ||
Size == 16))
714bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(
SDValue N,
SDValue &Reg,
716 EVT VT =
N.getValueType();
717 if (VT != MVT::i32 && VT != MVT::i64)
720 if (
N->getOpcode() !=
ISD::AND || !
N->hasOneUse())
726 unsigned LHSOpcode =
LHS->getOpcode();
740 unsigned LowZBits, MaskLen;
744 unsigned BitWidth =
N.getValueSizeInBits();
751 if (LowZBits <= ShiftAmtC || (
BitWidth != LowZBits + MaskLen))
754 NewShiftC = LowZBits - ShiftAmtC;
755 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
761 NewShiftC = LowZBits + ShiftAmtC;
774 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
776 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
780 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC,
DL, VT);
782 Reg =
SDValue(CurDAG->getMachineNode(NewShiftOp,
DL, VT,
LHS->getOperand(0),
783 NewShiftAmt, BitWidthMinus1),
786 Shift = CurDAG->getTargetConstant(ShVal,
DL, MVT::i32);
798 SrcVT = cast<VTSDNode>(
N.getOperand(1))->getVT();
800 SrcVT =
N.getOperand(0).getValueType();
802 if (!IsLoadStore && SrcVT == MVT::i8)
804 else if (!IsLoadStore && SrcVT == MVT::i16)
806 else if (SrcVT == MVT::i32)
808 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
813 EVT SrcVT =
N.getOperand(0).getValueType();
814 if (!IsLoadStore && SrcVT == MVT::i8)
816 else if (!IsLoadStore && SrcVT == MVT::i16)
818 else if (SrcVT == MVT::i32)
820 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
848bool AArch64DAGToDAGISel::isWorthFoldingALU(
SDValue V,
bool LSL)
const {
851 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
856 if (LSL && Subtarget->hasALULSLFast() &&
V.getOpcode() ==
ISD::SHL &&
857 V.getConstantOperandVal(1) <= 4 &&
870bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
872 if (SelectShiftedRegisterFromAnd(
N, Reg, Shift))
882 unsigned BitSize =
N.getValueSizeInBits();
883 unsigned Val =
RHS->getZExtValue() & (BitSize - 1);
886 Reg =
N.getOperand(0);
887 Shift = CurDAG->getTargetConstant(ShVal,
SDLoc(
N), MVT::i32);
888 return isWorthFoldingALU(
N,
true);
899 if (
N.getValueType() == MVT::i32)
907template<
signed Low,
signed High,
signed Scale>
909 if (!isa<ConstantSDNode>(
N))
912 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
913 if ((MulImm % std::abs(Scale)) == 0) {
914 int64_t RDVLImm = MulImm / Scale;
915 if ((RDVLImm >=
Low) && (RDVLImm <=
High)) {
916 Imm = CurDAG->getTargetConstant(RDVLImm,
SDLoc(
N), MVT::i32);
926bool AArch64DAGToDAGISel::SelectArithExtendedRegister(
SDValue N,
SDValue &Reg,
928 unsigned ShiftVal = 0;
943 Reg =
N.getOperand(0).getOperand(0);
949 Reg =
N.getOperand(0);
954 unsigned Opc =
N.getOpcode();
955 return Opc !=
ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
972 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
974 return isWorthFoldingALU(
N);
979bool AArch64DAGToDAGISel::SelectArithUXTXRegister(
SDValue N,
SDValue &Reg,
981 unsigned ShiftVal = 0;
995 Reg =
N.getOperand(0);
996 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
998 return isWorthFoldingALU(
N);
1007 for (
auto *
Use :
N->uses()) {
1034bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
1035 unsigned BW,
unsigned Size,
1042 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1044 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1050 if (CurDAG->isBaseWithConstantOffset(
N)) {
1051 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1053 int64_t RHSC =
RHS->getSExtValue();
1055 int64_t
Range = 0x1LL << (BW - 1);
1057 if ((RHSC & (
Size - 1)) == 0 && RHSC >= -(
Range << Scale) &&
1058 RHSC < (Range << Scale)) {
1059 Base =
N.getOperand(0);
1061 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1064 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1073 if ((RHSC & (
Size - 1)) == 0 && RHSC < (Range << Scale)) {
1074 Base =
N.getOperand(0);
1076 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1079 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1090 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1097bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
1103 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1105 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1111 dyn_cast<GlobalAddressSDNode>(
N.getOperand(1).getNode());
1112 Base =
N.getOperand(0);
1113 OffImm =
N.getOperand(1);
1122 if (CurDAG->isBaseWithConstantOffset(
N)) {
1123 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1124 int64_t RHSC = (int64_t)
RHS->getZExtValue();
1127 Base =
N.getOperand(0);
1129 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1132 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1140 if (SelectAddrModeUnscaled(
N,
Size,
Base, OffImm))
1148 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1157bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
1160 if (!CurDAG->isBaseWithConstantOffset(
N))
1162 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1163 int64_t RHSC =
RHS->getSExtValue();
1164 if (RHSC >= -256 && RHSC < 256) {
1165 Base =
N.getOperand(0);
1167 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1169 Base = CurDAG->getTargetFrameIndex(
1172 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i64);
1182 CurDAG->
getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1189bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
1209 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1215 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1218 return isWorthFoldingAddr(
N,
Size);
1221bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
1233 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
1241 if (!isa<MemSDNode>(*UI))
1246 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1249 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1250 SelectExtendedSHL(RHS,
Size,
true,
Offset, SignExtend)) {
1252 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1257 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1258 SelectExtendedSHL(LHS,
Size,
true,
Offset, SignExtend)) {
1260 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1265 DoShift = CurDAG->getTargetConstant(
false, dl, MVT::i32);
1269 if (IsExtendedRegisterWorthFolding &&
1276 if (isWorthFoldingAddr(LHS,
Size))
1281 if (IsExtendedRegisterWorthFolding &&
1288 if (isWorthFoldingAddr(RHS,
Size))
1300 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1303 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1305 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1306 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1310bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
1325 if (!isa<MemSDNode>(*UI))
1340 if (isa<ConstantSDNode>(RHS)) {
1341 int64_t ImmOff = (int64_t)
RHS->getAsZExtVal();
1351 CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
1354 N = CurDAG->getNode(
ISD::ADD,
DL, MVT::i64, LHS, MOVIV);
1358 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1361 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1362 SelectExtendedSHL(RHS,
Size,
false,
Offset, SignExtend)) {
1364 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1369 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1370 SelectExtendedSHL(LHS,
Size,
false,
Offset, SignExtend)) {
1372 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1379 SignExtend = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1380 DoShift = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1386 static const unsigned RegClassIDs[] = {
1387 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1388 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1389 AArch64::dsub2, AArch64::dsub3};
1395 static const unsigned RegClassIDs[] = {
1396 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1397 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1398 AArch64::qsub2, AArch64::qsub3};
1404 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1405 AArch64::ZPR3RegClassID,
1406 AArch64::ZPR4RegClassID};
1407 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1408 AArch64::zsub2, AArch64::zsub3};
1418 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1419 AArch64::ZPR4Mul4RegClassID};
1420 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1421 AArch64::zsub2, AArch64::zsub3};
1426 const unsigned RegClassIDs[],
1427 const unsigned SubRegs[]) {
1430 if (Regs.
size() == 1)
1441 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2],
DL, MVT::i32));
1444 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1446 Ops.
push_back(CurDAG->getTargetConstant(SubRegs[i],
DL, MVT::i32));
1450 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped, Ops);
1454void AArch64DAGToDAGISel::SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1457 EVT VT =
N->getValueType(0);
1459 unsigned ExtOff = isExt;
1462 unsigned Vec0Off = ExtOff + 1;
1464 N->op_begin() + Vec0Off + NumVecs);
1471 Ops.
push_back(
N->getOperand(NumVecs + ExtOff + 1));
1472 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1475bool AArch64DAGToDAGISel::tryIndexedLoad(
SDNode *
N) {
1477 if (
LD->isUnindexed())
1479 EVT VT =
LD->getMemoryVT();
1480 EVT DstVT =
N->getValueType(0);
1487 unsigned Opcode = 0;
1490 bool InsertTo64 =
false;
1492 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1493 else if (VT == MVT::i32) {
1495 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1497 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1499 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1505 }
else if (VT == MVT::i16) {
1507 if (DstVT == MVT::i64)
1508 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1510 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1512 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1513 InsertTo64 = DstVT == MVT::i64;
1518 }
else if (VT == MVT::i8) {
1520 if (DstVT == MVT::i64)
1521 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1523 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1525 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1526 InsertTo64 = DstVT == MVT::i64;
1531 }
else if (VT == MVT::f16) {
1532 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1533 }
else if (VT == MVT::bf16) {
1534 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1535 }
else if (VT == MVT::f32) {
1536 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1538 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1540 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1548 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1550 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1555 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {
MemOp});
1560 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1562 SDValue(CurDAG->getMachineNode(
1563 AArch64::SUBREG_TO_REG, dl, MVT::i64,
1564 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1569 ReplaceUses(
SDValue(
N, 0), LoadedVal);
1572 CurDAG->RemoveDeadNode(
N);
1576void AArch64DAGToDAGISel::SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1577 unsigned SubRegIdx) {
1579 EVT VT =
N->getValueType(0);
1585 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1587 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1589 for (
unsigned i = 0; i < NumVecs; ++i)
1591 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1597 if (
auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(
N)) {
1599 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
1602 CurDAG->RemoveDeadNode(
N);
1605void AArch64DAGToDAGISel::SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
1606 unsigned Opc,
unsigned SubRegIdx) {
1608 EVT VT =
N->getValueType(0);
1615 const EVT ResTys[] = {MVT::i64,
1616 MVT::Untyped, MVT::Other};
1618 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1626 ReplaceUses(
SDValue(
N, 0), SuperReg);
1628 for (
unsigned i = 0; i < NumVecs; ++i)
1630 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1634 CurDAG->RemoveDeadNode(
N);
1640std::tuple<unsigned, SDValue, SDValue>
1641AArch64DAGToDAGISel::findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
1647 SDValue NewOffset = OldOffset;
1649 const bool IsRegImm = SelectAddrModeIndexedSVE<-8, 7>(
1650 N, OldBase, NewBase, NewOffset);
1654 const bool IsRegReg =
1655 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
1658 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
1671template <SelectTypeKind Kind>
1683 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
1688 if (EltVT != MVT::i1)
1692 if (EltVT == MVT::bf16)
1694 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
1724void AArch64DAGToDAGISel::SelectPExtPair(
SDNode *
N,
unsigned Opc) {
1727 if (
Imm->getZExtValue() > 1)
1731 EVT VT =
N->getValueType(0);
1732 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1733 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1736 for (
unsigned I = 0;
I < 2; ++
I)
1737 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1738 AArch64::psub0 +
I,
DL, VT, SuperReg));
1740 CurDAG->RemoveDeadNode(
N);
1743void AArch64DAGToDAGISel::SelectWhilePair(
SDNode *
N,
unsigned Opc) {
1745 EVT VT =
N->getValueType(0);
1747 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1749 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1752 for (
unsigned I = 0;
I < 2; ++
I)
1753 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1754 AArch64::psub0 +
I,
DL, VT, SuperReg));
1756 CurDAG->RemoveDeadNode(
N);
1759void AArch64DAGToDAGISel::SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
1761 EVT VT =
N->getValueType(0);
1763 SDValue Ops = createZTuple(Regs);
1767 for (
unsigned i = 0; i < NumVecs; ++i)
1768 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1769 AArch64::zsub0 + i,
DL, VT, SuperReg));
1771 CurDAG->RemoveDeadNode(
N);
1774void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(
SDNode *
N,
1779 assert(Opcode != 0 &&
"Unexpected opcode");
1782 EVT VT =
N->getValueType(0);
1783 unsigned FirstVecIdx = HasPred ? 2 : 1;
1785 auto GetMultiVecOperand = [=](
unsigned StartIdx) {
1787 N->op_begin() + StartIdx + NumVecs);
1788 return createZMulTuple(Regs);
1791 SDValue Zdn = GetMultiVecOperand(FirstVecIdx);
1795 Zm = GetMultiVecOperand(NumVecs + FirstVecIdx);
1797 Zm =
N->getOperand(NumVecs + FirstVecIdx);
1801 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped,
1802 N->getOperand(1), Zdn, Zm);
1804 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped, Zdn, Zm);
1806 for (
unsigned i = 0; i < NumVecs; ++i)
1807 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1808 AArch64::zsub0 + i,
DL, VT, SuperReg));
1810 CurDAG->RemoveDeadNode(
N);
1813void AArch64DAGToDAGISel::SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
1814 unsigned Scale,
unsigned Opc_ri,
1815 unsigned Opc_rr,
bool IsIntr) {
1816 assert(Scale < 5 &&
"Invalid scaling value.");
1818 EVT VT =
N->getValueType(0);
1824 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
1825 N, Opc_rr, Opc_ri,
N->getOperand(IsIntr ? 3 : 2),
1826 CurDAG->getTargetConstant(0,
DL, MVT::i64), Scale);
1828 SDValue Ops[] = {
N->getOperand(IsIntr ? 2 : 1),
1832 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1834 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1836 for (
unsigned i = 0; i < NumVecs; ++i)
1837 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1838 AArch64::zsub0 + i,
DL, VT, SuperReg));
1841 unsigned ChainIdx = NumVecs;
1843 CurDAG->RemoveDeadNode(
N);
1846void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(
SDNode *
N,
1851 assert(Scale < 4 &&
"Invalid scaling value.");
1853 EVT VT =
N->getValueType(0);
1861 findAddrModeSVELoadStore(
N, Opc_rr, Opc_ri,
Base,
Offset, Scale);
1867 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1869 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1871 for (
unsigned i = 0; i < NumVecs; ++i)
1872 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1873 AArch64::zsub0 + i,
DL, VT, SuperReg));
1876 unsigned ChainIdx = NumVecs;
1878 CurDAG->RemoveDeadNode(
N);
1881void AArch64DAGToDAGISel::SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
1883 if (
N->getValueType(0) != MVT::nxv4f32)
1885 SelectUnaryMultiIntrinsic(
N, NumVecs,
true, Opcode);
1888void AArch64DAGToDAGISel::SelectMultiVectorLuti(
SDNode *
Node,
1889 unsigned NumOutVecs,
1892 if (
Imm->getZExtValue() > MaxImm)
1896 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
1900 EVT VT =
Node->getValueType(0);
1903 CurDAG->getMachineNode(Opc,
DL, {MVT::Untyped, MVT::Other}, Ops);
1906 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
1907 ReplaceUses(
SDValue(
Node,
I), CurDAG->getTargetExtractSubreg(
1908 AArch64::zsub0 +
I,
DL, VT, SuperReg));
1911 unsigned ChainIdx = NumOutVecs;
1913 CurDAG->RemoveDeadNode(
Node);
1916void AArch64DAGToDAGISel::SelectClamp(
SDNode *
N,
unsigned NumVecs,
1919 EVT VT =
N->getValueType(0);
1922 SDValue Zd = createZMulTuple(Regs);
1923 SDValue Zn =
N->getOperand(1 + NumVecs);
1924 SDValue Zm =
N->getOperand(2 + NumVecs);
1930 for (
unsigned i = 0; i < NumVecs; ++i)
1931 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1932 AArch64::zsub0 + i,
DL, VT, SuperReg));
1934 CurDAG->RemoveDeadNode(
N);
1964template <
unsigned MaxIdx,
unsigned Scale>
1965void AArch64DAGToDAGISel::SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
1966 unsigned BaseReg,
unsigned Op) {
1967 unsigned TileNum = 0;
1968 if (BaseReg != AArch64::ZA)
1969 TileNum =
N->getConstantOperandVal(2);
1975 if (BaseReg == AArch64::ZA)
1976 SliceBase =
N->getOperand(2);
1978 SliceBase =
N->getOperand(3);
1980 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
1986 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
1988 EVT VT =
N->getValueType(0);
1989 for (
unsigned I = 0;
I < NumVecs; ++
I)
1991 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
1994 unsigned ChainIdx = NumVecs;
1996 CurDAG->RemoveDeadNode(
N);
1999void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(
SDNode *
N,
2000 unsigned NumOutVecs,
2004 EVT VT =
N->getValueType(0);
2005 unsigned NumInVecs =
N->getNumOperands() - 1;
2009 assert((NumInVecs == 2 || NumInVecs == 4) &&
2010 "Don't know how to handle multi-register input!");
2012 N->op_begin() + 1 + NumInVecs);
2016 for (
unsigned I = 0;
I < NumInVecs;
I++)
2020 SDNode *Res = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
2023 for (
unsigned I = 0;
I < NumOutVecs;
I++)
2024 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2025 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2026 CurDAG->RemoveDeadNode(
N);
2029void AArch64DAGToDAGISel::SelectStore(
SDNode *
N,
unsigned NumVecs,
2032 EVT VT =
N->getOperand(2)->getValueType(0);
2039 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
N->getOperand(0)};
2040 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2044 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2049void AArch64DAGToDAGISel::SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
2050 unsigned Scale,
unsigned Opc_rr,
2056 SDValue RegSeq = createZTuple(Regs);
2061 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
2062 N, Opc_rr, Opc_ri,
N->getOperand(NumVecs + 3),
2063 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2065 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
2069 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2081 if (
auto FINode = dyn_cast<FrameIndexSDNode>(
N)) {
2082 int FI = FINode->getIndex();
2084 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
2091void AArch64DAGToDAGISel::SelectPostStore(
SDNode *
N,
unsigned NumVecs,
2094 EVT VT =
N->getOperand(2)->getValueType(0);
2095 const EVT ResTys[] = {MVT::i64,
2104 N->getOperand(NumVecs + 1),
2105 N->getOperand(NumVecs + 2),
2107 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2147void AArch64DAGToDAGISel::SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
2150 EVT VT =
N->getValueType(0);
2158 WidenVector(*CurDAG));
2162 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2164 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2166 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2167 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2168 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2172 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2173 AArch64::qsub2, AArch64::qsub3 };
2174 for (
unsigned i = 0; i < NumVecs; ++i) {
2175 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2182 CurDAG->RemoveDeadNode(
N);
2185void AArch64DAGToDAGISel::SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
2188 EVT VT =
N->getValueType(0);
2196 WidenVector(*CurDAG));
2200 const EVT ResTys[] = {MVT::i64,
2203 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2206 CurDAG->getTargetConstant(LaneNo, dl,
2208 N->getOperand(NumVecs + 2),
2209 N->getOperand(NumVecs + 3),
2211 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2223 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2224 AArch64::qsub2, AArch64::qsub3 };
2225 for (
unsigned i = 0; i < NumVecs; ++i) {
2226 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
2236 CurDAG->RemoveDeadNode(
N);
2239void AArch64DAGToDAGISel::SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
2242 EVT VT =
N->getOperand(2)->getValueType(0);
2250 WidenVector(*CurDAG));
2254 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2256 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2257 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2258 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
2262 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2267void AArch64DAGToDAGISel::SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
2270 EVT VT =
N->getOperand(2)->getValueType(0);
2278 WidenVector(*CurDAG));
2282 const EVT ResTys[] = {MVT::i64,
2285 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2287 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2288 N->getOperand(NumVecs + 2),
2289 N->getOperand(NumVecs + 3),
2291 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2295 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2302 unsigned &LSB,
unsigned &MSB,
2303 unsigned NumberOfIgnoredLowBits,
2304 bool BiggerPattern) {
2306 "N must be a AND operation to call this function");
2308 EVT VT =
N->getValueType(0);
2313 assert((VT == MVT::i32 || VT == MVT::i64) &&
2314 "Type checking must have been done before calling this function");
2328 const SDNode *Op0 =
N->getOperand(0).getNode();
2332 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
2335 if (AndImm & (AndImm + 1))
2338 bool ClampMSB =
false;
2358 ClampMSB = (VT == MVT::i32);
2359 }
else if (BiggerPattern) {
2365 Opd0 =
N->getOperand(0);
2371 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.
getSizeInBits())) {
2374 <<
": Found large shift immediate, this should not happen\n"));
2380 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(AndImm)
2381 : llvm::countr_one<uint64_t>(AndImm)) -
2388 MSB = MSB > 31 ? 31 : MSB;
2390 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2395 SDValue &Opd0,
unsigned &Immr,
2399 EVT VT =
N->getValueType(0);
2401 assert((VT == MVT::i32 || VT == MVT::i64) &&
2402 "Type checking must have been done before calling this function");
2406 Op =
Op->getOperand(0);
2407 VT =
Op->getValueType(0);
2416 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
2420 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2421 Opd0 =
Op.getOperand(0);
2423 Imms = ShiftImm + Width - 1;
2451 Opd0 =
N->getOperand(0).getOperand(0);
2461 Opc =
N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2468 unsigned &Immr,
unsigned &Imms,
2469 bool BiggerPattern) {
2471 "N must be a SHR/SRA operation to call this function");
2473 EVT VT =
N->getValueType(0);
2478 assert((VT == MVT::i32 || VT == MVT::i64) &&
2479 "Type checking must have been done before calling this function");
2489 Opd0 =
N->getOperand(0).getOperand(0);
2490 }
else if (VT == MVT::i32 &&
N->getOpcode() ==
ISD::SRL &&
2496 Opd0 =
N->getOperand(0).getOperand(0);
2499 assert(VT == MVT::i64 &&
"the promoted type should be i64");
2500 }
else if (BiggerPattern) {
2504 Opd0 =
N->getOperand(0);
2513 <<
": Found large shift immediate, this should not happen\n"));
2522 "bad amount in shift node!");
2523 int immr = SrlImm - ShlImm;
2528 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
2530 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
2534bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(
SDNode *
N) {
2537 EVT VT =
N->getValueType(0);
2538 EVT NarrowVT =
N->getOperand(0)->getValueType(0);
2539 if (VT != MVT::i64 || NarrowVT != MVT::i32)
2550 unsigned Immr = ShiftImm;
2552 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2553 CurDAG->getTargetConstant(Imms, dl, VT)};
2554 CurDAG->SelectNodeTo(
N, AArch64::SBFMXri, VT, Ops);
2559 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
2560 unsigned NumberOfIgnoredLowBits = 0,
2561 bool BiggerPattern =
false) {
2562 if (
N->getValueType(0) != MVT::i32 &&
N->getValueType(0) != MVT::i64)
2565 switch (
N->getOpcode()) {
2567 if (!
N->isMachineOpcode())
2572 NumberOfIgnoredLowBits, BiggerPattern);
2581 unsigned NOpc =
N->getMachineOpcode();
2585 case AArch64::SBFMWri:
2586 case AArch64::UBFMWri:
2587 case AArch64::SBFMXri:
2588 case AArch64::UBFMXri:
2590 Opd0 =
N->getOperand(0);
2591 Immr =
N->getConstantOperandVal(1);
2592 Imms =
N->getConstantOperandVal(2);
2599bool AArch64DAGToDAGISel::tryBitfieldExtractOp(
SDNode *
N) {
2600 unsigned Opc, Immr, Imms;
2605 EVT VT =
N->getValueType(0);
2610 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
2611 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
2612 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
2614 SDNode *
BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
2615 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
2621 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2622 CurDAG->getTargetConstant(Imms, dl, VT)};
2623 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
2632 unsigned NumberOfIgnoredHighBits,
EVT VT) {
2633 assert((VT == MVT::i32 || VT == MVT::i64) &&
2634 "i32 or i64 mask type expected!");
2640 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
2641 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
2664 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2674 APInt OpUsefulBits(UsefulBits);
2678 OpUsefulBits <<= MSB - Imm + 1;
2683 OpUsefulBits <<= Imm;
2685 OpUsefulBits <<= MSB + 1;
2688 OpUsefulBits <<= OpUsefulBits.
getBitWidth() - Imm;
2694 UsefulBits &= OpUsefulBits;
2700 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2702 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2710 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2711 APInt Mask(UsefulBits);
2712 Mask.clearAllBits();
2720 Mask.lshrInPlace(ShiftAmt);
2726 Mask.lshrInPlace(ShiftAmt);
2738 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2740 cast<const ConstantSDNode>(
Op.getOperand(3).getNode())->getZExtValue();
2742 APInt OpUsefulBits(UsefulBits);
2756 OpUsefulBits <<= Width;
2759 if (
Op.getOperand(1) == Orig) {
2761 Mask = ResultUsefulBits & OpUsefulBits;
2765 if (
Op.getOperand(0) == Orig)
2767 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2773 OpUsefulBits <<= Width;
2775 OpUsefulBits <<= LSB;
2777 if (
Op.getOperand(1) == Orig) {
2779 Mask = ResultUsefulBits & OpUsefulBits;
2780 Mask.lshrInPlace(LSB);
2783 if (
Op.getOperand(0) == Orig)
2784 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2801 case AArch64::ANDSWri:
2802 case AArch64::ANDSXri:
2803 case AArch64::ANDWri:
2804 case AArch64::ANDXri:
2808 case AArch64::UBFMWri:
2809 case AArch64::UBFMXri:
2812 case AArch64::ORRWrs:
2813 case AArch64::ORRXrs:
2818 case AArch64::BFMWri:
2819 case AArch64::BFMXri:
2822 case AArch64::STRBBui:
2823 case AArch64::STURBBi:
2829 case AArch64::STRHHui:
2830 case AArch64::STURHHi:
2843 unsigned Bitwidth =
Op.getScalarValueSizeInBits();
2845 UsefulBits =
APInt(Bitwidth, 0);
2854 UsersUsefulBits |= UsefulBitsForUse;
2859 UsefulBits &= UsersUsefulBits;
2869 EVT VT =
Op.getValueType();
2872 unsigned UBFMOpc =
BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2875 if (ShlAmount > 0) {
2878 UBFMOpc, dl, VT,
Op,
2883 assert(ShlAmount < 0 &&
"expected right shift");
2884 int ShrAmount = -ShlAmount;
2910 bool BiggerPattern,
SDValue &Src,
2911 int &DstLSB,
int &Width) {
2912 EVT VT =
Op.getValueType();
2921 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
2925 switch (
Op.getOpcode()) {
2930 NonZeroBits, Src, DstLSB, Width);
2933 NonZeroBits, Src, DstLSB, Width);
2946 EVT VT =
Op.getValueType();
2947 assert((VT == MVT::i32 || VT == MVT::i64) &&
2948 "Caller guarantees VT is one of i32 or i64");
2961 assert((~AndImm & NonZeroBits) == 0 &&
2962 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
2991 if (!BiggerPattern && !AndOp0.
hasOneUse())
3010 <<
"Found large Width in bit-field-positioning -- this indicates no "
3011 "proper combining / constant folding was performed\n");
3020 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3035 "Op.getNode() should be a SHL node to call this function");
3037 "Op.getNode() should shift ShlImm to call this function");
3044 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3068 EVT VT =
Op.getValueType();
3069 assert((VT == MVT::i32 || VT == MVT::i64) &&
3070 "Caller guarantees that type is i32 or i64");
3077 if (!BiggerPattern && !
Op.hasOneUse())
3086 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3094 assert(VT == MVT::i32 || VT == MVT::i64);
3105 EVT VT =
N->getValueType(0);
3106 if (VT != MVT::i32 && VT != MVT::i64)
3124 if (!
And.hasOneUse() ||
3134 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3141 if ((OrImm & NotKnownZero) != 0) {
3153 unsigned ImmS = Width - 1;
3159 bool IsBFI = LSB != 0;
3164 unsigned OrChunks = 0, BFIChunks = 0;
3165 for (
unsigned Shift = 0; Shift <
BitWidth; Shift += 16) {
3166 if (((OrImm >> Shift) & 0xFFFF) != 0)
3168 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3171 if (BFIChunks > OrChunks)
3177 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3185 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3194 if (!Dst.hasOneUse())
3197 EVT VT = Dst.getValueType();
3198 assert((VT == MVT::i32 || VT == MVT::i64) &&
3199 "Caller should guarantee that VT is one of i32 or i64");
3207 SDValue DstOp0 = Dst.getOperand(0);
3227 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3228 unsigned MaskWidth =
3231 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3237 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1,
DL, VT));
3238 ShiftedOperand =
SDValue(UBFMNode, 0);
3248 ShiftedOperand = Dst.getOperand(0);
3255 ShiftedOperand = Dst.getOperand(0);
3267 const bool BiggerPattern) {
3268 EVT VT =
N->getValueType(0);
3269 assert(
N->getOpcode() ==
ISD::OR &&
"Expect N to be an OR node");
3270 assert(((
N->getOperand(0) == OrOpd0 &&
N->getOperand(1) == OrOpd1) ||
3271 (
N->getOperand(1) == OrOpd0 &&
N->getOperand(0) == OrOpd1)) &&
3272 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3273 assert((VT == MVT::i32 || VT == MVT::i64) &&
3274 "Expect result type to be i32 or i64 since N is combinable to BFM");
3281 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3284 if (BiggerPattern) {
3298 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3307 assert((!BiggerPattern) &&
"BiggerPattern should be handled above");
3369 EVT VT =
N->getValueType(0);
3370 if (VT != MVT::i32 && VT != MVT::i64)
3378 unsigned NumberOfIgnoredLowBits = UsefulBits.
countr_zero();
3379 unsigned NumberOfIgnoredHighBits = UsefulBits.
countl_zero();
3399 for (
int I = 0;
I < 4; ++
I) {
3402 unsigned ImmR, ImmS;
3403 bool BiggerPattern =
I / 2;
3404 SDValue OrOpd0Val =
N->getOperand(
I % 2);
3406 SDValue OrOpd1Val =
N->getOperand((
I + 1) % 2);
3412 NumberOfIgnoredLowBits, BiggerPattern)) {
3415 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3416 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3421 Width = ImmS - ImmR + 1;
3432 Src, DstLSB, Width)) {
3440 assert((VT == MVT::i32 || VT == MVT::i64) &&
"unexpected OR operand");
3450 APInt BitsToBeInserted =
3453 if ((BitsToBeInserted & ~Known.
Zero) != 0)
3477 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3510 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3512 if (Src->hasOneUse() &&
3515 Src = Src->getOperand(0);
3525 unsigned ImmS = Width - 1;
3531 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3539bool AArch64DAGToDAGISel::tryBitfieldInsertOp(
SDNode *
N) {
3548 CurDAG->SelectNodeTo(
N, TargetOpcode::IMPLICIT_DEF,
N->getValueType(0));
3561bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(
SDNode *
N) {
3565 EVT VT =
N->getValueType(0);
3566 if (VT != MVT::i32 && VT != MVT::i64)
3572 Op0, DstLSB, Width))
3578 unsigned ImmS = Width - 1;
3581 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR,
DL, VT),
3582 CurDAG->getTargetConstant(ImmS,
DL, VT)};
3583 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3584 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3590bool AArch64DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
3591 EVT VT =
N->getValueType(0);
3594 switch (
N->getOpcode()) {
3596 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
3599 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
3602 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
3605 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
3613 if (VT == MVT::i32) {
3616 }
else if (VT == MVT::i64) {
3622 SDValue ShiftAmt =
N->getOperand(1);
3642 (Add0Imm %
Size == 0)) {
3648 if (SubVT == MVT::i32) {
3649 NegOpc = AArch64::SUBWrr;
3650 ZeroReg = AArch64::WZR;
3652 assert(SubVT == MVT::i64);
3653 NegOpc = AArch64::SUBXrr;
3654 ZeroReg = AArch64::XZR;
3657 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3659 CurDAG->getMachineNode(NegOpc,
DL, SubVT, Zero, Add1);
3660 NewShiftAmt =
SDValue(Neg, 0);
3668 if (SubVT == MVT::i32) {
3669 NotOpc = AArch64::ORNWrr;
3670 ZeroReg = AArch64::WZR;
3672 assert(SubVT == MVT::i64);
3673 NotOpc = AArch64::ORNXrr;
3674 ZeroReg = AArch64::XZR;
3677 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3679 CurDAG->getMachineNode(NotOpc,
DL, SubVT, Zero, Add1);
3680 NewShiftAmt =
SDValue(Not, 0);
3701 else if (VT == MVT::i64 && NewShiftAmt->
getValueType(0) == MVT::i32) {
3702 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32,
DL, MVT::i32);
3704 AArch64::SUBREG_TO_REG,
DL, VT,
3705 CurDAG->getTargetConstant(0,
DL, MVT::i64), NewShiftAmt,
SubReg);
3706 NewShiftAmt =
SDValue(Ext, 0);
3709 SDValue Ops[] = {
N->getOperand(0), NewShiftAmt};
3710 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3717 bool isReciprocal) {
3720 FVal = CN->getValueAPF();
3721 else if (
LoadSDNode *LN = dyn_cast<LoadSDNode>(
N)) {
3724 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
3728 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
3729 FVal = cast<ConstantFP>(CN->
getConstVal())->getValueAPF();
3752 if (!IsExact || !IntVal.isPowerOf2())
3754 unsigned FBits = IntVal.logBase2();
3758 if (FBits == 0 || FBits > RegWidth)
return false;
3764bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
3765 unsigned RegWidth) {
3770bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(
SDValue N,
3772 unsigned RegWidth) {
3782 RegString.
split(Fields,
':');
3784 if (Fields.
size() == 1)
3788 &&
"Invalid number of fields in read register string");
3791 bool AllIntFields =
true;
3795 AllIntFields &= !
Field.getAsInteger(10, IntField);
3800 "Unexpected non-integer value in special register string.");
3805 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
3806 (Ops[3] << 3) | (Ops[4]);
3813bool AArch64DAGToDAGISel::tryReadRegister(
SDNode *
N) {
3814 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3815 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3820 unsigned Opcode64Bit = AArch64::MRS;
3825 const auto *TheReg =
3827 if (TheReg && TheReg->Readable &&
3828 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3829 Imm = TheReg->Encoding;
3835 if (!ReadIs128Bit && RegString->getString() ==
"pc") {
3836 Opcode64Bit = AArch64::ADR;
3844 SDValue InChain =
N->getOperand(0);
3845 SDValue SysRegImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
3846 if (!ReadIs128Bit) {
3847 CurDAG->SelectNodeTo(
N, Opcode64Bit, MVT::i64, MVT::Other ,
3848 {SysRegImm, InChain});
3852 {MVT::Untyped , MVT::Other },
3853 {SysRegImm, InChain});
3857 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64,
DL, MVT::i64,
3859 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64,
DL, MVT::i64,
3865 ReplaceUses(
SDValue(
N, 2), OutChain);
3874bool AArch64DAGToDAGISel::tryWriteRegister(
SDNode *
N) {
3875 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
3876 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3881 if (!WriteIs128Bit) {
3887 auto trySelectPState = [&](
auto PMapper,
unsigned State) {
3889 assert(isa<ConstantSDNode>(
N->getOperand(2)) &&
3890 "Expected a constant integer expression.");
3891 unsigned Reg = PMapper->Encoding;
3892 uint64_t Immed =
N->getConstantOperandVal(2);
3893 CurDAG->SelectNodeTo(
3894 N, State, MVT::Other, CurDAG->getTargetConstant(Reg,
DL, MVT::i32),
3895 CurDAG->getTargetConstant(Immed,
DL, MVT::i16),
N->getOperand(0));
3901 if (trySelectPState(
3902 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
3903 AArch64::MSRpstateImm4))
3905 if (trySelectPState(
3906 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
3907 AArch64::MSRpstateImm1))
3917 if (TheReg && TheReg->Writeable &&
3918 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3919 Imm = TheReg->Encoding;
3927 SDValue InChain =
N->getOperand(0);
3928 if (!WriteIs128Bit) {
3929 CurDAG->SelectNodeTo(
N, AArch64::MSR, MVT::Other,
3930 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
3931 N->getOperand(2), InChain);
3935 SDNode *Pair = CurDAG->getMachineNode(
3936 TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped ,
3937 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(),
DL,
3940 CurDAG->getTargetConstant(AArch64::sube64,
DL, MVT::i32),
3942 CurDAG->getTargetConstant(AArch64::subo64,
DL, MVT::i32)});
3944 CurDAG->SelectNodeTo(
N, AArch64::MSRR, MVT::Other,
3945 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
3953bool AArch64DAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
3955 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
3958 if (Subtarget->hasLSE())
return false;
3960 if (MemTy == MVT::i8)
3961 Opcode = AArch64::CMP_SWAP_8;
3962 else if (MemTy == MVT::i16)
3963 Opcode = AArch64::CMP_SWAP_16;
3964 else if (MemTy == MVT::i32)
3965 Opcode = AArch64::CMP_SWAP_32;
3966 else if (MemTy == MVT::i64)
3967 Opcode = AArch64::CMP_SWAP_64;
3971 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
3972 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3974 SDNode *CmpSwap = CurDAG->getMachineNode(
3976 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
3979 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
3983 CurDAG->RemoveDeadNode(
N);
3990 if (!isa<ConstantSDNode>(
N))
4002 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4003 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4010 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4011 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4015 if (Val <= 65280 && Val % 256 == 0) {
4016 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4017 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4028bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(
SDValue N,
MVT VT,
4031 if (!isa<ConstantSDNode>(
N))
4035 int64_t Val = cast<ConstantSDNode>(
N)
4052 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4053 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4060 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4061 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4065 if (Val <= 65280 && Val % 256 == 0) {
4066 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4067 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4080 if (!isa<ConstantSDNode>(
N))
4084 int64_t Val = cast<ConstantSDNode>(
N)
4092 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4093 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4099 if (Val >= -128 && Val <= 127) {
4100 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4101 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4105 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) {
4106 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4107 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF,
DL, MVT::i32);
4118bool AArch64DAGToDAGISel::SelectSVESignedArithImm(
SDValue N,
SDValue &Imm) {
4119 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4120 int64_t ImmVal = CNode->getSExtValue();
4122 if (ImmVal >= -128 && ImmVal < 128) {
4123 Imm = CurDAG->getTargetConstant(ImmVal,
DL, MVT::i32);
4131 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4132 uint64_t ImmVal = CNode->getZExtValue();
4142 ImmVal &= 0xFFFFFFFF;
4151 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4160 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4161 uint64_t ImmVal = CNode->getZExtValue();
4171 ImmVal |= ImmVal << 8;
4172 ImmVal |= ImmVal << 16;
4173 ImmVal |= ImmVal << 32;
4177 ImmVal |= ImmVal << 16;
4178 ImmVal |= ImmVal << 32;
4181 ImmVal &= 0xFFFFFFFF;
4182 ImmVal |= ImmVal << 32;
4192 Imm = CurDAG->getTargetConstant(encoding,
DL, MVT::i64);
4207 if (
auto *CN = dyn_cast<ConstantSDNode>(
N)) {
4208 uint64_t ImmVal = CN->getZExtValue();
4215 if (ImmVal >
High) {
4216 if (!AllowSaturation)
4221 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4228bool AArch64DAGToDAGISel::trySelectStackSlotTagP(
SDNode *
N) {
4232 if (!(isa<FrameIndexSDNode>(
N->getOperand(1)))) {
4244 int FI = cast<FrameIndexSDNode>(
N->getOperand(1))->getIndex();
4245 SDValue FiOp = CurDAG->getTargetFrameIndex(
4247 int TagOffset =
N->getConstantOperandVal(3);
4249 SDNode *Out = CurDAG->getMachineNode(
4250 AArch64::TAGPstack,
DL, MVT::i64,
4251 {FiOp, CurDAG->getTargetConstant(0,
DL, MVT::i64),
N->getOperand(2),
4252 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4253 ReplaceNode(
N, Out);
4257void AArch64DAGToDAGISel::SelectTagP(
SDNode *
N) {
4258 assert(isa<ConstantSDNode>(
N->getOperand(3)) &&
4259 "llvm.aarch64.tagp third argument must be an immediate");
4260 if (trySelectStackSlotTagP(
N))
4267 int TagOffset =
N->getConstantOperandVal(3);
4268 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP,
DL, MVT::i64,
4269 {
N->getOperand(1),
N->getOperand(2)});
4270 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr,
DL, MVT::i64,
4271 {
SDValue(N1, 0),
N->getOperand(2)});
4272 SDNode *N3 = CurDAG->getMachineNode(
4273 AArch64::ADDG,
DL, MVT::i64,
4274 {
SDValue(N2, 0), CurDAG->getTargetConstant(0,
DL, MVT::i64),
4275 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4279bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(
SDNode *
N) {
4283 if (
N->getConstantOperandVal(2) != 0)
4285 if (!
N->getOperand(0).isUndef())
4289 EVT VT =
N->getValueType(0);
4290 EVT InVT =
N->getOperand(1).getValueType();
4301 "Expected to insert into a packed scalable vector!");
4304 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4305 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4306 N->getOperand(1), RC));
4310bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(
SDNode *
N) {
4314 if (
N->getConstantOperandVal(1) != 0)
4318 EVT VT =
N->getValueType(0);
4319 EVT InVT =
N->getOperand(0).getValueType();
4330 "Expected to extract from a packed scalable vector!");
4333 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4334 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4335 N->getOperand(0), RC));
4339bool AArch64DAGToDAGISel::trySelectXAR(
SDNode *
N) {
4344 EVT VT =
N->getValueType(0);
4364 if (!TLI->isAllActivePredicate(*CurDAG, N0.
getOperand(0)) ||
4365 !TLI->isAllActivePredicate(*CurDAG, N1.
getOperand(0)))
4372 APInt ShlAmt, ShrAmt;
4385 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
4386 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
4387 AArch64::XAR_ZZZI_D})) {
4388 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
4394 if (!Subtarget->hasSHA3())
4416 if (ShAmt + HsAmt != 64)
4420 CurDAG->SelectNodeTo(
N, AArch64::XAR, N0.
getValueType(), Ops);
4425void AArch64DAGToDAGISel::Select(
SDNode *
Node) {
4427 if (
Node->isMachineOpcode()) {
4429 Node->setNodeId(-1);
4434 EVT VT =
Node->getValueType(0);
4436 switch (
Node->getOpcode()) {
4441 if (SelectCMP_SWAP(
Node))
4447 if (tryReadRegister(
Node))
4453 if (tryWriteRegister(
Node))
4460 if (tryIndexedLoad(
Node))
4469 if (tryBitfieldExtractOp(
Node))
4471 if (tryBitfieldInsertInZeroOp(
Node))
4476 if (tryShiftAmountMod(
Node))
4481 if (tryBitfieldExtractOpFromSExt(
Node))
4486 if (tryBitfieldInsertOp(
Node))
4488 if (trySelectXAR(
Node))
4493 if (trySelectCastScalableToFixedLengthVector(
Node))
4499 if (trySelectCastFixedLengthToScalableVector(
Node))
4508 if (ConstNode->
isZero()) {
4509 if (VT == MVT::i32) {
4511 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::WZR, MVT::i32);
4512 ReplaceNode(
Node,
New.getNode());
4514 }
else if (VT == MVT::i64) {
4516 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::XZR, MVT::i64);
4517 ReplaceNode(
Node,
New.getNode());
4526 int FI = cast<FrameIndexSDNode>(
Node)->getIndex();
4529 SDValue TFI = CurDAG->getTargetFrameIndex(
4532 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0,
DL, MVT::i32),
4533 CurDAG->getTargetConstant(Shifter,
DL, MVT::i32) };
4534 CurDAG->SelectNodeTo(
Node, AArch64::ADDXri, MVT::i64, Ops);
4538 unsigned IntNo =
Node->getConstantOperandVal(1);
4542 case Intrinsic::aarch64_ldaxp:
4543 case Intrinsic::aarch64_ldxp: {
4545 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
4550 SDNode *Ld = CurDAG->getMachineNode(
Op,
DL, MVT::i64, MVT::i64,
4551 MVT::Other, MemAddr, Chain);
4555 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4556 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4557 ReplaceNode(
Node, Ld);
4560 case Intrinsic::aarch64_stlxp:
4561 case Intrinsic::aarch64_stxp: {
4563 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
4571 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
4573 SDNode *St = CurDAG->getMachineNode(
Op,
DL, MVT::i32, MVT::Other, Ops);
4576 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4577 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4579 ReplaceNode(
Node, St);
4582 case Intrinsic::aarch64_neon_ld1x2:
4583 if (VT == MVT::v8i8) {
4584 SelectLoad(
Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
4586 }
else if (VT == MVT::v16i8) {
4587 SelectLoad(
Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
4589 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4590 SelectLoad(
Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
4592 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4593 SelectLoad(
Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
4595 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4596 SelectLoad(
Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
4598 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4599 SelectLoad(
Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
4601 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4602 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4604 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4605 SelectLoad(
Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
4609 case Intrinsic::aarch64_neon_ld1x3:
4610 if (VT == MVT::v8i8) {
4611 SelectLoad(
Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
4613 }
else if (VT == MVT::v16i8) {
4614 SelectLoad(
Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
4616 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4617 SelectLoad(
Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
4619 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4620 SelectLoad(
Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
4622 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4623 SelectLoad(
Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
4625 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4626 SelectLoad(
Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
4628 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4629 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4631 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4632 SelectLoad(
Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
4636 case Intrinsic::aarch64_neon_ld1x4:
4637 if (VT == MVT::v8i8) {
4638 SelectLoad(
Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
4640 }
else if (VT == MVT::v16i8) {
4641 SelectLoad(
Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
4643 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4644 SelectLoad(
Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
4646 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4647 SelectLoad(
Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
4649 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4650 SelectLoad(
Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
4652 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4653 SelectLoad(
Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
4655 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4656 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4658 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4659 SelectLoad(
Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
4663 case Intrinsic::aarch64_neon_ld2:
4664 if (VT == MVT::v8i8) {
4665 SelectLoad(
Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
4667 }
else if (VT == MVT::v16i8) {
4668 SelectLoad(
Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
4670 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4671 SelectLoad(
Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
4673 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4674 SelectLoad(
Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
4676 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4677 SelectLoad(
Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
4679 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4680 SelectLoad(
Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
4682 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4683 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4685 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4686 SelectLoad(
Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
4690 case Intrinsic::aarch64_neon_ld3:
4691 if (VT == MVT::v8i8) {
4692 SelectLoad(
Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
4694 }
else if (VT == MVT::v16i8) {
4695 SelectLoad(
Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
4697 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4698 SelectLoad(
Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
4700 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4701 SelectLoad(
Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
4703 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4704 SelectLoad(
Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
4706 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4707 SelectLoad(
Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
4709 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4710 SelectLoad(
Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
4712 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4713 SelectLoad(
Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
4717 case Intrinsic::aarch64_neon_ld4:
4718 if (VT == MVT::v8i8) {
4719 SelectLoad(
Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
4721 }
else if (VT == MVT::v16i8) {
4722 SelectLoad(
Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
4724 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4725 SelectLoad(
Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
4727 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4728 SelectLoad(
Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
4730 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4731 SelectLoad(
Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
4733 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4734 SelectLoad(
Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
4736 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4737 SelectLoad(
Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
4739 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4740 SelectLoad(
Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
4744 case Intrinsic::aarch64_neon_ld2r:
4745 if (VT == MVT::v8i8) {
4746 SelectLoad(
Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
4748 }
else if (VT == MVT::v16i8) {
4749 SelectLoad(
Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
4751 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4752 SelectLoad(
Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
4754 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4755 SelectLoad(
Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
4757 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4758 SelectLoad(
Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
4760 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4761 SelectLoad(
Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
4763 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4764 SelectLoad(
Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
4766 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4767 SelectLoad(
Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
4771 case Intrinsic::aarch64_neon_ld3r:
4772 if (VT == MVT::v8i8) {
4773 SelectLoad(
Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
4775 }
else if (VT == MVT::v16i8) {
4776 SelectLoad(
Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
4778 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4779 SelectLoad(
Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
4781 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4782 SelectLoad(
Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
4784 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4785 SelectLoad(
Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
4787 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4788 SelectLoad(
Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
4790 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4791 SelectLoad(
Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
4793 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4794 SelectLoad(
Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
4798 case Intrinsic::aarch64_neon_ld4r:
4799 if (VT == MVT::v8i8) {
4800 SelectLoad(
Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
4802 }
else if (VT == MVT::v16i8) {
4803 SelectLoad(
Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
4805 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4806 SelectLoad(
Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
4808 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4809 SelectLoad(
Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
4811 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4812 SelectLoad(
Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
4814 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4815 SelectLoad(
Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
4817 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4818 SelectLoad(
Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
4820 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4821 SelectLoad(
Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
4825 case Intrinsic::aarch64_neon_ld2lane:
4826 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4827 SelectLoadLane(
Node, 2, AArch64::LD2i8);
4829 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4830 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4831 SelectLoadLane(
Node, 2, AArch64::LD2i16);
4833 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4835 SelectLoadLane(
Node, 2, AArch64::LD2i32);
4837 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4839 SelectLoadLane(
Node, 2, AArch64::LD2i64);
4843 case Intrinsic::aarch64_neon_ld3lane:
4844 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4845 SelectLoadLane(
Node, 3, AArch64::LD3i8);
4847 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4848 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4849 SelectLoadLane(
Node, 3, AArch64::LD3i16);
4851 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4853 SelectLoadLane(
Node, 3, AArch64::LD3i32);
4855 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4857 SelectLoadLane(
Node, 3, AArch64::LD3i64);
4861 case Intrinsic::aarch64_neon_ld4lane:
4862 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4863 SelectLoadLane(
Node, 4, AArch64::LD4i8);
4865 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4866 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
4867 SelectLoadLane(
Node, 4, AArch64::LD4i16);
4869 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4871 SelectLoadLane(
Node, 4, AArch64::LD4i32);
4873 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4875 SelectLoadLane(
Node, 4, AArch64::LD4i64);
4879 case Intrinsic::aarch64_ld64b:
4880 SelectLoad(
Node, 8, AArch64::LD64B, AArch64::x8sub_0);
4882 case Intrinsic::aarch64_sve_ld2q_sret: {
4883 SelectPredicatedLoad(
Node, 2, 4, AArch64::LD2Q_IMM, AArch64::LD2Q,
true);
4886 case Intrinsic::aarch64_sve_ld3q_sret: {
4887 SelectPredicatedLoad(
Node, 3, 4, AArch64::LD3Q_IMM, AArch64::LD3Q,
true);
4890 case Intrinsic::aarch64_sve_ld4q_sret: {
4891 SelectPredicatedLoad(
Node, 4, 4, AArch64::LD4Q_IMM, AArch64::LD4Q,
true);
4894 case Intrinsic::aarch64_sve_ld2_sret: {
4895 if (VT == MVT::nxv16i8) {
4896 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B,
4899 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
4900 VT == MVT::nxv8bf16) {
4901 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H,
4904 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
4905 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W,
4908 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
4909 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D,
4915 case Intrinsic::aarch64_sve_ld1_pn_x2: {
4916 if (VT == MVT::nxv16i8) {
4917 if (Subtarget->hasSME2())
4918 SelectContiguousMultiVectorLoad(
4919 Node, 2, 0, AArch64::LD1B_2Z_IMM_PSEUDO, AArch64::LD1B_2Z_PSEUDO);
4920 else if (Subtarget->hasSVE2p1())
4921 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LD1B_2Z_IMM,
4926 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
4927 VT == MVT::nxv8bf16) {
4928 if (Subtarget->hasSME2())
4929 SelectContiguousMultiVectorLoad(
4930 Node, 2, 1, AArch64::LD1H_2Z_IMM_PSEUDO, AArch64::LD1H_2Z_PSEUDO);
4931 else if (Subtarget->hasSVE2p1())
4932 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LD1H_2Z_IMM,
4937 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
4938 if (Subtarget->hasSME2())
4939 SelectContiguousMultiVectorLoad(
4940 Node, 2, 2, AArch64::LD1W_2Z_IMM_PSEUDO, AArch64::LD1W_2Z_PSEUDO);
4941 else if (Subtarget->hasSVE2p1())
4942 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LD1W_2Z_IMM,
4947 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
4948 if (Subtarget->hasSME2())
4949 SelectContiguousMultiVectorLoad(
4950 Node, 2, 3, AArch64::LD1D_2Z_IMM_PSEUDO, AArch64::LD1D_2Z_PSEUDO);
4951 else if (Subtarget->hasSVE2p1())
4952 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LD1D_2Z_IMM,
4960 case Intrinsic::aarch64_sve_ld1_pn_x4: {
4961 if (VT == MVT::nxv16i8) {
4962 if (Subtarget->hasSME2())
4963 SelectContiguousMultiVectorLoad(
4964 Node, 4, 0, AArch64::LD1B_4Z_IMM_PSEUDO, AArch64::LD1B_4Z_PSEUDO);
4965 else if (Subtarget->hasSVE2p1())
4966 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LD1B_4Z_IMM,
4971 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
4972 VT == MVT::nxv8bf16) {
4973 if (Subtarget->hasSME2())
4974 SelectContiguousMultiVectorLoad(
4975 Node, 4, 1, AArch64::LD1H_4Z_IMM_PSEUDO, AArch64::LD1H_4Z_PSEUDO);
4976 else if (Subtarget->hasSVE2p1())
4977 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LD1H_4Z_IMM,
4982 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
4983 if (Subtarget->hasSME2())
4984 SelectContiguousMultiVectorLoad(
4985 Node, 4, 2, AArch64::LD1W_4Z_IMM_PSEUDO, AArch64::LD1W_4Z_PSEUDO);
4986 else if (Subtarget->hasSVE2p1())
4987 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LD1W_4Z_IMM,
4992 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
4993 if (Subtarget->hasSME2())
4994 SelectContiguousMultiVectorLoad(
4995 Node, 4, 3, AArch64::LD1D_4Z_IMM_PSEUDO, AArch64::LD1D_4Z_PSEUDO);
4996 else if (Subtarget->hasSVE2p1())
4997 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LD1D_4Z_IMM,
5005 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5006 if (VT == MVT::nxv16i8) {
5007 if (Subtarget->hasSME2())
5008 SelectContiguousMultiVectorLoad(
Node, 2, 0,
5009 AArch64::LDNT1B_2Z_IMM_PSEUDO,
5010 AArch64::LDNT1B_2Z_PSEUDO);
5011 else if (Subtarget->hasSVE2p1())
5012 SelectContiguousMultiVectorLoad(
Node, 2, 0, AArch64::LDNT1B_2Z_IMM,
5013 AArch64::LDNT1B_2Z);
5017 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5018 VT == MVT::nxv8bf16) {
5019 if (Subtarget->hasSME2())
5020 SelectContiguousMultiVectorLoad(
Node, 2, 1,
5021 AArch64::LDNT1H_2Z_IMM_PSEUDO,
5022 AArch64::LDNT1H_2Z_PSEUDO);
5023 else if (Subtarget->hasSVE2p1())
5024 SelectContiguousMultiVectorLoad(
Node, 2, 1, AArch64::LDNT1H_2Z_IMM,
5025 AArch64::LDNT1H_2Z);
5029 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5030 if (Subtarget->hasSME2())
5031 SelectContiguousMultiVectorLoad(
Node, 2, 2,
5032 AArch64::LDNT1W_2Z_IMM_PSEUDO,
5033 AArch64::LDNT1W_2Z_PSEUDO);
5034 else if (Subtarget->hasSVE2p1())
5035 SelectContiguousMultiVectorLoad(
Node, 2, 2, AArch64::LDNT1W_2Z_IMM,
5036 AArch64::LDNT1W_2Z);
5040 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5041 if (Subtarget->hasSME2())
5042 SelectContiguousMultiVectorLoad(
Node, 2, 3,
5043 AArch64::LDNT1D_2Z_IMM_PSEUDO,
5044 AArch64::LDNT1D_2Z_PSEUDO);
5045 else if (Subtarget->hasSVE2p1())
5046 SelectContiguousMultiVectorLoad(
Node, 2, 3, AArch64::LDNT1D_2Z_IMM,
5047 AArch64::LDNT1D_2Z);
5054 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5055 if (VT == MVT::nxv16i8) {
5056 if (Subtarget->hasSME2())
5057 SelectContiguousMultiVectorLoad(
Node, 4, 0,
5058 AArch64::LDNT1B_4Z_IMM_PSEUDO,
5059 AArch64::LDNT1B_4Z_PSEUDO);
5060 else if (Subtarget->hasSVE2p1())
5061 SelectContiguousMultiVectorLoad(
Node, 4, 0, AArch64::LDNT1B_4Z_IMM,
5062 AArch64::LDNT1B_4Z);
5066 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5067 VT == MVT::nxv8bf16) {
5068 if (Subtarget->hasSME2())
5069 SelectContiguousMultiVectorLoad(
Node, 4, 1,
5070 AArch64::LDNT1H_4Z_IMM_PSEUDO,
5071 AArch64::LDNT1H_4Z_PSEUDO);
5072 else if (Subtarget->hasSVE2p1())
5073 SelectContiguousMultiVectorLoad(
Node, 4, 1, AArch64::LDNT1H_4Z_IMM,
5074 AArch64::LDNT1H_4Z);
5078 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5079 if (Subtarget->hasSME2())
5080 SelectContiguousMultiVectorLoad(
Node, 4, 2,
5081 AArch64::LDNT1W_4Z_IMM_PSEUDO,
5082 AArch64::LDNT1W_4Z_PSEUDO);
5083 else if (Subtarget->hasSVE2p1())
5084 SelectContiguousMultiVectorLoad(
Node, 4, 2, AArch64::LDNT1W_4Z_IMM,
5085 AArch64::LDNT1W_4Z);
5089 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5090 if (Subtarget->hasSME2())
5091 SelectContiguousMultiVectorLoad(
Node, 4, 3,
5092 AArch64::LDNT1D_4Z_IMM_PSEUDO,
5093 AArch64::LDNT1D_4Z_PSEUDO);
5094 else if (Subtarget->hasSVE2p1())
5095 SelectContiguousMultiVectorLoad(
Node, 4, 3, AArch64::LDNT1D_4Z_IMM,
5096 AArch64::LDNT1D_4Z);
5103 case Intrinsic::aarch64_sve_ld3_sret: {
5104 if (VT == MVT::nxv16i8) {
5105 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B,
5108 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5109 VT == MVT::nxv8bf16) {
5110 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H,
5113 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5114 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W,
5117 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5118 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D,
5124 case Intrinsic::aarch64_sve_ld4_sret: {
5125 if (VT == MVT::nxv16i8) {
5126 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B,
5129 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5130 VT == MVT::nxv8bf16) {
5131 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H,
5134 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5135 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W,
5138 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5139 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D,
5145 case Intrinsic::aarch64_sme_read_hor_vg2: {
5146 if (VT == MVT::nxv16i8) {
5147 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5148 AArch64::MOVA_2ZMXI_H_B);
5150 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5151 VT == MVT::nxv8bf16) {
5152 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5153 AArch64::MOVA_2ZMXI_H_H);
5155 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5156 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5157 AArch64::MOVA_2ZMXI_H_S);
5159 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5160 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5161 AArch64::MOVA_2ZMXI_H_D);
5166 case Intrinsic::aarch64_sme_read_ver_vg2: {
5167 if (VT == MVT::nxv16i8) {
5168 SelectMultiVectorMove<14, 2>(
Node, 2, AArch64::ZAB0,
5169 AArch64::MOVA_2ZMXI_V_B);
5171 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5172 VT == MVT::nxv8bf16) {
5173 SelectMultiVectorMove<6, 2>(
Node, 2, AArch64::ZAH0,
5174 AArch64::MOVA_2ZMXI_V_H);
5176 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5177 SelectMultiVectorMove<2, 2>(
Node, 2, AArch64::ZAS0,
5178 AArch64::MOVA_2ZMXI_V_S);
5180 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5181 SelectMultiVectorMove<0, 2>(
Node, 2, AArch64::ZAD0,
5182 AArch64::MOVA_2ZMXI_V_D);
5187 case Intrinsic::aarch64_sme_read_hor_vg4: {
5188 if (VT == MVT::nxv16i8) {
5189 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5190 AArch64::MOVA_4ZMXI_H_B);
5192 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5193 VT == MVT::nxv8bf16) {
5194 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5195 AArch64::MOVA_4ZMXI_H_H);
5197 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5198 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAS0,
5199 AArch64::MOVA_4ZMXI_H_S);
5201 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5202 SelectMultiVectorMove<0, 2>(
Node, 4, AArch64::ZAD0,
5203 AArch64::MOVA_4ZMXI_H_D);
5208 case Intrinsic::aarch64_sme_read_ver_vg4: {
5209 if (VT == MVT::nxv16i8) {
5210 SelectMultiVectorMove<12, 4>(
Node, 4, AArch64::ZAB0,
5211 AArch64::MOVA_4ZMXI_V_B);
5213 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5214 VT == MVT::nxv8bf16) {
5215 SelectMultiVectorMove<4, 4>(
Node, 4, AArch64::ZAH0,
5216 AArch64::MOVA_4ZMXI_V_H);
5218 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5219 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAS0,
5220 AArch64::MOVA_4ZMXI_V_S);
5222 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5223 SelectMultiVectorMove<0, 4>(
Node, 4, AArch64::ZAD0,
5224 AArch64::MOVA_4ZMXI_V_D);
5229 case Intrinsic::aarch64_sme_read_vg1x2: {
5230 SelectMultiVectorMove<7, 1>(
Node, 2, AArch64::ZA,
5231 AArch64::MOVA_VG2_2ZMXI);
5234 case Intrinsic::aarch64_sme_read_vg1x4: {
5235 SelectMultiVectorMove<7, 1>(
Node, 4, AArch64::ZA,
5236 AArch64::MOVA_VG4_4ZMXI);
5239 case Intrinsic::swift_async_context_addr: {
5242 SDValue CopyFP = CurDAG->getCopyFromReg(Chain,
DL, AArch64::FP, MVT::i64);
5244 CurDAG->getMachineNode(AArch64::SUBXri,
DL, MVT::i64, CopyFP,
5245 CurDAG->getTargetConstant(8,
DL, MVT::i32),
5246 CurDAG->getTargetConstant(0,
DL, MVT::i32)),
5250 CurDAG->RemoveDeadNode(
Node);
5252 auto &MF = CurDAG->getMachineFunction();
5253 MF.getFrameInfo().setFrameAddressIsTaken(
true);
5257 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
5258 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5259 Node->getValueType(0),
5260 {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
5261 AArch64::LUTI2_4ZTZI_S}))
5263 SelectMultiVectorLuti(
Node, 4, Opc, 3);
5266 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
5267 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5268 Node->getValueType(0),
5269 {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
5271 SelectMultiVectorLuti(
Node, 4, Opc, 1);
5274 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
5275 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5276 Node->getValueType(0),
5277 {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
5278 AArch64::LUTI2_2ZTZI_S}))
5280 SelectMultiVectorLuti(
Node, 2, Opc, 7);
5283 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
5284 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5285 Node->getValueType(0),
5286 {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
5287 AArch64::LUTI4_2ZTZI_S}))
5289 SelectMultiVectorLuti(
Node, 2, Opc, 3);
5295 unsigned IntNo =
Node->getConstantOperandVal(0);
5299 case Intrinsic::aarch64_tagp:
5302 case Intrinsic::aarch64_neon_tbl2:
5303 SelectTable(
Node, 2,
5304 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
5307 case Intrinsic::aarch64_neon_tbl3:
5308 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
5309 : AArch64::TBLv16i8Three,
5312 case Intrinsic::aarch64_neon_tbl4:
5313 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
5314 : AArch64::TBLv16i8Four,
5317 case Intrinsic::aarch64_neon_tbx2:
5318 SelectTable(
Node, 2,
5319 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
5322 case Intrinsic::aarch64_neon_tbx3:
5323 SelectTable(
Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
5324 : AArch64::TBXv16i8Three,
5327 case Intrinsic::aarch64_neon_tbx4:
5328 SelectTable(
Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
5329 : AArch64::TBXv16i8Four,
5332 case Intrinsic::aarch64_sve_srshl_single_x2:
5333 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5334 Node->getValueType(0),
5335 {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
5336 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
5337 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5339 case Intrinsic::aarch64_sve_srshl_single_x4:
5340 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5341 Node->getValueType(0),
5342 {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
5343 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
5344 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5346 case Intrinsic::aarch64_sve_urshl_single_x2:
5347 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5348 Node->getValueType(0),
5349 {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
5350 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
5351 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5353 case Intrinsic::aarch64_sve_urshl_single_x4:
5354 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5355 Node->getValueType(0),
5356 {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
5357 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
5358 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5360 case Intrinsic::aarch64_sve_srshl_x2:
5361 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5362 Node->getValueType(0),
5363 {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
5364 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
5365 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5367 case Intrinsic::aarch64_sve_srshl_x4:
5368 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5369 Node->getValueType(0),
5370 {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
5371 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
5372 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5374 case Intrinsic::aarch64_sve_urshl_x2:
5375 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5376 Node->getValueType(0),
5377 {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
5378 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
5379 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5381 case Intrinsic::aarch64_sve_urshl_x4:
5382 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5383 Node->getValueType(0),
5384 {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
5385 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
5386 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5388 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
5389 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5390 Node->getValueType(0),
5391 {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
5392 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
5393 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5395 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
5396 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5397 Node->getValueType(0),
5398 {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
5399 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
5400 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5402 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
5403 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5404 Node->getValueType(0),
5405 {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
5406 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
5407 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5409 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
5410 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5411 Node->getValueType(0),
5412 {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
5413 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
5414 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5416 case Intrinsic::aarch64_sve_whilege_x2:
5417 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5418 Node->getValueType(0),
5419 {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
5420 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
5421 SelectWhilePair(
Node,
Op);
5423 case Intrinsic::aarch64_sve_whilegt_x2:
5424 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5425 Node->getValueType(0),
5426 {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
5427 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
5428 SelectWhilePair(
Node,
Op);
5430 case Intrinsic::aarch64_sve_whilehi_x2:
5431 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5432 Node->getValueType(0),
5433 {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
5434 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
5435 SelectWhilePair(
Node,
Op);
5437 case Intrinsic::aarch64_sve_whilehs_x2:
5438 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5439 Node->getValueType(0),
5440 {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
5441 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
5442 SelectWhilePair(
Node,
Op);
5444 case Intrinsic::aarch64_sve_whilele_x2:
5445 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5446 Node->getValueType(0),
5447 {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
5448 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
5449 SelectWhilePair(
Node,
Op);
5451 case Intrinsic::aarch64_sve_whilelo_x2:
5452 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5453 Node->getValueType(0),
5454 {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
5455 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
5456 SelectWhilePair(
Node,
Op);
5458 case Intrinsic::aarch64_sve_whilels_x2:
5459 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5460 Node->getValueType(0),
5461 {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
5462 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
5463 SelectWhilePair(
Node,
Op);
5465 case Intrinsic::aarch64_sve_whilelt_x2:
5466 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
5467 Node->getValueType(0),
5468 {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
5469 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
5470 SelectWhilePair(
Node,
Op);
5472 case Intrinsic::aarch64_sve_smax_single_x2:
5473 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5474 Node->getValueType(0),
5475 {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
5476 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
5477 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5479 case Intrinsic::aarch64_sve_umax_single_x2:
5480 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5481 Node->getValueType(0),
5482 {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
5483 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
5484 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5486 case Intrinsic::aarch64_sve_fmax_single_x2:
5487 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5488 Node->getValueType(0),
5489 {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
5490 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
5491 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5493 case Intrinsic::aarch64_sve_smax_single_x4:
5494 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5495 Node->getValueType(0),
5496 {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
5497 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
5498 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5500 case Intrinsic::aarch64_sve_umax_single_x4:
5501 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5502 Node->getValueType(0),
5503 {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
5504 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
5505 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5507 case Intrinsic::aarch64_sve_fmax_single_x4:
5508 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5509 Node->getValueType(0),
5510 {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
5511 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
5512 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5514 case Intrinsic::aarch64_sve_smin_single_x2:
5515 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5516 Node->getValueType(0),
5517 {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
5518 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
5519 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5521 case Intrinsic::aarch64_sve_umin_single_x2:
5522 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5523 Node->getValueType(0),
5524 {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
5525 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
5526 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5528 case Intrinsic::aarch64_sve_fmin_single_x2:
5529 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5530 Node->getValueType(0),
5531 {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
5532 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
5533 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5535 case Intrinsic::aarch64_sve_smin_single_x4:
5536 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5537 Node->getValueType(0),
5538 {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
5539 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
5540 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5542 case Intrinsic::aarch64_sve_umin_single_x4:
5543 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5544 Node->getValueType(0),
5545 {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
5546 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
5547 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5549 case Intrinsic::aarch64_sve_fmin_single_x4:
5550 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5551 Node->getValueType(0),
5552 {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
5553 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
5554 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5556 case Intrinsic::aarch64_sve_smax_x2:
5557 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5558 Node->getValueType(0),
5559 {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
5560 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
5561 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5563 case Intrinsic::aarch64_sve_umax_x2:
5564 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5565 Node->getValueType(0),
5566 {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
5567 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
5568 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5570 case Intrinsic::aarch64_sve_fmax_x2:
5571 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5572 Node->getValueType(0),
5573 {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
5574 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
5575 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5577 case Intrinsic::aarch64_sve_smax_x4:
5578 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5579 Node->getValueType(0),
5580 {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
5581 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
5582 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5584 case Intrinsic::aarch64_sve_umax_x4:
5585 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5586 Node->getValueType(0),
5587 {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
5588 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
5589 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5591 case Intrinsic::aarch64_sve_fmax_x4:
5592 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5593 Node->getValueType(0),
5594 {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
5595 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
5596 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5598 case Intrinsic::aarch64_sve_smin_x2:
5599 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5600 Node->getValueType(0),
5601 {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
5602 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
5603 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5605 case Intrinsic::aarch64_sve_umin_x2:
5606 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5607 Node->getValueType(0),
5608 {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
5609 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
5610 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5612 case Intrinsic::aarch64_sve_fmin_x2:
5613 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5614 Node->getValueType(0),
5615 {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
5616 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
5617 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5619 case Intrinsic::aarch64_sve_smin_x4:
5620 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5621 Node->getValueType(0),
5622 {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
5623 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
5624 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5626 case Intrinsic::aarch64_sve_umin_x4:
5627 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5628 Node->getValueType(0),
5629 {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
5630 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
5631 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5633 case Intrinsic::aarch64_sve_fmin_x4:
5634 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5635 Node->getValueType(0),
5636 {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
5637 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
5638 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5640 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
5641 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5642 Node->getValueType(0),
5643 {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
5644 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
5645 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5647 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
5648 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5649 Node->getValueType(0),
5650 {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
5651 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
5652 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5654 case Intrinsic::aarch64_sve_fminnm_single_x2:
5655 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5656 Node->getValueType(0),
5657 {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
5658 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
5659 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5661 case Intrinsic::aarch64_sve_fminnm_single_x4:
5662 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5663 Node->getValueType(0),
5664 {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
5665 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
5666 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5668 case Intrinsic::aarch64_sve_fmaxnm_x2:
5669 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5670 Node->getValueType(0),
5671 {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
5672 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
5673 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5675 case Intrinsic::aarch64_sve_fmaxnm_x4:
5676 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5677 Node->getValueType(0),
5678 {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
5679 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
5680 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5682 case Intrinsic::aarch64_sve_fminnm_x2:
5683 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5684 Node->getValueType(0),
5685 {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
5686 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
5687 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op);
5689 case Intrinsic::aarch64_sve_fminnm_x4:
5690 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5691 Node->getValueType(0),
5692 {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
5693 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
5694 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op);
5696 case Intrinsic::aarch64_sve_fcvtzs_x2:
5697 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZS_2Z2Z_StoS);
5699 case Intrinsic::aarch64_sve_scvtf_x2:
5700 SelectCVTIntrinsic(
Node, 2, AArch64::SCVTF_2Z2Z_StoS);
5702 case Intrinsic::aarch64_sve_fcvtzu_x2:
5703 SelectCVTIntrinsic(
Node, 2, AArch64::FCVTZU_2Z2Z_StoS);
5705 case Intrinsic::aarch64_sve_ucvtf_x2:
5706 SelectCVTIntrinsic(
Node, 2, AArch64::UCVTF_2Z2Z_StoS);
5708 case Intrinsic::aarch64_sve_fcvtzs_x4:
5709 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZS_4Z4Z_StoS);
5711 case Intrinsic::aarch64_sve_scvtf_x4:
5712 SelectCVTIntrinsic(
Node, 4, AArch64::SCVTF_4Z4Z_StoS);
5714 case Intrinsic::aarch64_sve_fcvtzu_x4:
5715 SelectCVTIntrinsic(
Node, 4, AArch64::FCVTZU_4Z4Z_StoS);
5717 case Intrinsic::aarch64_sve_ucvtf_x4:
5718 SelectCVTIntrinsic(
Node, 4, AArch64::UCVTF_4Z4Z_StoS);
5720 case Intrinsic::aarch64_sve_sclamp_single_x2:
5721 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5722 Node->getValueType(0),
5723 {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
5724 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
5725 SelectClamp(
Node, 2,
Op);
5727 case Intrinsic::aarch64_sve_uclamp_single_x2:
5728 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5729 Node->getValueType(0),
5730 {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
5731 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
5732 SelectClamp(
Node, 2,
Op);
5734 case Intrinsic::aarch64_sve_fclamp_single_x2:
5735 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5736 Node->getValueType(0),
5737 {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
5738 AArch64::FCLAMP_VG2_2Z2Z_D}))
5739 SelectClamp(
Node, 2,
Op);
5741 case Intrinsic::aarch64_sve_sclamp_single_x4:
5742 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5743 Node->getValueType(0),
5744 {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
5745 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
5746 SelectClamp(
Node, 4,
Op);
5748 case Intrinsic::aarch64_sve_uclamp_single_x4:
5749 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5750 Node->getValueType(0),
5751 {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
5752 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
5753 SelectClamp(
Node, 4,
Op);
5755 case Intrinsic::aarch64_sve_fclamp_single_x4:
5756 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
5757 Node->getValueType(0),
5758 {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
5759 AArch64::FCLAMP_VG4_4Z4Z_D}))
5760 SelectClamp(
Node, 4,
Op);
5762 case Intrinsic::aarch64_sve_add_single_x2:
5763 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5764 Node->getValueType(0),
5765 {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
5766 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
5767 SelectDestructiveMultiIntrinsic(
Node, 2,
false,
Op);
5769 case Intrinsic::aarch64_sve_add_single_x4:
5770 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5771 Node->getValueType(0),
5772 {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
5773 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
5774 SelectDestructiveMultiIntrinsic(
Node, 4,
false,
Op);
5776 case Intrinsic::aarch64_sve_zip_x2:
5777 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5778 Node->getValueType(0),
5779 {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
5780 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
5781 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5783 case Intrinsic::aarch64_sve_zipq_x2:
5784 SelectUnaryMultiIntrinsic(
Node, 2,
false,
5785 AArch64::ZIP_VG2_2ZZZ_Q);
5787 case Intrinsic::aarch64_sve_zip_x4:
5788 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5789 Node->getValueType(0),
5790 {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
5791 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
5792 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5794 case Intrinsic::aarch64_sve_zipq_x4:
5795 SelectUnaryMultiIntrinsic(
Node, 4,
true,
5796 AArch64::ZIP_VG4_4Z4Z_Q);
5798 case Intrinsic::aarch64_sve_uzp_x2:
5799 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5800 Node->getValueType(0),
5801 {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
5802 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
5803 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5805 case Intrinsic::aarch64_sve_uzpq_x2:
5806 SelectUnaryMultiIntrinsic(
Node, 2,
false,
5807 AArch64::UZP_VG2_2ZZZ_Q);
5809 case Intrinsic::aarch64_sve_uzp_x4:
5810 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5811 Node->getValueType(0),
5812 {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
5813 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
5814 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5816 case Intrinsic::aarch64_sve_uzpq_x4:
5817 SelectUnaryMultiIntrinsic(
Node, 4,
true,
5818 AArch64::UZP_VG4_4Z4Z_Q);
5820 case Intrinsic::aarch64_sve_sel_x2:
5821 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5822 Node->getValueType(0),
5823 {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
5824 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
5825 SelectDestructiveMultiIntrinsic(
Node, 2,
true,
Op,
true);
5827 case Intrinsic::aarch64_sve_sel_x4:
5828 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5829 Node->getValueType(0),
5830 {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
5831 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
5832 SelectDestructiveMultiIntrinsic(
Node, 4,
true,
Op,
true);
5834 case Intrinsic::aarch64_sve_frinta_x2:
5835 SelectFrintFromVT(
Node, 2, AArch64::FRINTA_2Z2Z_S);
5837 case Intrinsic::aarch64_sve_frinta_x4:
5838 SelectFrintFromVT(
Node, 4, AArch64::FRINTA_4Z4Z_S);
5840 case Intrinsic::aarch64_sve_frintm_x2:
5841 SelectFrintFromVT(
Node, 2, AArch64::FRINTM_2Z2Z_S);
5843 case Intrinsic::aarch64_sve_frintm_x4:
5844 SelectFrintFromVT(
Node, 4, AArch64::FRINTM_4Z4Z_S);
5846 case Intrinsic::aarch64_sve_frintn_x2:
5847 SelectFrintFromVT(
Node, 2, AArch64::FRINTN_2Z2Z_S);
5849 case Intrinsic::aarch64_sve_frintn_x4:
5850 SelectFrintFromVT(
Node, 4, AArch64::FRINTN_4Z4Z_S);
5852 case Intrinsic::aarch64_sve_frintp_x2:
5853 SelectFrintFromVT(
Node, 2, AArch64::FRINTP_2Z2Z_S);
5855 case Intrinsic::aarch64_sve_frintp_x4:
5856 SelectFrintFromVT(
Node, 4, AArch64::FRINTP_4Z4Z_S);
5858 case Intrinsic::aarch64_sve_sunpk_x2:
5859 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5860 Node->getValueType(0),
5861 {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
5862 AArch64::SUNPK_VG2_2ZZ_D}))
5863 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5865 case Intrinsic::aarch64_sve_uunpk_x2:
5866 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5867 Node->getValueType(0),
5868 {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
5869 AArch64::UUNPK_VG2_2ZZ_D}))
5870 SelectUnaryMultiIntrinsic(
Node, 2,
false,
Op);
5872 case Intrinsic::aarch64_sve_sunpk_x4:
5873 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5874 Node->getValueType(0),
5875 {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
5876 AArch64::SUNPK_VG4_4Z2Z_D}))
5877 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5879 case Intrinsic::aarch64_sve_uunpk_x4:
5880 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
5881 Node->getValueType(0),
5882 {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
5883 AArch64::UUNPK_VG4_4Z2Z_D}))
5884 SelectUnaryMultiIntrinsic(
Node, 4,
true,
Op);
5886 case Intrinsic::aarch64_sve_pext_x2: {
5887 if (
auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
5888 Node->getValueType(0),
5889 {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
5890 AArch64::PEXT_2PCI_D}))
5891 SelectPExtPair(
Node,
Op);
5898 unsigned IntNo =
Node->getConstantOperandVal(1);
5899 if (
Node->getNumOperands() >= 3)
5900 VT =
Node->getOperand(2)->getValueType(0);
5904 case Intrinsic::aarch64_neon_st1x2: {
5905 if (VT == MVT::v8i8) {
5906 SelectStore(
Node, 2, AArch64::ST1Twov8b);
5908 }
else if (VT == MVT::v16i8) {
5909 SelectStore(
Node, 2, AArch64::ST1Twov16b);
5911 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
5912 VT == MVT::v4bf16) {
5913 SelectStore(
Node, 2, AArch64::ST1Twov4h);
5915 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
5916 VT == MVT::v8bf16) {
5917 SelectStore(
Node, 2, AArch64::ST1Twov8h);
5919 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5920 SelectStore(
Node, 2, AArch64::ST1Twov2s);
5922 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5923 SelectStore(
Node, 2, AArch64::ST1Twov4s);
5925 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5926 SelectStore(
Node, 2, AArch64::ST1Twov2d);
5928 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5929 SelectStore(
Node, 2, AArch64::ST1Twov1d);
5934 case Intrinsic::aarch64_neon_st1x3: {
5935 if (VT == MVT::v8i8) {
5936 SelectStore(
Node, 3, AArch64::ST1Threev8b);
5938 }
else if (VT == MVT::v16i8) {
5939 SelectStore(
Node, 3, AArch64::ST1Threev16b);
5941 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
5942 VT == MVT::v4bf16) {
5943 SelectStore(
Node, 3, AArch64::ST1Threev4h);
5945 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
5946 VT == MVT::v8bf16) {
5947 SelectStore(
Node, 3, AArch64::ST1Threev8h);
5949 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5950 SelectStore(
Node, 3, AArch64::ST1Threev2s);
5952 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5953 SelectStore(
Node, 3, AArch64::ST1Threev4s);
5955 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5956 SelectStore(
Node, 3, AArch64::ST1Threev2d);
5958 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5959 SelectStore(
Node, 3, AArch64::ST1Threev1d);
5964 case Intrinsic::aarch64_neon_st1x4: {
5965 if (VT == MVT::v8i8) {
5966 SelectStore(
Node, 4, AArch64::ST1Fourv8b);
5968 }
else if (VT == MVT::v16i8) {
5969 SelectStore(
Node, 4, AArch64::ST1Fourv16b);
5971 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
5972 VT == MVT::v4bf16) {
5973 SelectStore(
Node, 4, AArch64::ST1Fourv4h);
5975 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
5976 VT == MVT::v8bf16) {
5977 SelectStore(
Node, 4, AArch64::ST1Fourv8h);
5979 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5980 SelectStore(
Node, 4, AArch64::ST1Fourv2s);
5982 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5983 SelectStore(
Node, 4, AArch64::ST1Fourv4s);
5985 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5986 SelectStore(
Node, 4, AArch64::ST1Fourv2d);
5988 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5989 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
5994 case Intrinsic::aarch64_neon_st2: {
5995 if (VT == MVT::v8i8) {
5996 SelectStore(
Node, 2, AArch64::ST2Twov8b);
5998 }
else if (VT == MVT::v16i8) {
5999 SelectStore(
Node, 2, AArch64::ST2Twov16b);
6001 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6002 VT == MVT::v4bf16) {
6003 SelectStore(
Node, 2, AArch64::ST2Twov4h);
6005 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6006 VT == MVT::v8bf16) {
6007 SelectStore(
Node, 2, AArch64::ST2Twov8h);
6009 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6010 SelectStore(
Node, 2, AArch64::ST2Twov2s);
6012 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6013 SelectStore(
Node, 2, AArch64::ST2Twov4s);
6015 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6016 SelectStore(
Node, 2, AArch64::ST2Twov2d);
6018 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6019 SelectStore(
Node, 2, AArch64::ST1Twov1d);
6024 case Intrinsic::aarch64_neon_st3: {
6025 if (VT == MVT::v8i8) {
6026 SelectStore(
Node, 3, AArch64::ST3Threev8b);
6028 }
else if (VT == MVT::v16i8) {
6029 SelectStore(
Node, 3, AArch64::ST3Threev16b);
6031 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6032 VT == MVT::v4bf16) {
6033 SelectStore(
Node, 3, AArch64::ST3Threev4h);
6035 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6036 VT == MVT::v8bf16) {
6037 SelectStore(
Node, 3, AArch64::ST3Threev8h);
6039 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6040 SelectStore(
Node, 3, AArch64::ST3Threev2s);
6042 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6043 SelectStore(
Node, 3, AArch64::ST3Threev4s);
6045 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6046 SelectStore(
Node, 3, AArch64::ST3Threev2d);
6048 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6049 SelectStore(
Node, 3, AArch64::ST1Threev1d);
6054 case Intrinsic::aarch64_neon_st4: {
6055 if (VT == MVT::v8i8) {
6056 SelectStore(
Node, 4, AArch64::ST4Fourv8b);
6058 }
else if (VT == MVT::v16i8) {
6059 SelectStore(
Node, 4, AArch64::ST4Fourv16b);
6061 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6062 VT == MVT::v4bf16) {
6063 SelectStore(
Node, 4, AArch64::ST4Fourv4h);
6065 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6066 VT == MVT::v8bf16) {
6067 SelectStore(
Node, 4, AArch64::ST4Fourv8h);
6069 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6070 SelectStore(
Node, 4, AArch64::ST4Fourv2s);
6072 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6073 SelectStore(
Node, 4, AArch64::ST4Fourv4s);
6075 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6076 SelectStore(
Node, 4, AArch64::ST4Fourv2d);
6078 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6079 SelectStore(
Node, 4, AArch64::ST1Fourv1d);
6084 case Intrinsic::aarch64_neon_st2lane: {
6085 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6086 SelectStoreLane(
Node, 2, AArch64::ST2i8);
6088 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6089 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6090 SelectStoreLane(
Node, 2, AArch64::ST2i16);
6092 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6094 SelectStoreLane(
Node, 2, AArch64::ST2i32);
6096 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6098 SelectStoreLane(
Node, 2, AArch64::ST2i64);
6103 case Intrinsic::aarch64_neon_st3lane: {
6104 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6105 SelectStoreLane(
Node, 3, AArch64::ST3i8);
6107 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6108 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6109 SelectStoreLane(
Node, 3, AArch64::ST3i16);
6111 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6113 SelectStoreLane(
Node, 3, AArch64::ST3i32);
6115 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6117 SelectStoreLane(
Node, 3, AArch64::ST3i64);
6122 case Intrinsic::aarch64_neon_st4lane: {
6123 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6124 SelectStoreLane(
Node, 4, AArch64::ST4i8);
6126 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6127 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6128 SelectStoreLane(
Node, 4, AArch64::ST4i16);
6130 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6132 SelectStoreLane(
Node, 4, AArch64::ST4i32);
6134 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6136 SelectStoreLane(
Node, 4, AArch64::ST4i64);
6141 case Intrinsic::aarch64_sve_st2q: {
6142 SelectPredicatedStore(
Node, 2, 4, AArch64::ST2Q, AArch64::ST2Q_IMM);
6145 case Intrinsic::aarch64_sve_st3q: {
6146 SelectPredicatedStore(
Node, 3, 4, AArch64::ST3Q, AArch64::ST3Q_IMM);
6149 case Intrinsic::aarch64_sve_st4q: {
6150 SelectPredicatedStore(
Node, 4, 4, AArch64::ST4Q, AArch64::ST4Q_IMM);
6153 case Intrinsic::aarch64_sve_st2: {
6154 if (VT == MVT::nxv16i8) {
6155 SelectPredicatedStore(
Node, 2, 0, AArch64::ST2B, AArch64::ST2B_IMM);
6157 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6158 VT == MVT::nxv8bf16) {
6159 SelectPredicatedStore(
Node, 2, 1, AArch64::ST2H, AArch64::ST2H_IMM);
6161 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6162 SelectPredicatedStore(
Node, 2, 2, AArch64::ST2W, AArch64::ST2W_IMM);
6164 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6165 SelectPredicatedStore(
Node, 2, 3, AArch64::ST2D, AArch64::ST2D_IMM);
6170 case Intrinsic::aarch64_sve_st3: {
6171 if (VT == MVT::nxv16i8) {
6172 SelectPredicatedStore(
Node, 3, 0, AArch64::ST3B, AArch64::ST3B_IMM);
6174 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6175 VT == MVT::nxv8bf16) {
6176 SelectPredicatedStore(
Node, 3, 1, AArch64::ST3H, AArch64::ST3H_IMM);
6178 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6179 SelectPredicatedStore(
Node, 3, 2, AArch64::ST3W, AArch64::ST3W_IMM);
6181 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6182 SelectPredicatedStore(
Node, 3, 3, AArch64::ST3D, AArch64::ST3D_IMM);
6187 case Intrinsic::aarch64_sve_st4: {
6188 if (VT == MVT::nxv16i8) {
6189 SelectPredicatedStore(
Node, 4, 0, AArch64::ST4B, AArch64::ST4B_IMM);
6191 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6192 VT == MVT::nxv8bf16) {
6193 SelectPredicatedStore(
Node, 4, 1, AArch64::ST4H, AArch64::ST4H_IMM);
6195 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6196 SelectPredicatedStore(
Node, 4, 2, AArch64::ST4W, AArch64::ST4W_IMM);
6198 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6199 SelectPredicatedStore(
Node, 4, 3, AArch64::ST4D, AArch64::ST4D_IMM);
6208 if (VT == MVT::v8i8) {
6209 SelectPostLoad(
Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
6211 }
else if (VT == MVT::v16i8) {
6212 SelectPostLoad(
Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
6214 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6215 SelectPostLoad(
Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
6217 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6218 SelectPostLoad(
Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
6220 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6221 SelectPostLoad(
Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
6223 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6224 SelectPostLoad(
Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
6226 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6227 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6229 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6230 SelectPostLoad(
Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
6236 if (VT == MVT::v8i8) {
6237 SelectPostLoad(
Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
6239 }
else if (VT == MVT::v16i8) {
6240 SelectPostLoad(
Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
6242 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6243 SelectPostLoad(
Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
6245 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6246 SelectPostLoad(
Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
6248 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6249 SelectPostLoad(
Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
6251 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6252 SelectPostLoad(
Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
6254 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6255 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6257 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6258 SelectPostLoad(
Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
6264 if (VT == MVT::v8i8) {
6265 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
6267 }
else if (VT == MVT::v16i8) {
6268 SelectPostLoad(
Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
6270 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6271 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
6273 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6274 SelectPostLoad(
Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
6276 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6277 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
6279 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6280 SelectPostLoad(
Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
6282 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6283 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6285 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6286 SelectPostLoad(
Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
6292 if (VT == MVT::v8i8) {
6293 SelectPostLoad(
Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
6295 }
else if (VT == MVT::v16i8) {
6296 SelectPostLoad(
Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
6298 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6299 SelectPostLoad(
Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
6301 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6302 SelectPostLoad(
Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
6304 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6305 SelectPostLoad(
Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
6307 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6308 SelectPostLoad(
Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
6310 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6311 SelectPostLoad(
Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
6313 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6314 SelectPostLoad(
Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
6320 if (VT == MVT::v8i8) {
6321 SelectPostLoad(
Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
6323 }
else if (VT == MVT::v16i8) {
6324 SelectPostLoad(
Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
6326 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6327 SelectPostLoad(
Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
6329 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6330 SelectPostLoad(
Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
6332 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6333 SelectPostLoad(
Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
6335 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6336 SelectPostLoad(
Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
6338 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6339 SelectPostLoad(
Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
6341 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6342 SelectPostLoad(
Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
6348 if (VT == MVT::v8i8) {
6349 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
6351 }
else if (VT == MVT::v16i8) {
6352 SelectPostLoad(
Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
6354 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6355 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
6357 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6358 SelectPostLoad(
Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
6360 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6361 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
6363 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6364 SelectPostLoad(
Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
6366 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6367 SelectPostLoad(
Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
6369 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6370 SelectPostLoad(
Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
6376 if (VT == MVT::v8i8) {
6377 SelectPostLoad(
Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
6379 }
else if (VT == MVT::v16i8) {
6380 SelectPostLoad(
Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
6382 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6383 SelectPostLoad(
Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
6385 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6386 SelectPostLoad(
Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
6388 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6389 SelectPostLoad(
Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
6391 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6392 SelectPostLoad(
Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
6394 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6395 SelectPostLoad(
Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
6397 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6398 SelectPostLoad(
Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
6404 if (VT == MVT::v8i8) {
6405 SelectPostLoad(
Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
6407 }
else if (VT == MVT::v16i8) {
6408 SelectPostLoad(
Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
6410 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6411 SelectPostLoad(
Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
6413 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6414 SelectPostLoad(
Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
6416 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6417 SelectPostLoad(
Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
6419 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6420 SelectPostLoad(
Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
6422 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6423 SelectPostLoad(
Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
6425 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6426 SelectPostLoad(
Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
6432 if (VT == MVT::v8i8) {
6433 SelectPostLoad(
Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
6435 }
else if (VT == MVT::v16i8) {
6436 SelectPostLoad(
Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
6438 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6439 SelectPostLoad(
Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
6441 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6442 SelectPostLoad(
Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
6444 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6445 SelectPostLoad(
Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
6447 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6448 SelectPostLoad(
Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
6450 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6451 SelectPostLoad(
Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
6453 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6454 SelectPostLoad(
Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
6460 if (VT == MVT::v8i8) {
6461 SelectPostLoad(
Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
6463 }
else if (VT == MVT::v16i8) {
6464 SelectPostLoad(
Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
6466 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6467 SelectPostLoad(
Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
6469 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6470 SelectPostLoad(
Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
6472 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6473 SelectPostLoad(
Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
6475 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6476 SelectPostLoad(
Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
6478 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6479 SelectPostLoad(
Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
6481 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6482 SelectPostLoad(
Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
6488 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6489 SelectPostLoadLane(
Node, 1, AArch64::LD1i8_POST);
6491 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6492 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6493 SelectPostLoadLane(
Node, 1, AArch64::LD1i16_POST);
6495 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6497 SelectPostLoadLane(
Node, 1, AArch64::LD1i32_POST);
6499 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6501 SelectPostLoadLane(
Node, 1, AArch64::LD1i64_POST);
6507 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6508 SelectPostLoadLane(
Node, 2, AArch64::LD2i8_POST);
6510 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6511 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6512 SelectPostLoadLane(
Node, 2, AArch64::LD2i16_POST);
6514 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6516 SelectPostLoadLane(
Node, 2, AArch64::LD2i32_POST);
6518 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6520 SelectPostLoadLane(
Node, 2, AArch64::LD2i64_POST);
6526 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6527 SelectPostLoadLane(
Node, 3, AArch64::LD3i8_POST);
6529 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6530 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6531 SelectPostLoadLane(
Node, 3, AArch64::LD3i16_POST);
6533 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6535 SelectPostLoadLane(
Node, 3, AArch64::LD3i32_POST);
6537 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6539 SelectPostLoadLane(
Node, 3, AArch64::LD3i64_POST);
6545 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6546 SelectPostLoadLane(
Node, 4, AArch64::LD4i8_POST);
6548 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6549 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6550 SelectPostLoadLane(
Node, 4, AArch64::LD4i16_POST);
6552 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6554 SelectPostLoadLane(
Node, 4, AArch64::LD4i32_POST);
6556 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6558 SelectPostLoadLane(
Node, 4, AArch64::LD4i64_POST);
6564 VT =
Node->getOperand(1).getValueType();
6565 if (VT == MVT::v8i8) {
6566 SelectPostStore(
Node, 2, AArch64::ST2Twov8b_POST);
6568 }
else if (VT == MVT::v16i8) {
6569 SelectPostStore(
Node, 2, AArch64::ST2Twov16b_POST);
6571 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6572 SelectPostStore(
Node, 2, AArch64::ST2Twov4h_POST);
6574 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6575 SelectPostStore(
Node, 2, AArch64::ST2Twov8h_POST);
6577 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6578 SelectPostStore(
Node, 2, AArch64::ST2Twov2s_POST);
6580 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6581 SelectPostStore(
Node, 2, AArch64::ST2Twov4s_POST);
6583 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6584 SelectPostStore(
Node, 2, AArch64::ST2Twov2d_POST);
6586 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6587 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6593 VT =
Node->getOperand(1).getValueType();
6594 if (VT == MVT::v8i8) {
6595 SelectPostStore(
Node, 3, AArch64::ST3Threev8b_POST);
6597 }
else if (VT == MVT::v16i8) {
6598 SelectPostStore(
Node, 3, AArch64::ST3Threev16b_POST);
6600 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6601 SelectPostStore(
Node, 3, AArch64::ST3Threev4h_POST);
6603 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6604 SelectPostStore(
Node, 3, AArch64::ST3Threev8h_POST);
6606 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6607 SelectPostStore(
Node, 3, AArch64::ST3Threev2s_POST);
6609 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6610 SelectPostStore(
Node, 3, AArch64::ST3Threev4s_POST);
6612 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6613 SelectPostStore(
Node, 3, AArch64::ST3Threev2d_POST);
6615 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6616 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6622 VT =
Node->getOperand(1).getValueType();
6623 if (VT == MVT::v8i8) {
6624 SelectPostStore(
Node, 4, AArch64::ST4Fourv8b_POST);
6626 }
else if (VT == MVT::v16i8) {
6627 SelectPostStore(
Node, 4, AArch64::ST4Fourv16b_POST);
6629 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6630 SelectPostStore(
Node, 4, AArch64::ST4Fourv4h_POST);
6632 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6633 SelectPostStore(
Node, 4, AArch64::ST4Fourv8h_POST);
6635 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6636 SelectPostStore(
Node, 4, AArch64::ST4Fourv2s_POST);
6638 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6639 SelectPostStore(
Node, 4, AArch64::ST4Fourv4s_POST);
6641 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6642 SelectPostStore(
Node, 4, AArch64::ST4Fourv2d_POST);
6644 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6645 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6651 VT =
Node->getOperand(1).getValueType();
6652 if (VT == MVT::v8i8) {
6653 SelectPostStore(
Node, 2, AArch64::ST1Twov8b_POST);
6655 }
else if (VT == MVT::v16i8) {
6656 SelectPostStore(
Node, 2, AArch64::ST1Twov16b_POST);
6658 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6659 SelectPostStore(
Node, 2, AArch64::ST1Twov4h_POST);
6661 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6662 SelectPostStore(
Node, 2, AArch64::ST1Twov8h_POST);
6664 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6665 SelectPostStore(
Node, 2, AArch64::ST1Twov2s_POST);
6667 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6668 SelectPostStore(
Node, 2, AArch64::ST1Twov4s_POST);
6670 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6671 SelectPostStore(
Node, 2, AArch64::ST1Twov1d_POST);
6673 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6674 SelectPostStore(
Node, 2, AArch64::ST1Twov2d_POST);
6680 VT =
Node->getOperand(1).getValueType();
6681 if (VT == MVT::v8i8) {
6682 SelectPostStore(
Node, 3, AArch64::ST1Threev8b_POST);
6684 }
else if (VT == MVT::v16i8) {
6685 SelectPostStore(
Node, 3, AArch64::ST1Threev16b_POST);
6687 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6688 SelectPostStore(
Node, 3, AArch64::ST1Threev4h_POST);
6690 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
6691 SelectPostStore(
Node, 3, AArch64::ST1Threev8h_POST);
6693 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6694 SelectPostStore(
Node, 3, AArch64::ST1Threev2s_POST);
6696 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6697 SelectPostStore(
Node, 3, AArch64::ST1Threev4s_POST);
6699 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6700 SelectPostStore(
Node, 3, AArch64::ST1Threev1d_POST);
6702 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6703 SelectPostStore(
Node, 3, AArch64::ST1Threev2d_POST);
6709 VT =
Node->getOperand(1).getValueType();
6710 if (VT == MVT::v8i8) {
6711 SelectPostStore(
Node, 4, AArch64::ST1Fourv8b_POST);
6713 }
else if (VT == MVT::v16i8) {
6714 SelectPostStore(
Node, 4, AArch64::ST1Fourv16b_POST);
6716 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
6717 SelectPostStore(
Node, 4, AArch64::ST1Fourv4h_POST);
6719 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
6720 SelectPostStore(
Node, 4, AArch64::ST1Fourv8h_POST);
6722 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6723 SelectPostStore(
Node, 4, AArch64::ST1Fourv2s_POST);
6725 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6726 SelectPostStore(
Node, 4, AArch64::ST1Fourv4s_POST);
6728 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6729 SelectPostStore(
Node, 4, AArch64::ST1Fourv1d_POST);
6731 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6732 SelectPostStore(
Node, 4, AArch64::ST1Fourv2d_POST);
6738 VT =
Node->getOperand(1).getValueType();
6739 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6740 SelectPostStoreLane(
Node, 2, AArch64::ST2i8_POST);
6742 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6743 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6744 SelectPostStoreLane(
Node, 2, AArch64::ST2i16_POST);
6746 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6748 SelectPostStoreLane(
Node, 2, AArch64::ST2i32_POST);
6750 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6752 SelectPostStoreLane(
Node, 2, AArch64::ST2i64_POST);
6758 VT =
Node->getOperand(1).getValueType();
6759 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6760 SelectPostStoreLane(
Node, 3, AArch64::ST3i8_POST);
6762 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6763 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6764 SelectPostStoreLane(
Node, 3, AArch64::ST3i16_POST);
6766 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6768 SelectPostStoreLane(
Node, 3, AArch64::ST3i32_POST);
6770 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6772 SelectPostStoreLane(
Node, 3, AArch64::ST3i64_POST);
6778 VT =
Node->getOperand(1).getValueType();
6779 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
6780 SelectPostStoreLane(
Node, 4, AArch64::ST4i8_POST);
6782 }
else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
6783 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
6784 SelectPostStoreLane(
Node, 4, AArch64::ST4i16_POST);
6786 }
else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
6788 SelectPostStoreLane(
Node, 4, AArch64::ST4i32_POST);
6790 }
else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
6792 SelectPostStoreLane(
Node, 4, AArch64::ST4i64_POST);
6798 if (VT == MVT::nxv16i8) {
6799 SelectPredicatedLoad(
Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B);
6801 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6802 VT == MVT::nxv8bf16) {
6803 SelectPredicatedLoad(
Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H);
6805 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6806 SelectPredicatedLoad(
Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W);
6808 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6809 SelectPredicatedLoad(
Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D);
6815 if (VT == MVT::nxv16i8) {
6816 SelectPredicatedLoad(
Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B);
6818 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6819 VT == MVT::nxv8bf16) {
6820 SelectPredicatedLoad(
Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H);
6822 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6823 SelectPredicatedLoad(
Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W);
6825 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6826 SelectPredicatedLoad(
Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D);
6832 if (VT == MVT::nxv16i8) {
6833 SelectPredicatedLoad(
Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B);
6835 }
else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6836 VT == MVT::nxv8bf16) {
6837 SelectPredicatedLoad(
Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H);
6839 }
else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6840 SelectPredicatedLoad(
Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W);
6842 }
else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6843 SelectPredicatedLoad(
Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D);
6858 return new AArch64DAGToDAGISel(
TM, OptLevel);
6870 assert(NumVec > 0 && NumVec < 5 &&
"Invalid number of vectors.");
6874 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
6875 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
6889 if (isa<MemSDNode>(Root))
6890 return cast<MemSDNode>(Root)->getMemoryVT();
6892 if (isa<MemIntrinsicSDNode>(Root))
6893 return cast<MemIntrinsicSDNode>(Root)->getMemoryVT();
6895 const unsigned Opcode = Root->
getOpcode();
6903 return cast<VTSDNode>(Root->
getOperand(3))->getVT();
6905 return cast<VTSDNode>(Root->
getOperand(4))->getVT();
6925 case Intrinsic::aarch64_sme_ldr:
6926 case Intrinsic::aarch64_sme_str:
6927 return MVT::nxv16i8;
6928 case Intrinsic::aarch64_sve_prf:
6933 case Intrinsic::aarch64_sve_ld2_sret:
6934 case Intrinsic::aarch64_sve_ld2q_sret:
6937 case Intrinsic::aarch64_sve_st2q:
6940 case Intrinsic::aarch64_sve_ld3_sret:
6941 case Intrinsic::aarch64_sve_ld3q_sret:
6944 case Intrinsic::aarch64_sve_st3q:
6947 case Intrinsic::aarch64_sve_ld4_sret:
6948 case Intrinsic::aarch64_sve_ld4q_sret:
6951 case Intrinsic::aarch64_sve_st4q:
6954 case Intrinsic::aarch64_sve_ld1udq:
6955 case Intrinsic::aarch64_sve_st1dq:
6956 return EVT(MVT::nxv1i64);
6957 case Intrinsic::aarch64_sve_ld1uwq:
6958 case Intrinsic::aarch64_sve_st1wq:
6959 return EVT(MVT::nxv1i32);
6966template <
int64_t Min,
int64_t Max>
6967bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(
SDNode *Root,
SDValue N,
6975 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
6980 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
6999 int64_t MulImm = cast<ConstantSDNode>(VScale.
getOperand(0))->getSExtValue();
7001 if ((MulImm % MemWidthBytes) != 0)
7004 int64_t
Offset = MulImm / MemWidthBytes;
7005 if (Offset < Min || Offset > Max)
7008 Base =
N.getOperand(0);
7010 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
7017 OffImm = CurDAG->getTargetConstant(
Offset,
SDLoc(
N), MVT::i64);
7023bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(
SDValue N,
unsigned Scale,
7041 if (
auto C = dyn_cast<ConstantSDNode>(RHS)) {
7042 int64_t ImmOff =
C->getSExtValue();
7043 unsigned Size = 1 << Scale;
7052 Offset = CurDAG->getTargetConstant(ImmOff >> Scale,
DL, MVT::i64);
7054 SDNode *
MI = CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
7064 if (
auto *
C = dyn_cast<ConstantSDNode>(ShiftRHS))
7065 if (
C->getZExtValue() == Scale) {
7074bool AArch64DAGToDAGISel::SelectAllActivePredicate(
SDValue N) {
7081bool AArch64DAGToDAGISel::SelectAnyPredicate(
SDValue N) {
7082 EVT VT =
N.getValueType();
7086bool AArch64DAGToDAGISel::SelectSMETileSlice(
SDValue N,
unsigned MaxSize,
7091 if (
auto C = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
7092 int64_t ImmOff =
C->getSExtValue();
7093 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0))) {
7094 Base =
N.getOperand(0);
7095 Offset = CurDAG->getTargetConstant(ImmOff / Scale,
SDLoc(
N), MVT::i64);
7102 Offset = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
static int getIntOperandFromRegisterString(StringRef RegString)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
Does DstMask form a complementary pair with the mask provided by BitsToBeInserted,...
static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N)
Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32,...
static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op, SDValue &Src, int &DstLSB, int &Width)
static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &DstLSB, int &Width)
Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL,...
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG)
static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
static bool isShiftedMask(uint64_t Mask, EVT VT)
bool SelectSMETile(unsigned &BaseReg, unsigned TileNum)
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root)
Return the EVT of the data associated to a memory operation in Root.
static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static bool isWorthFoldingADDlow(SDValue N)
If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD i...
static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N)
getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef< unsigned > Opcodes)
This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bi...
static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT, unsigned NumVec)
When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent sca...
static bool isPreferredADD(int64_t ImmOff)
static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
Create a machine node performing a notional SHL of Op by ShlAmount.
static bool isWorthFoldingSHL(SDValue V)
Determine whether it is worth it to fold SHL into the addressing mode.
static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1, SDValue Src, SDValue Dst, SelectionDAG *CurDAG, const bool BiggerPattern)
static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits, unsigned Depth)
static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth=0)
static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected)
static AArch64_AM::ShiftExtendType getExtendTypeForNode(SDValue N, bool IsLoadStore=false)
getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
static bool isIntImmediate(const SDNode *N, uint64_t &Imm)
isIntImmediate - This method tests to see if the node is a constant operand.
static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG, SDValue &ShiftedOperand, uint64_t &EncodedShiftImm)
static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range, unsigned Size)
Check if the immediate offset is valid as a scaled immediate.
static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool isLittleEndian() const
bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const
bool getExactInverse(APFloat *inv) const
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
void flipAllBits()
Toggle every bit to its opposite value.
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
An arbitrary precision integer that knows its signedness.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
const Constant * getConstVal() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
FunctionPass class - This class is used to implement most global optimizations.
int64_t getOffset() const
const GlobalValue * getGlobal() const
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint8_t getStackID(int ObjectIdx) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
static constexpr unsigned MaxRecursionDepth
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
unsigned getID() const
Return the register class ID number.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A Use represents the edge between a Value definition and its users.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
uint32_t parseGenericRegister(StringRef Name)
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static bool processLogicalImmediate(uint64_t Imm, unsigned RegSize, uint64_t &Encoding)
processLogicalImmediate - Determine if an immediate value can be encoded as the immediate operand of ...
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static constexpr unsigned SVEBitsPerBlock
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
@ Undef
Value of the register doesn't matter.
Not(const Pred &P) -> Not< Pred >
Reg
All possible values of the reg field in the ModR/M byte.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
CodeGenOptLevel
Code generation optimization level.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
FunctionPass * createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOptLevel OptLevel)
createAArch64ISelDag - This pass converts a legalized DAG into a AArch64-specific DAG,...
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
constexpr unsigned BitWidth
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
unsigned getBitWidth() const
Get the bit width of this value.