22#include "llvm/IR/IntrinsicsAArch64.h"
31#define DEBUG_TYPE "aarch64-isel"
32#define PASS_NAME "AArch64 Instruction Selection"
35#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
36#pragma inline_depth(0)
52 AArch64DAGToDAGISel() =
delete;
69 std::vector<SDValue> &OutOps)
override;
71 template <
signed Low,
signed High,
signed Scale>
79 return SelectShiftedRegister(
N,
false, Reg, Shift);
82 return SelectShiftedRegister(
N,
true, Reg, Shift);
85 return SelectAddrModeIndexed7S(
N, 1,
Base, OffImm);
88 return SelectAddrModeIndexed7S(
N, 2,
Base, OffImm);
91 return SelectAddrModeIndexed7S(
N, 4,
Base, OffImm);
94 return SelectAddrModeIndexed7S(
N, 8,
Base, OffImm);
97 return SelectAddrModeIndexed7S(
N, 16,
Base, OffImm);
100 return SelectAddrModeIndexedBitWidth(
N,
true, 9, 16,
Base, OffImm);
103 return SelectAddrModeIndexedBitWidth(
N,
false, 6, 16,
Base, OffImm);
106 return SelectAddrModeIndexed(
N, 1,
Base, OffImm);
109 return SelectAddrModeIndexed(
N, 2,
Base, OffImm);
112 return SelectAddrModeIndexed(
N, 4,
Base, OffImm);
115 return SelectAddrModeIndexed(
N, 8,
Base, OffImm);
118 return SelectAddrModeIndexed(
N, 16,
Base, OffImm);
121 return SelectAddrModeUnscaled(
N, 1,
Base, OffImm);
124 return SelectAddrModeUnscaled(
N, 2,
Base, OffImm);
127 return SelectAddrModeUnscaled(
N, 4,
Base, OffImm);
130 return SelectAddrModeUnscaled(
N, 8,
Base, OffImm);
133 return SelectAddrModeUnscaled(
N, 16,
Base, OffImm);
135 template <
unsigned Size,
unsigned Max>
139 bool Found = SelectAddrModeIndexed(
N,
Size,
Base, OffImm);
141 if (
auto *CI = dyn_cast<ConstantSDNode>(OffImm)) {
142 int64_t
C = CI->getSExtValue();
150 OffImm = CurDAG->getTargetConstant(0,
SDLoc(
N), MVT::i64);
157 return SelectAddrModeWRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
163 return SelectAddrModeXRO(
N, Width / 8,
Base,
Offset, SignExtend, DoShift);
168 N =
N->getOperand(0);
170 !isa<ConstantSDNode>(
N->getOperand(1)))
172 EVT VT =
N->getValueType(0);
173 EVT LVT =
N->getOperand(0).getValueType();
174 unsigned Index =
N->getConstantOperandVal(1);
178 Res =
N->getOperand(0);
186 EVT VT =
Op.getValueType();
187 unsigned ShtAmt =
N->getConstantOperandVal(1);
194 Op.getOperand(1).getConstantOperandVal(0)
195 <<
Op.getOperand(1).getConstantOperandVal(1));
197 isa<ConstantSDNode>(
Op.getOperand(1).getOperand(0)))
199 Op.getOperand(1).getConstantOperandVal(0));
203 if (Imm != 1ULL << (ShtAmt - 1))
206 Res1 =
Op.getOperand(0);
207 Res2 = CurDAG->getTargetConstant(ShtAmt,
SDLoc(
N), MVT::i32);
211 bool SelectDupZeroOrUndef(
SDValue N) {
212 switch(
N->getOpcode()) {
217 auto Opnd0 =
N->getOperand(0);
231 bool SelectAny(
SDValue) {
return true; }
234 switch(
N->getOpcode()) {
237 auto Opnd0 =
N->getOperand(0);
249 template<MVT::SimpleValueType VT>
251 return SelectSVEAddSubImm(
N, VT, Imm, Shift);
254 template <MVT::SimpleValueType VT,
bool Negate>
256 return SelectSVEAddSubSSatImm(
N, VT, Imm, Shift, Negate);
259 template <MVT::SimpleValueType VT>
261 return SelectSVECpyDupImm(
N, VT, Imm, Shift);
264 template <MVT::SimpleValueType VT,
bool Invert = false>
266 return SelectSVELogicalImm(
N, VT, Imm, Invert);
269 template <MVT::SimpleValueType VT>
271 return SelectSVEArithImm(
N, VT, Imm);
274 template <
unsigned Low,
unsigned High,
bool AllowSaturation = false>
276 return SelectSVEShiftImm(
N,
Low,
High, AllowSaturation, Imm);
283 EVT EltVT =
N->getValueType(0).getVectorElementType();
284 return SelectSVEShiftImm(
N->getOperand(0), 1,
290 template<
signed Min,
signed Max,
signed Scale,
bool Shift>
292 if (!isa<ConstantSDNode>(
N))
295 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
297 MulImm = 1LL << MulImm;
299 if ((MulImm % std::abs(Scale)) != 0)
303 if ((MulImm >= Min) && (MulImm <= Max)) {
304 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
311 template <
signed Max,
signed Scale>
313 if (!isa<ConstantSDNode>(
N))
316 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
318 if (MulImm >= 0 && MulImm <= Max) {
320 Imm = CurDAG->getTargetConstant(MulImm,
SDLoc(
N), MVT::i32);
327 template <
unsigned BaseReg,
unsigned Max>
329 if (
auto *CI = dyn_cast<ConstantSDNode>(
N)) {
335 Imm = CurDAG->getRegister(BaseReg +
C, MVT::Other);
358 const unsigned SubRegs[]);
360 void SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
bool isExt);
362 bool tryIndexedLoad(
SDNode *
N);
364 void SelectPtrauthAuth(
SDNode *
N);
365 void SelectPtrauthResign(
SDNode *
N);
367 bool trySelectStackSlotTagP(
SDNode *
N);
370 void SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
372 void SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
374 void SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
375 void SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
376 void SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
377 unsigned Opc_rr,
unsigned Opc_ri,
378 bool IsIntr =
false);
379 void SelectContiguousMultiVectorLoad(
SDNode *
N,
unsigned NumVecs,
380 unsigned Scale,
unsigned Opc_ri,
382 void SelectDestructiveMultiIntrinsic(
SDNode *
N,
unsigned NumVecs,
383 bool IsZmMulti,
unsigned Opcode,
384 bool HasPred =
false);
385 void SelectPExtPair(
SDNode *
N,
unsigned Opc);
386 void SelectWhilePair(
SDNode *
N,
unsigned Opc);
387 void SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
388 void SelectCVTIntrinsicFP8(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
389 void SelectClamp(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
390 void SelectUnaryMultiIntrinsic(
SDNode *
N,
unsigned NumOutVecs,
391 bool IsTupleInput,
unsigned Opc);
392 void SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
unsigned Opcode);
394 template <
unsigned MaxIdx,
unsigned Scale>
395 void SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
unsigned BaseReg,
397 void SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
398 unsigned Op,
unsigned MaxIdx,
unsigned Scale,
399 unsigned BaseReg = 0);
402 template <
int64_t Min,
int64_t Max>
406 template <
unsigned Scale>
408 return SelectSVERegRegAddrMode(
N, Scale,
Base,
Offset);
411 void SelectMultiVectorLutiLane(
SDNode *
Node,
unsigned NumOutVecs,
414 void SelectMultiVectorLuti(
SDNode *
Node,
unsigned NumOutVecs,
unsigned Opc);
416 template <
unsigned MaxIdx,
unsigned Scale>
421 void SelectStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
422 void SelectPostStore(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
423 void SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
424 void SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
unsigned Opc);
425 void SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
unsigned Scale,
426 unsigned Opc_rr,
unsigned Opc_ri);
427 std::tuple<unsigned, SDValue, SDValue>
428 findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
unsigned Opc_ri,
432 bool tryBitfieldExtractOp(
SDNode *
N);
433 bool tryBitfieldExtractOpFromSExt(
SDNode *
N);
434 bool tryBitfieldInsertOp(
SDNode *
N);
435 bool tryBitfieldInsertInZeroOp(
SDNode *
N);
436 bool tryShiftAmountMod(
SDNode *
N);
438 bool tryReadRegister(
SDNode *
N);
439 bool tryWriteRegister(
SDNode *
N);
441 bool trySelectCastFixedLengthToScalableVector(
SDNode *
N);
442 bool trySelectCastScalableToFixedLengthVector(
SDNode *
N);
447#include "AArch64GenDAGISel.inc"
455 return SelectAddrModeIndexedBitWidth(
N,
true, 7,
Size,
Base, OffImm);
457 bool SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
unsigned BW,
470 bool isWorthFoldingALU(
SDValue V,
bool LSL =
false)
const;
471 bool isWorthFoldingAddr(
SDValue V,
unsigned Size)
const;
472 bool SelectExtendedSHL(
SDValue N,
unsigned Size,
bool WantExtend,
475 template<
unsigned RegW
idth>
477 return SelectCVTFixedPosOperand(
N, FixedPos, RegWidth);
480 bool SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
unsigned Width);
482 template<
unsigned RegW
idth>
484 return SelectCVTFixedPosRecipOperand(
N, FixedPos, RegWidth);
490 bool SelectCMP_SWAP(
SDNode *
N);
500 bool AllowSaturation,
SDValue &Imm);
508 bool SelectAllActivePredicate(
SDValue N);
518 ID, std::make_unique<AArch64DAGToDAGISel>(tm, OptLevel)) {}
522char AArch64DAGToDAGISelLegacy::ID = 0;
530 Imm =
C->getZExtValue();
547 return N->getOpcode() == Opc &&
558 return Imm == ImmExpected;
562bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
564 std::vector<SDValue> &OutOps) {
565 switch(ConstraintID) {
568 case InlineAsm::ConstraintCode::m:
569 case InlineAsm::ConstraintCode::o:
570 case InlineAsm::ConstraintCode::Q:
576 SDValue RC = CurDAG->getTargetConstant(TRC->
getID(), dl, MVT::i64);
578 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
579 dl,
Op.getValueType(),
581 OutOps.push_back(NewOp);
597 if (!isa<ConstantSDNode>(
N.getNode()))
600 uint64_t Immed =
N.getNode()->getAsZExtVal();
603 if (Immed >> 12 == 0) {
605 }
else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
613 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
614 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
627 if (!isa<ConstantSDNode>(
N.getNode()))
631 uint64_t Immed =
N.getNode()->getAsZExtVal();
639 if (
N.getValueType() == MVT::i32)
642 Immed = ~Immed + 1ULL;
643 if (Immed & 0xFFFFFFFFFF000000ULL)
646 Immed &= 0xFFFFFFULL;
647 return SelectArithImmed(CurDAG->getConstant(Immed,
SDLoc(
N), MVT::i32), Val,
654 switch (
N.getOpcode()) {
673 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
676 unsigned ShiftVal = CSD->getZExtValue();
685 if (!isa<MemSDNode>(*UI))
687 if (!isa<MemSDNode>(*UII))
694bool AArch64DAGToDAGISel::isWorthFoldingAddr(
SDValue V,
unsigned Size)
const {
697 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
702 if (Subtarget->hasAddrLSLSlow14() && (
Size == 2 ||
Size == 16))
724bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(
SDValue N,
SDValue &Reg,
726 EVT VT =
N.getValueType();
727 if (VT != MVT::i32 && VT != MVT::i64)
730 if (
N->getOpcode() !=
ISD::AND || !
N->hasOneUse())
736 unsigned LHSOpcode =
LHS->getOpcode();
750 unsigned LowZBits, MaskLen;
754 unsigned BitWidth =
N.getValueSizeInBits();
761 if (LowZBits <= ShiftAmtC || (
BitWidth != LowZBits + MaskLen))
764 NewShiftC = LowZBits - ShiftAmtC;
765 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
771 NewShiftC = LowZBits + ShiftAmtC;
784 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
786 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
790 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC,
DL, VT);
792 Reg =
SDValue(CurDAG->getMachineNode(NewShiftOp,
DL, VT,
LHS->getOperand(0),
793 NewShiftAmt, BitWidthMinus1),
796 Shift = CurDAG->getTargetConstant(ShVal,
DL, MVT::i32);
808 SrcVT = cast<VTSDNode>(
N.getOperand(1))->getVT();
810 SrcVT =
N.getOperand(0).getValueType();
812 if (!IsLoadStore && SrcVT == MVT::i8)
814 else if (!IsLoadStore && SrcVT == MVT::i16)
816 else if (SrcVT == MVT::i32)
818 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
823 EVT SrcVT =
N.getOperand(0).getValueType();
824 if (!IsLoadStore && SrcVT == MVT::i8)
826 else if (!IsLoadStore && SrcVT == MVT::i16)
828 else if (SrcVT == MVT::i32)
830 assert(SrcVT != MVT::i64 &&
"extend from 64-bits?");
858bool AArch64DAGToDAGISel::isWorthFoldingALU(
SDValue V,
bool LSL)
const {
861 if (CurDAG->shouldOptForSize() ||
V.hasOneUse())
866 if (LSL && Subtarget->hasALULSLFast() &&
V.getOpcode() ==
ISD::SHL &&
867 V.getConstantOperandVal(1) <= 4 &&
880bool AArch64DAGToDAGISel::SelectShiftedRegister(
SDValue N,
bool AllowROR,
882 if (SelectShiftedRegisterFromAnd(
N, Reg, Shift))
892 unsigned BitSize =
N.getValueSizeInBits();
893 unsigned Val =
RHS->getZExtValue() & (BitSize - 1);
896 Reg =
N.getOperand(0);
897 Shift = CurDAG->getTargetConstant(ShVal,
SDLoc(
N), MVT::i32);
898 return isWorthFoldingALU(
N,
true);
909 if (
N.getValueType() == MVT::i32)
917template<
signed Low,
signed High,
signed Scale>
919 if (!isa<ConstantSDNode>(
N))
922 int64_t MulImm = cast<ConstantSDNode>(
N)->getSExtValue();
923 if ((MulImm % std::abs(Scale)) == 0) {
924 int64_t RDVLImm = MulImm / Scale;
925 if ((RDVLImm >=
Low) && (RDVLImm <=
High)) {
926 Imm = CurDAG->getSignedTargetConstant(RDVLImm,
SDLoc(
N), MVT::i32);
936bool AArch64DAGToDAGISel::SelectArithExtendedRegister(
SDValue N,
SDValue &Reg,
938 unsigned ShiftVal = 0;
953 Reg =
N.getOperand(0).getOperand(0);
959 Reg =
N.getOperand(0);
964 unsigned Opc =
N.getOpcode();
965 return Opc !=
ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
982 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
984 return isWorthFoldingALU(
N);
989bool AArch64DAGToDAGISel::SelectArithUXTXRegister(
SDValue N,
SDValue &Reg,
991 unsigned ShiftVal = 0;
1005 Reg =
N.getOperand(0);
1006 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal),
SDLoc(
N),
1008 return isWorthFoldingALU(
N);
1017 for (
auto *
User :
N->users()) {
1044bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(
SDValue N,
bool IsSignedImm,
1045 unsigned BW,
unsigned Size,
1052 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1054 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1060 if (CurDAG->isBaseWithConstantOffset(
N)) {
1061 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1063 int64_t RHSC =
RHS->getSExtValue();
1065 int64_t
Range = 0x1LL << (BW - 1);
1067 if ((RHSC & (
Size - 1)) == 0 && RHSC >= -(
Range << Scale) &&
1068 RHSC < (
Range << Scale)) {
1069 Base =
N.getOperand(0);
1071 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1074 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1083 if ((RHSC & (
Size - 1)) == 0 && RHSC < (
Range << Scale)) {
1084 Base =
N.getOperand(0);
1086 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1089 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1100 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1107bool AArch64DAGToDAGISel::SelectAddrModeIndexed(
SDValue N,
unsigned Size,
1113 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
1115 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1121 dyn_cast<GlobalAddressSDNode>(
N.getOperand(1).getNode());
1122 Base =
N.getOperand(0);
1123 OffImm =
N.getOperand(1);
1132 if (CurDAG->isBaseWithConstantOffset(
N)) {
1133 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1134 int64_t RHSC = (int64_t)
RHS->getZExtValue();
1137 Base =
N.getOperand(0);
1139 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1142 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1150 if (SelectAddrModeUnscaled(
N,
Size,
Base, OffImm))
1158 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1167bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(
SDValue N,
unsigned Size,
1170 if (!CurDAG->isBaseWithConstantOffset(
N))
1172 if (
ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(
N.getOperand(1))) {
1173 int64_t RHSC =
RHS->getSExtValue();
1174 if (RHSC >= -256 && RHSC < 256) {
1175 Base =
N.getOperand(0);
1177 int FI = cast<FrameIndexSDNode>(
Base)->getIndex();
1179 Base = CurDAG->getTargetFrameIndex(
1182 OffImm = CurDAG->getTargetConstant(RHSC,
SDLoc(
N), MVT::i64);
1192 CurDAG->
getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1199bool AArch64DAGToDAGISel::SelectExtendedSHL(
SDValue N,
unsigned Size,
1219 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1225 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1228 return isWorthFoldingAddr(
N,
Size);
1231bool AArch64DAGToDAGISel::SelectAddrModeWRO(
SDValue N,
unsigned Size,
1243 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
1251 if (!isa<MemSDNode>(*UI))
1256 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1259 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1260 SelectExtendedSHL(RHS,
Size,
true,
Offset, SignExtend)) {
1262 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1267 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1268 SelectExtendedSHL(LHS,
Size,
true,
Offset, SignExtend)) {
1270 DoShift = CurDAG->getTargetConstant(
true, dl, MVT::i32);
1275 DoShift = CurDAG->getTargetConstant(
false, dl, MVT::i32);
1279 if (IsExtendedRegisterWorthFolding &&
1286 if (isWorthFoldingAddr(LHS,
Size))
1291 if (IsExtendedRegisterWorthFolding &&
1298 if (isWorthFoldingAddr(RHS,
Size))
1310 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1313 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1315 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1316 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1320bool AArch64DAGToDAGISel::SelectAddrModeXRO(
SDValue N,
unsigned Size,
1335 if (!isa<MemSDNode>(*UI))
1350 if (isa<ConstantSDNode>(RHS)) {
1351 int64_t ImmOff = (int64_t)
RHS->getAsZExtVal();
1361 CurDAG->getMachineNode(AArch64::MOVi64imm,
DL, MVT::i64, Ops);
1364 N = CurDAG->getNode(
ISD::ADD,
DL, MVT::i64, LHS, MOVIV);
1368 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(
N,
Size);
1371 if (IsExtendedRegisterWorthFolding &&
RHS.getOpcode() ==
ISD::SHL &&
1372 SelectExtendedSHL(RHS,
Size,
false,
Offset, SignExtend)) {
1374 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1379 if (IsExtendedRegisterWorthFolding &&
LHS.getOpcode() ==
ISD::SHL &&
1380 SelectExtendedSHL(LHS,
Size,
false,
Offset, SignExtend)) {
1382 DoShift = CurDAG->getTargetConstant(
true,
DL, MVT::i32);
1389 SignExtend = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1390 DoShift = CurDAG->getTargetConstant(
false,
DL, MVT::i32);
1396 static const unsigned RegClassIDs[] = {
1397 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1398 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1399 AArch64::dsub2, AArch64::dsub3};
1405 static const unsigned RegClassIDs[] = {
1406 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1407 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1408 AArch64::qsub2, AArch64::qsub3};
1414 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1415 AArch64::ZPR3RegClassID,
1416 AArch64::ZPR4RegClassID};
1417 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1418 AArch64::zsub2, AArch64::zsub3};
1428 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1429 AArch64::ZPR4Mul4RegClassID};
1430 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1431 AArch64::zsub2, AArch64::zsub3};
1436 const unsigned RegClassIDs[],
1437 const unsigned SubRegs[]) {
1440 if (Regs.
size() == 1)
1451 CurDAG->getTargetConstant(RegClassIDs[Regs.
size() - 2],
DL, MVT::i32));
1454 for (
unsigned i = 0; i < Regs.
size(); ++i) {
1456 Ops.
push_back(CurDAG->getTargetConstant(SubRegs[i],
DL, MVT::i32));
1460 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped, Ops);
1464void AArch64DAGToDAGISel::SelectTable(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1467 EVT VT =
N->getValueType(0);
1469 unsigned ExtOff = isExt;
1472 unsigned Vec0Off = ExtOff + 1;
1474 N->op_begin() + Vec0Off + NumVecs);
1481 Ops.
push_back(
N->getOperand(NumVecs + ExtOff + 1));
1482 ReplaceNode(
N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1485static std::tuple<SDValue, SDValue>
1505 auto *ConstDiscN = dyn_cast<ConstantSDNode>(ConstDisc);
1506 if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue()))
1511 AddrDisc = DAG->
getRegister(AArch64::XZR, MVT::i64);
1513 return std::make_tuple(
1518void AArch64DAGToDAGISel::SelectPtrauthAuth(
SDNode *
N) {
1523 SDValue AUTDisc =
N->getOperand(3);
1525 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1526 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
1528 SDValue AUTAddrDisc, AUTConstDisc;
1529 std::tie(AUTConstDisc, AUTAddrDisc) =
1532 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
1533 AArch64::X16, Val,
SDValue());
1534 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.
getValue(1)};
1536 SDNode *AUT = CurDAG->getMachineNode(AArch64::AUT,
DL, MVT::i64, Ops);
1537 ReplaceNode(
N, AUT);
1540void AArch64DAGToDAGISel::SelectPtrauthResign(
SDNode *
N) {
1545 SDValue AUTDisc =
N->getOperand(3);
1547 SDValue PACDisc =
N->getOperand(5);
1549 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1550 unsigned PACKeyC = cast<ConstantSDNode>(PACKey)->getZExtValue();
1552 AUTKey = CurDAG->getTargetConstant(AUTKeyC,
DL, MVT::i64);
1553 PACKey = CurDAG->getTargetConstant(PACKeyC,
DL, MVT::i64);
1555 SDValue AUTAddrDisc, AUTConstDisc;
1556 std::tie(AUTConstDisc, AUTAddrDisc) =
1559 SDValue PACAddrDisc, PACConstDisc;
1560 std::tie(PACConstDisc, PACAddrDisc) =
1563 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(),
DL,
1564 AArch64::X16, Val,
SDValue());
1566 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1567 PACConstDisc, PACAddrDisc, X16Copy.
getValue(1)};
1569 SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC,
DL, MVT::i64, Ops);
1570 ReplaceNode(
N, AUTPAC);
1573bool AArch64DAGToDAGISel::tryIndexedLoad(
SDNode *
N) {
1575 if (
LD->isUnindexed())
1577 EVT VT =
LD->getMemoryVT();
1578 EVT DstVT =
N->getValueType(0);
1585 unsigned Opcode = 0;
1588 bool InsertTo64 =
false;
1590 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1591 else if (VT == MVT::i32) {
1593 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1595 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1597 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1603 }
else if (VT == MVT::i16) {
1605 if (DstVT == MVT::i64)
1606 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1608 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1610 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1611 InsertTo64 = DstVT == MVT::i64;
1616 }
else if (VT == MVT::i8) {
1618 if (DstVT == MVT::i64)
1619 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1621 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1623 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1624 InsertTo64 = DstVT == MVT::i64;
1629 }
else if (VT == MVT::f16) {
1630 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1631 }
else if (VT == MVT::bf16) {
1632 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1633 }
else if (VT == MVT::f32) {
1634 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1636 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1638 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1646 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1648 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1653 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {
MemOp});
1658 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1660 SDValue(CurDAG->getMachineNode(
1661 AArch64::SUBREG_TO_REG, dl, MVT::i64,
1662 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1667 ReplaceUses(
SDValue(
N, 0), LoadedVal);
1670 CurDAG->RemoveDeadNode(
N);
1674void AArch64DAGToDAGISel::SelectLoad(
SDNode *
N,
unsigned NumVecs,
unsigned Opc,
1675 unsigned SubRegIdx) {
1677 EVT VT =
N->getValueType(0);
1683 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1685 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1687 for (
unsigned i = 0; i < NumVecs; ++i)
1689 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1695 if (
auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(
N)) {
1697 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
1700 CurDAG->RemoveDeadNode(
N);
1703void AArch64DAGToDAGISel::SelectPostLoad(
SDNode *
N,
unsigned NumVecs,
1704 unsigned Opc,
unsigned SubRegIdx) {
1706 EVT VT =
N->getValueType(0);
1713 const EVT ResTys[] = {MVT::i64,
1714 MVT::Untyped, MVT::Other};
1716 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1724 ReplaceUses(
SDValue(
N, 0), SuperReg);
1726 for (
unsigned i = 0; i < NumVecs; ++i)
1728 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1732 CurDAG->RemoveDeadNode(
N);
1738std::tuple<unsigned, SDValue, SDValue>
1739AArch64DAGToDAGISel::findAddrModeSVELoadStore(
SDNode *
N,
unsigned Opc_rr,
1745 SDValue NewOffset = OldOffset;
1747 const bool IsRegImm = SelectAddrModeIndexedSVE<-8, 7>(
1748 N, OldBase, NewBase, NewOffset);
1752 const bool IsRegReg =
1753 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
1756 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
1769template <SelectTypeKind Kind>
1781 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
1786 if (EltVT != MVT::i1)
1790 if (EltVT == MVT::bf16)
1792 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
1822void AArch64DAGToDAGISel::SelectPExtPair(
SDNode *
N,
unsigned Opc) {
1825 if (
Imm->getZExtValue() > 1)
1829 EVT VT =
N->getValueType(0);
1830 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1831 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1834 for (
unsigned I = 0;
I < 2; ++
I)
1835 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1836 AArch64::psub0 +
I,
DL, VT, SuperReg));
1838 CurDAG->RemoveDeadNode(
N);
1841void AArch64DAGToDAGISel::SelectWhilePair(
SDNode *
N,
unsigned Opc) {
1843 EVT VT =
N->getValueType(0);
1845 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2)};
1847 SDNode *WhilePair = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
1850 for (
unsigned I = 0;
I < 2; ++
I)
1851 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
1852 AArch64::psub0 +
I,
DL, VT, SuperReg));
1854 CurDAG->RemoveDeadNode(
N);
1857void AArch64DAGToDAGISel::SelectCVTIntrinsic(
SDNode *
N,
unsigned NumVecs,
1859 EVT VT =
N->getValueType(0);
1861 SDValue Ops = createZTuple(Regs);
1865 for (
unsigned i = 0; i < NumVecs; ++i)
1866 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1867 AArch64::zsub0 + i,
DL, VT, SuperReg));
1869 CurDAG->RemoveDeadNode(
N);
1872void AArch64DAGToDAGISel::SelectCVTIntrinsicFP8(
SDNode *
N,
unsigned NumVecs,
1875 EVT VT =
N->getValueType(0);
1877 Ops.push_back(
N->getOperand(0));
1880 CurDAG->getMachineNode(Opcode,
DL, {MVT::Untyped, MVT::Other}, Ops);
1883 for (
unsigned i = 0; i < NumVecs; ++i)
1884 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1885 AArch64::zsub0 + i,
DL, VT, SuperReg));
1888 unsigned ChainIdx = NumVecs;
1890 CurDAG->RemoveDeadNode(
N);
1893void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(
SDNode *
N,
1898 assert(Opcode != 0 &&
"Unexpected opcode");
1901 EVT VT =
N->getValueType(0);
1902 unsigned FirstVecIdx = HasPred ? 2 : 1;
1904 auto GetMultiVecOperand = [=](
unsigned StartIdx) {
1906 return createZMulTuple(Regs);
1909 SDValue Zdn = GetMultiVecOperand(FirstVecIdx);
1913 Zm = GetMultiVecOperand(NumVecs + FirstVecIdx);
1915 Zm =
N->getOperand(NumVecs + FirstVecIdx);
1919 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped,
1920 N->getOperand(1), Zdn, Zm);
1922 Intrinsic = CurDAG->getMachineNode(Opcode,
DL, MVT::Untyped, Zdn, Zm);
1924 for (
unsigned i = 0; i < NumVecs; ++i)
1925 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1926 AArch64::zsub0 + i,
DL, VT, SuperReg));
1928 CurDAG->RemoveDeadNode(
N);
1931void AArch64DAGToDAGISel::SelectPredicatedLoad(
SDNode *
N,
unsigned NumVecs,
1932 unsigned Scale,
unsigned Opc_ri,
1933 unsigned Opc_rr,
bool IsIntr) {
1934 assert(Scale < 5 &&
"Invalid scaling value.");
1936 EVT VT =
N->getValueType(0);
1942 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
1943 N, Opc_rr, Opc_ri,
N->getOperand(IsIntr ? 3 : 2),
1944 CurDAG->getTargetConstant(0,
DL, MVT::i64), Scale);
1946 SDValue Ops[] = {
N->getOperand(IsIntr ? 2 : 1),
1950 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1952 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1954 for (
unsigned i = 0; i < NumVecs; ++i)
1955 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1956 AArch64::zsub0 + i,
DL, VT, SuperReg));
1959 unsigned ChainIdx = NumVecs;
1961 CurDAG->RemoveDeadNode(
N);
1964void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(
SDNode *
N,
1969 assert(Scale < 4 &&
"Invalid scaling value.");
1971 EVT VT =
N->getValueType(0);
1979 findAddrModeSVELoadStore(
N, Opc_rr, Opc_ri,
Base,
Offset, Scale);
1985 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1987 SDNode *
Load = CurDAG->getMachineNode(Opc,
DL, ResTys, Ops);
1989 for (
unsigned i = 0; i < NumVecs; ++i)
1990 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
1991 AArch64::zsub0 + i,
DL, VT, SuperReg));
1994 unsigned ChainIdx = NumVecs;
1996 CurDAG->RemoveDeadNode(
N);
1999void AArch64DAGToDAGISel::SelectFrintFromVT(
SDNode *
N,
unsigned NumVecs,
2001 if (
N->getValueType(0) != MVT::nxv4f32)
2003 SelectUnaryMultiIntrinsic(
N, NumVecs,
true, Opcode);
2006void AArch64DAGToDAGISel::SelectMultiVectorLutiLane(
SDNode *
Node,
2007 unsigned NumOutVecs,
2011 if (
Imm->getZExtValue() > MaxImm)
2015 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
2020 EVT VT =
Node->getValueType(0);
2023 CurDAG->getMachineNode(Opc,
DL, {MVT::Untyped, MVT::Other}, Ops);
2026 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
2027 ReplaceUses(
SDValue(
Node,
I), CurDAG->getTargetExtractSubreg(
2028 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2031 unsigned ChainIdx = NumOutVecs;
2033 CurDAG->RemoveDeadNode(
Node);
2036void AArch64DAGToDAGISel::SelectMultiVectorLuti(
SDNode *
Node,
2037 unsigned NumOutVecs,
2042 if (!ImmToReg<AArch64::ZT0, 0>(
Node->getOperand(2), ZtValue))
2048 EVT VT =
Node->getValueType(0);
2051 CurDAG->getMachineNode(Opc,
DL, {MVT::Untyped, MVT::Other}, Ops);
2054 for (
unsigned I = 0;
I < NumOutVecs; ++
I)
2055 ReplaceUses(
SDValue(
Node,
I), CurDAG->getTargetExtractSubreg(
2056 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2059 unsigned ChainIdx = NumOutVecs;
2061 CurDAG->RemoveDeadNode(
Node);
2064void AArch64DAGToDAGISel::SelectClamp(
SDNode *
N,
unsigned NumVecs,
2067 EVT VT =
N->getValueType(0);
2070 SDValue Zd = createZMulTuple(Regs);
2071 SDValue Zn =
N->getOperand(1 + NumVecs);
2072 SDValue Zm =
N->getOperand(2 + NumVecs);
2078 for (
unsigned i = 0; i < NumVecs; ++i)
2079 ReplaceUses(
SDValue(
N, i), CurDAG->getTargetExtractSubreg(
2080 AArch64::zsub0 + i,
DL, VT, SuperReg));
2082 CurDAG->RemoveDeadNode(
N);
2112template <
unsigned MaxIdx,
unsigned Scale>
2113void AArch64DAGToDAGISel::SelectMultiVectorMove(
SDNode *
N,
unsigned NumVecs,
2114 unsigned BaseReg,
unsigned Op) {
2115 unsigned TileNum = 0;
2116 if (BaseReg != AArch64::ZA)
2117 TileNum =
N->getConstantOperandVal(2);
2123 if (BaseReg == AArch64::ZA)
2124 SliceBase =
N->getOperand(2);
2126 SliceBase =
N->getOperand(3);
2128 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2134 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
2136 EVT VT =
N->getValueType(0);
2137 for (
unsigned I = 0;
I < NumVecs; ++
I)
2139 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2142 unsigned ChainIdx = NumVecs;
2144 CurDAG->RemoveDeadNode(
N);
2147void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(
SDNode *
N,
unsigned NumVecs,
2148 unsigned Op,
unsigned MaxIdx,
2149 unsigned Scale,
unsigned BaseReg) {
2153 SDValue SliceBase =
N->getOperand(2);
2154 if (BaseReg != AArch64::ZA)
2155 SliceBase =
N->getOperand(3);
2158 if (!SelectSMETileSlice(SliceBase, MaxIdx,
Base,
Offset, Scale))
2165 if (BaseReg != AArch64::ZA )
2170 SDNode *Mov = CurDAG->getMachineNode(
Op,
DL, {MVT::Untyped, MVT::Other}, Ops);
2172 EVT VT =
N->getValueType(0);
2173 for (
unsigned I = 0;
I < NumVecs; ++
I)
2175 CurDAG->getTargetExtractSubreg(AArch64::zsub0 +
I,
DL, VT,
2179 unsigned ChainIdx = NumVecs;
2181 CurDAG->RemoveDeadNode(
N);
2184void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(
SDNode *
N,
2185 unsigned NumOutVecs,
2189 EVT VT =
N->getValueType(0);
2190 unsigned NumInVecs =
N->getNumOperands() - 1;
2194 assert((NumInVecs == 2 || NumInVecs == 4) &&
2195 "Don't know how to handle multi-register input!");
2200 for (
unsigned I = 0;
I < NumInVecs;
I++)
2204 SDNode *Res = CurDAG->getMachineNode(Opc,
DL, MVT::Untyped, Ops);
2207 for (
unsigned I = 0;
I < NumOutVecs;
I++)
2208 ReplaceUses(
SDValue(
N,
I), CurDAG->getTargetExtractSubreg(
2209 AArch64::zsub0 +
I,
DL, VT, SuperReg));
2210 CurDAG->RemoveDeadNode(
N);
2213void AArch64DAGToDAGISel::SelectStore(
SDNode *
N,
unsigned NumVecs,
2216 EVT VT =
N->getOperand(2)->getValueType(0);
2223 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
N->getOperand(0)};
2224 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2228 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2233void AArch64DAGToDAGISel::SelectPredicatedStore(
SDNode *
N,
unsigned NumVecs,
2234 unsigned Scale,
unsigned Opc_rr,
2240 SDValue RegSeq = createZTuple(Regs);
2245 std::tie(Opc,
Base,
Offset) = findAddrModeSVELoadStore(
2246 N, Opc_rr, Opc_ri,
N->getOperand(NumVecs + 3),
2247 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2249 SDValue Ops[] = {RegSeq,
N->getOperand(NumVecs + 2),
2253 SDNode *St = CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), Ops);
2265 if (
auto FINode = dyn_cast<FrameIndexSDNode>(
N)) {
2266 int FI = FINode->getIndex();
2268 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
2275void AArch64DAGToDAGISel::SelectPostStore(
SDNode *
N,
unsigned NumVecs,
2278 EVT VT =
N->getOperand(2)->getValueType(0);
2279 const EVT ResTys[] = {MVT::i64,
2288 N->getOperand(NumVecs + 1),
2289 N->getOperand(NumVecs + 2),
2291 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2331void AArch64DAGToDAGISel::SelectLoadLane(
SDNode *
N,
unsigned NumVecs,
2334 EVT VT =
N->getValueType(0);
2342 WidenVector(*CurDAG));
2346 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2348 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2350 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2351 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2352 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2356 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2357 AArch64::qsub2, AArch64::qsub3 };
2358 for (
unsigned i = 0; i < NumVecs; ++i) {
2359 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2366 CurDAG->RemoveDeadNode(
N);
2369void AArch64DAGToDAGISel::SelectPostLoadLane(
SDNode *
N,
unsigned NumVecs,
2372 EVT VT =
N->getValueType(0);
2380 WidenVector(*CurDAG));
2384 const EVT ResTys[] = {MVT::i64,
2387 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2390 CurDAG->getTargetConstant(LaneNo, dl,
2392 N->getOperand(NumVecs + 2),
2393 N->getOperand(NumVecs + 3),
2395 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2407 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2408 AArch64::qsub2, AArch64::qsub3 };
2409 for (
unsigned i = 0; i < NumVecs; ++i) {
2410 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
2420 CurDAG->RemoveDeadNode(
N);
2423void AArch64DAGToDAGISel::SelectStoreLane(
SDNode *
N,
unsigned NumVecs,
2426 EVT VT =
N->getOperand(2)->getValueType(0);
2434 WidenVector(*CurDAG));
2438 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 2);
2440 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2441 N->getOperand(NumVecs + 3),
N->getOperand(0)};
2442 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
2446 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2451void AArch64DAGToDAGISel::SelectPostStoreLane(
SDNode *
N,
unsigned NumVecs,
2454 EVT VT =
N->getOperand(2)->getValueType(0);
2462 WidenVector(*CurDAG));
2466 const EVT ResTys[] = {MVT::i64,
2469 unsigned LaneNo =
N->getConstantOperandVal(NumVecs + 1);
2471 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2472 N->getOperand(NumVecs + 2),
2473 N->getOperand(NumVecs + 3),
2475 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2479 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
2486 unsigned &LSB,
unsigned &MSB,
2487 unsigned NumberOfIgnoredLowBits,
2488 bool BiggerPattern) {
2490 "N must be a AND operation to call this function");
2492 EVT VT =
N->getValueType(0);
2497 assert((VT == MVT::i32 || VT == MVT::i64) &&
2498 "Type checking must have been done before calling this function");
2512 const SDNode *Op0 =
N->getOperand(0).getNode();
2516 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
2519 if (AndImm & (AndImm + 1))
2522 bool ClampMSB =
false;
2542 ClampMSB = (VT == MVT::i32);
2543 }
else if (BiggerPattern) {
2549 Opd0 =
N->getOperand(0);
2555 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.
getSizeInBits())) {
2558 <<
": Found large shift immediate, this should not happen\n"));
2564 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(AndImm)
2565 : llvm::countr_one<uint64_t>(AndImm)) -
2572 MSB = MSB > 31 ? 31 : MSB;
2574 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2579 SDValue &Opd0,
unsigned &Immr,
2583 EVT VT =
N->getValueType(0);
2585 assert((VT == MVT::i32 || VT == MVT::i64) &&
2586 "Type checking must have been done before calling this function");
2590 Op =
Op->getOperand(0);
2591 VT =
Op->getValueType(0);
2600 unsigned Width = cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
2604 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2605 Opd0 =
Op.getOperand(0);
2607 Imms = ShiftImm + Width - 1;
2635 Opd0 =
N->getOperand(0).getOperand(0);
2645 Opc =
N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2652 unsigned &Immr,
unsigned &Imms,
2653 bool BiggerPattern) {
2655 "N must be a SHR/SRA operation to call this function");
2657 EVT VT =
N->getValueType(0);
2662 assert((VT == MVT::i32 || VT == MVT::i64) &&
2663 "Type checking must have been done before calling this function");
2673 Opd0 =
N->getOperand(0).getOperand(0);
2674 }
else if (VT == MVT::i32 &&
N->getOpcode() ==
ISD::SRL &&
2680 Opd0 =
N->getOperand(0).getOperand(0);
2683 assert(VT == MVT::i64 &&
"the promoted type should be i64");
2684 }
else if (BiggerPattern) {
2688 Opd0 =
N->getOperand(0);
2697 <<
": Found large shift immediate, this should not happen\n"));
2706 "bad amount in shift node!");
2707 int immr = SrlImm - ShlImm;
2712 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
2714 Opc =
N->getOpcode() ==
ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
2718bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(
SDNode *
N) {
2721 EVT VT =
N->getValueType(0);
2722 EVT NarrowVT =
N->getOperand(0)->getValueType(0);
2723 if (VT != MVT::i64 || NarrowVT != MVT::i32)
2734 unsigned Immr = ShiftImm;
2736 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2737 CurDAG->getTargetConstant(Imms, dl, VT)};
2738 CurDAG->SelectNodeTo(
N, AArch64::SBFMXri, VT, Ops);
2743 SDValue &Opd0,
unsigned &Immr,
unsigned &Imms,
2744 unsigned NumberOfIgnoredLowBits = 0,
2745 bool BiggerPattern =
false) {
2746 if (
N->getValueType(0) != MVT::i32 &&
N->getValueType(0) != MVT::i64)
2749 switch (
N->getOpcode()) {
2751 if (!
N->isMachineOpcode())
2756 NumberOfIgnoredLowBits, BiggerPattern);
2765 unsigned NOpc =
N->getMachineOpcode();
2769 case AArch64::SBFMWri:
2770 case AArch64::UBFMWri:
2771 case AArch64::SBFMXri:
2772 case AArch64::UBFMXri:
2774 Opd0 =
N->getOperand(0);
2775 Immr =
N->getConstantOperandVal(1);
2776 Imms =
N->getConstantOperandVal(2);
2783bool AArch64DAGToDAGISel::tryBitfieldExtractOp(
SDNode *
N) {
2784 unsigned Opc, Immr, Imms;
2789 EVT VT =
N->getValueType(0);
2794 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
2795 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
2796 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
2798 SDNode *
BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
2799 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
2805 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2806 CurDAG->getTargetConstant(Imms, dl, VT)};
2807 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
2816 unsigned NumberOfIgnoredHighBits,
EVT VT) {
2817 assert((VT == MVT::i32 || VT == MVT::i64) &&
2818 "i32 or i64 mask type expected!");
2822 APInt SignificantDstMask =
2826 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
2827 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
2850 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2860 APInt OpUsefulBits(UsefulBits);
2864 OpUsefulBits <<= MSB - Imm + 1;
2869 OpUsefulBits <<= Imm;
2871 OpUsefulBits <<= MSB + 1;
2874 OpUsefulBits <<= OpUsefulBits.
getBitWidth() - Imm;
2880 UsefulBits &= OpUsefulBits;
2886 cast<const ConstantSDNode>(
Op.getOperand(1).getNode())->getZExtValue();
2888 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2896 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2897 APInt Mask(UsefulBits);
2898 Mask.clearAllBits();
2906 Mask.lshrInPlace(ShiftAmt);
2912 Mask.lshrInPlace(ShiftAmt);
2924 cast<const ConstantSDNode>(
Op.getOperand(2).getNode())->getZExtValue();
2926 cast<const ConstantSDNode>(
Op.getOperand(3).getNode())->getZExtValue();
2928 APInt OpUsefulBits(UsefulBits);
2942 OpUsefulBits <<= Width;
2945 if (
Op.getOperand(1) == Orig) {
2947 Mask = ResultUsefulBits & OpUsefulBits;
2951 if (
Op.getOperand(0) == Orig)
2953 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2959 OpUsefulBits <<= Width;
2961 OpUsefulBits <<= LSB;
2963 if (
Op.getOperand(1) == Orig) {
2965 Mask = ResultUsefulBits & OpUsefulBits;
2966 Mask.lshrInPlace(LSB);
2969 if (
Op.getOperand(0) == Orig)
2970 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2987 case AArch64::ANDSWri:
2988 case AArch64::ANDSXri:
2989 case AArch64::ANDWri:
2990 case AArch64::ANDXri:
2994 case AArch64::UBFMWri:
2995 case AArch64::UBFMXri:
2998 case AArch64::ORRWrs:
2999 case AArch64::ORRXrs:
3004 case AArch64::BFMWri:
3005 case AArch64::BFMXri:
3008 case AArch64::STRBBui:
3009 case AArch64::STURBBi:
3015 case AArch64::STRHHui:
3016 case AArch64::STURHHi:
3029 unsigned Bitwidth =
Op.getScalarValueSizeInBits();
3031 UsefulBits =
APInt(Bitwidth, 0);
3040 UsersUsefulBits |= UsefulBitsForUse;
3045 UsefulBits &= UsersUsefulBits;
3055 EVT VT =
Op.getValueType();
3058 unsigned UBFMOpc =
BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3061 if (ShlAmount > 0) {
3064 UBFMOpc, dl, VT,
Op,
3069 assert(ShlAmount < 0 &&
"expected right shift");
3070 int ShrAmount = -ShlAmount;
3096 bool BiggerPattern,
SDValue &Src,
3097 int &DstLSB,
int &Width) {
3098 EVT VT =
Op.getValueType();
3107 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
3111 switch (
Op.getOpcode()) {
3116 NonZeroBits, Src, DstLSB, Width);
3119 NonZeroBits, Src, DstLSB, Width);
3132 EVT VT =
Op.getValueType();
3133 assert((VT == MVT::i32 || VT == MVT::i64) &&
3134 "Caller guarantees VT is one of i32 or i64");
3147 assert((~AndImm & NonZeroBits) == 0 &&
3148 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3177 if (!BiggerPattern && !AndOp0.
hasOneUse())
3196 <<
"Found large Width in bit-field-positioning -- this indicates no "
3197 "proper combining / constant folding was performed\n");
3206 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3221 "Op.getNode() should be a SHL node to call this function");
3223 "Op.getNode() should shift ShlImm to call this function");
3230 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3254 EVT VT =
Op.getValueType();
3255 assert((VT == MVT::i32 || VT == MVT::i64) &&
3256 "Caller guarantees that type is i32 or i64");
3263 if (!BiggerPattern && !
Op.hasOneUse())
3272 if (ShlImm !=
uint64_t(DstLSB) && !BiggerPattern)
3280 assert(VT == MVT::i32 || VT == MVT::i64);
3291 EVT VT =
N->getValueType(0);
3292 if (VT != MVT::i32 && VT != MVT::i64)
3310 if (!
And.hasOneUse() ||
3320 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3327 if ((OrImm & NotKnownZero) != 0) {
3339 unsigned ImmS = Width - 1;
3345 bool IsBFI = LSB != 0;
3350 unsigned OrChunks = 0, BFIChunks = 0;
3351 for (
unsigned Shift = 0; Shift <
BitWidth; Shift += 16) {
3352 if (((OrImm >> Shift) & 0xFFFF) != 0)
3354 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3357 if (BFIChunks > OrChunks)
3363 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3371 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3380 if (!Dst.hasOneUse())
3383 EVT VT = Dst.getValueType();
3384 assert((VT == MVT::i32 || VT == MVT::i64) &&
3385 "Caller should guarantee that VT is one of i32 or i64");
3393 SDValue DstOp0 = Dst.getOperand(0);
3413 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3414 unsigned MaskWidth =
3417 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3423 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1,
DL, VT));
3424 ShiftedOperand =
SDValue(UBFMNode, 0);
3434 ShiftedOperand = Dst.getOperand(0);
3441 ShiftedOperand = Dst.getOperand(0);
3453 const bool BiggerPattern) {
3454 EVT VT =
N->getValueType(0);
3455 assert(
N->getOpcode() ==
ISD::OR &&
"Expect N to be an OR node");
3456 assert(((
N->getOperand(0) == OrOpd0 &&
N->getOperand(1) == OrOpd1) ||
3457 (
N->getOperand(1) == OrOpd0 &&
N->getOperand(0) == OrOpd1)) &&
3458 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3459 assert((VT == MVT::i32 || VT == MVT::i64) &&
3460 "Expect result type to be i32 or i64 since N is combinable to BFM");
3467 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3470 if (BiggerPattern) {
3484 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3493 assert((!BiggerPattern) &&
"BiggerPattern should be handled above");
3555 EVT VT =
N->getValueType(0);
3556 if (VT != MVT::i32 && VT != MVT::i64)
3564 unsigned NumberOfIgnoredLowBits = UsefulBits.
countr_zero();
3565 unsigned NumberOfIgnoredHighBits = UsefulBits.
countl_zero();
3585 for (
int I = 0;
I < 4; ++
I) {
3588 unsigned ImmR, ImmS;
3589 bool BiggerPattern =
I / 2;
3590 SDValue OrOpd0Val =
N->getOperand(
I % 2);
3592 SDValue OrOpd1Val =
N->getOperand((
I + 1) % 2);
3598 NumberOfIgnoredLowBits, BiggerPattern)) {
3601 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3602 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3607 Width = ImmS - ImmR + 1;
3618 Src, DstLSB, Width)) {
3626 assert((VT == MVT::i32 || VT == MVT::i64) &&
"unexpected OR operand");
3636 APInt BitsToBeInserted =
3639 if ((BitsToBeInserted & ~Known.
Zero) != 0)
3663 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3696 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3698 if (Src->hasOneUse() &&
3701 Src = Src->getOperand(0);
3711 unsigned ImmS = Width - 1;
3717 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3725bool AArch64DAGToDAGISel::tryBitfieldInsertOp(
SDNode *
N) {
3734 CurDAG->SelectNodeTo(
N, TargetOpcode::IMPLICIT_DEF,
N->getValueType(0));
3747bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(
SDNode *
N) {
3751 EVT VT =
N->getValueType(0);
3752 if (VT != MVT::i32 && VT != MVT::i64)
3758 Op0, DstLSB, Width))
3764 unsigned ImmS = Width - 1;
3767 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR,
DL, VT),
3768 CurDAG->getTargetConstant(ImmS,
DL, VT)};
3769 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3770 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3776bool AArch64DAGToDAGISel::tryShiftAmountMod(
SDNode *
N) {
3777 EVT VT =
N->getValueType(0);
3780 switch (
N->getOpcode()) {
3782 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
3785 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
3788 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
3791 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
3799 if (VT == MVT::i32) {
3802 }
else if (VT == MVT::i64) {
3808 SDValue ShiftAmt =
N->getOperand(1);
3828 (Add0Imm %
Size == 0)) {
3834 if (SubVT == MVT::i32) {
3835 NegOpc = AArch64::SUBWrr;
3836 ZeroReg = AArch64::WZR;
3838 assert(SubVT == MVT::i64);
3839 NegOpc = AArch64::SUBXrr;
3840 ZeroReg = AArch64::XZR;
3843 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3845 CurDAG->getMachineNode(NegOpc,
DL, SubVT, Zero, Add1);
3846 NewShiftAmt =
SDValue(Neg, 0);
3854 if (SubVT == MVT::i32) {
3855 NotOpc = AArch64::ORNWrr;
3856 ZeroReg = AArch64::WZR;
3858 assert(SubVT == MVT::i64);
3859 NotOpc = AArch64::ORNXrr;
3860 ZeroReg = AArch64::XZR;
3863 CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
DL, ZeroReg, SubVT);
3865 CurDAG->getMachineNode(NotOpc,
DL, SubVT, Zero, Add1);
3866 NewShiftAmt =
SDValue(Not, 0);
3887 else if (VT == MVT::i64 && NewShiftAmt->
getValueType(0) == MVT::i32) {
3888 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32,
DL, MVT::i32);
3890 AArch64::SUBREG_TO_REG,
DL, VT,
3891 CurDAG->getTargetConstant(0,
DL, MVT::i64), NewShiftAmt,
SubReg);
3892 NewShiftAmt =
SDValue(Ext, 0);
3895 SDValue Ops[] = {
N->getOperand(0), NewShiftAmt};
3896 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
3903 bool isReciprocal) {
3906 FVal = CN->getValueAPF();
3907 else if (
LoadSDNode *LN = dyn_cast<LoadSDNode>(
N)) {
3910 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
3914 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
3915 FVal = cast<ConstantFP>(CN->
getConstVal())->getValueAPF();
3938 if (!IsExact || !IntVal.isPowerOf2())
3940 unsigned FBits = IntVal.logBase2();
3944 if (FBits == 0 || FBits > RegWidth)
return false;
3950bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(
SDValue N,
SDValue &FixedPos,
3951 unsigned RegWidth) {
3956bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(
SDValue N,
3958 unsigned RegWidth) {
3968 RegString.
split(Fields,
':');
3970 if (Fields.
size() == 1)
3974 &&
"Invalid number of fields in read register string");
3977 bool AllIntFields =
true;
3981 AllIntFields &= !
Field.getAsInteger(10, IntField);
3986 "Unexpected non-integer value in special register string.");
3991 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
3992 (Ops[3] << 3) | (Ops[4]);
3999bool AArch64DAGToDAGISel::tryReadRegister(
SDNode *
N) {
4000 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
4001 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
4006 unsigned Opcode64Bit = AArch64::MRS;
4011 const auto *TheReg =
4013 if (TheReg && TheReg->Readable &&
4014 TheReg->haveFeatures(Subtarget->getFeatureBits()))
4015 Imm = TheReg->Encoding;
4021 if (!ReadIs128Bit && RegString->getString() ==
"pc") {
4022 Opcode64Bit = AArch64::ADR;
4030 SDValue InChain =
N->getOperand(0);
4031 SDValue SysRegImm = CurDAG->getTargetConstant(Imm,
DL, MVT::i32);
4032 if (!ReadIs128Bit) {
4033 CurDAG->SelectNodeTo(
N, Opcode64Bit, MVT::i64, MVT::Other ,
4034 {SysRegImm, InChain});
4038 {MVT::Untyped , MVT::Other },
4039 {SysRegImm, InChain});
4043 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64,
DL, MVT::i64,
4045 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64,
DL, MVT::i64,
4051 ReplaceUses(
SDValue(
N, 2), OutChain);
4060bool AArch64DAGToDAGISel::tryWriteRegister(
SDNode *
N) {
4061 const auto *MD = cast<MDNodeSDNode>(
N->getOperand(1));
4062 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
4067 if (!WriteIs128Bit) {
4073 auto trySelectPState = [&](
auto PMapper,
unsigned State) {
4075 assert(isa<ConstantSDNode>(
N->getOperand(2)) &&
4076 "Expected a constant integer expression.");
4077 unsigned Reg = PMapper->Encoding;
4078 uint64_t Immed =
N->getConstantOperandVal(2);
4079 CurDAG->SelectNodeTo(
4080 N, State, MVT::Other, CurDAG->getTargetConstant(Reg,
DL, MVT::i32),
4081 CurDAG->getTargetConstant(Immed,
DL, MVT::i16),
N->getOperand(0));
4087 if (trySelectPState(
4088 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
4089 AArch64::MSRpstateImm4))
4091 if (trySelectPState(
4092 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
4093 AArch64::MSRpstateImm1))
4103 if (TheReg && TheReg->Writeable &&
4104 TheReg->haveFeatures(Subtarget->getFeatureBits()))
4105 Imm = TheReg->Encoding;
4113 SDValue InChain =
N->getOperand(0);
4114 if (!WriteIs128Bit) {
4115 CurDAG->SelectNodeTo(
N, AArch64::MSR, MVT::Other,
4116 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
4117 N->getOperand(2), InChain);
4121 SDNode *Pair = CurDAG->getMachineNode(
4122 TargetOpcode::REG_SEQUENCE,
DL, MVT::Untyped ,
4123 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(),
DL,
4126 CurDAG->getTargetConstant(AArch64::sube64,
DL, MVT::i32),
4128 CurDAG->getTargetConstant(AArch64::subo64,
DL, MVT::i32)});
4130 CurDAG->SelectNodeTo(
N, AArch64::MSRR, MVT::Other,
4131 CurDAG->getTargetConstant(Imm,
DL, MVT::i32),
4139bool AArch64DAGToDAGISel::SelectCMP_SWAP(
SDNode *
N) {
4141 EVT MemTy = cast<MemSDNode>(
N)->getMemoryVT();
4144 if (Subtarget->hasLSE())
return false;
4146 if (MemTy == MVT::i8)
4147 Opcode = AArch64::CMP_SWAP_8;
4148 else if (MemTy == MVT::i16)
4149 Opcode = AArch64::CMP_SWAP_16;
4150 else if (MemTy == MVT::i32)
4151 Opcode = AArch64::CMP_SWAP_32;
4152 else if (MemTy == MVT::i64)
4153 Opcode = AArch64::CMP_SWAP_64;
4157 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4158 SDValue Ops[] = {
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
4160 SDNode *CmpSwap = CurDAG->getMachineNode(
4162 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
4165 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {
MemOp});
4169 CurDAG->RemoveDeadNode(
N);
4176 if (!isa<ConstantSDNode>(
N))
4188 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4189 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4196 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4197 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4201 if (Val <= 65280 && Val % 256 == 0) {
4202 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4203 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4214bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(
SDValue N,
MVT VT,
4217 if (!isa<ConstantSDNode>(
N))
4221 int64_t Val = cast<ConstantSDNode>(
N)
4238 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4239 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4246 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4247 Imm = CurDAG->getTargetConstant(Val,
DL, MVT::i32);
4251 if (Val <= 65280 && Val % 256 == 0) {
4252 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4253 Imm = CurDAG->getTargetConstant(Val >> 8,
DL, MVT::i32);
4266 if (!isa<ConstantSDNode>(
N))
4270 int64_t Val = cast<ConstantSDNode>(
N)
4278 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4279 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4285 if (Val >= -128 && Val <= 127) {
4286 Shift = CurDAG->getTargetConstant(0,
DL, MVT::i32);
4287 Imm = CurDAG->getTargetConstant(Val & 0xFF,
DL, MVT::i32);
4291 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) {
4292 Shift = CurDAG->getTargetConstant(8,
DL, MVT::i32);
4293 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF,
DL, MVT::i32);
4304bool AArch64DAGToDAGISel::SelectSVESignedArithImm(
SDValue N,
SDValue &Imm) {
4305 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4306 int64_t ImmVal = CNode->getSExtValue();
4308 if (ImmVal >= -128 && ImmVal < 128) {
4309 Imm = CurDAG->getSignedTargetConstant(ImmVal,
DL, MVT::i32);
4317 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4318 uint64_t ImmVal = CNode->getZExtValue();
4328 ImmVal &= 0xFFFFFFFF;
4337 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4346 if (
auto CNode = dyn_cast<ConstantSDNode>(
N)) {
4347 uint64_t ImmVal = CNode->getZExtValue();
4357 ImmVal |= ImmVal << 8;
4358 ImmVal |= ImmVal << 16;
4359 ImmVal |= ImmVal << 32;
4363 ImmVal |= ImmVal << 16;
4364 ImmVal |= ImmVal << 32;
4367 ImmVal &= 0xFFFFFFFF;
4368 ImmVal |= ImmVal << 32;
4378 Imm = CurDAG->getTargetConstant(encoding,
DL, MVT::i64);
4393 if (
auto *CN = dyn_cast<ConstantSDNode>(
N)) {
4394 uint64_t ImmVal = CN->getZExtValue();
4401 if (ImmVal >
High) {
4402 if (!AllowSaturation)
4407 Imm = CurDAG->getTargetConstant(ImmVal,
SDLoc(
N), MVT::i32);
4414bool AArch64DAGToDAGISel::trySelectStackSlotTagP(
SDNode *
N) {
4418 if (!(isa<FrameIndexSDNode>(
N->getOperand(1)))) {
4430 int FI = cast<FrameIndexSDNode>(
N->getOperand(1))->getIndex();
4431 SDValue FiOp = CurDAG->getTargetFrameIndex(
4433 int TagOffset =
N->getConstantOperandVal(3);
4435 SDNode *Out = CurDAG->getMachineNode(
4436 AArch64::TAGPstack,
DL, MVT::i64,
4437 {FiOp, CurDAG->getTargetConstant(0,
DL, MVT::i64),
N->getOperand(2),
4438 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4439 ReplaceNode(
N, Out);
4443void AArch64DAGToDAGISel::SelectTagP(
SDNode *
N) {
4444 assert(isa<ConstantSDNode>(
N->getOperand(3)) &&
4445 "llvm.aarch64.tagp third argument must be an immediate");
4446 if (trySelectStackSlotTagP(
N))
4453 int TagOffset =
N->getConstantOperandVal(3);
4454 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP,
DL, MVT::i64,
4455 {
N->getOperand(1),
N->getOperand(2)});
4456 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr,
DL, MVT::i64,
4457 {
SDValue(N1, 0),
N->getOperand(2)});
4458 SDNode *N3 = CurDAG->getMachineNode(
4459 AArch64::ADDG,
DL, MVT::i64,
4460 {
SDValue(N2, 0), CurDAG->getTargetConstant(0,
DL, MVT::i64),
4461 CurDAG->getTargetConstant(TagOffset,
DL, MVT::i64)});
4465bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(
SDNode *
N) {
4469 if (
N->getConstantOperandVal(2) != 0)
4471 if (!
N->getOperand(0).isUndef())
4475 EVT VT =
N->getValueType(0);
4476 EVT InVT =
N->getOperand(1).getValueType();
4487 "Expected to insert into a packed scalable vector!");
4490 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4491 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4492 N->getOperand(1), RC));
4496bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(
SDNode *
N) {
4500 if (
N->getConstantOperandVal(1) != 0)
4504 EVT VT =
N->getValueType(0);
4505 EVT InVT =
N->getOperand(0).getValueType();
4516 "Expected to extract from a packed scalable vector!");
4519 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID,
DL, MVT::i64);
4520 ReplaceNode(
N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
DL, VT,
4521 N->getOperand(0), RC));
4525bool AArch64DAGToDAGISel::trySelectXAR(
SDNode *
N) {
4530 EVT VT =
N->getValueType(0);
4542 (Subtarget->hasSVE2() ||
4543 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4552 if (!TLI->isAllActivePredicate(*CurDAG, N0.
getOperand(0)) ||
4553 !TLI->isAllActivePredicate(*CurDAG, N1.
getOperand(0)))
4560 APInt ShlAmt, ShrAmt;
4573 if (
auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
4574 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
4575 AArch64::XAR_ZZZI_D})) {
4576 CurDAG->SelectNodeTo(
N, Opc, VT, Ops);
4582 if (!Subtarget->hasSHA3())
4604 if (ShAmt + HsAmt != 64)
4608 CurDAG->SelectNodeTo(
N, AArch64::XAR, N0.
getValueType(), Ops);
4613void AArch64DAGToDAGISel::Select(
SDNode *
Node) {
4615 if (
Node->isMachineOpcode()) {
4617 Node->setNodeId(-1);
4622 EVT VT =
Node->getValueType(0);
4624 switch (
Node->getOpcode()) {
4629 if (SelectCMP_SWAP(
Node))
4635 if (tryReadRegister(
Node))
4641 if (tryWriteRegister(
Node))
4648 if (tryIndexedLoad(
Node))
4657 if (tryBitfieldExtractOp(
Node))
4659 if (tryBitfieldInsertInZeroOp(
Node))
4664 if (tryShiftAmountMod(
Node))
4669 if (tryBitfieldExtractOpFromSExt(
Node))
4674 if (tryBitfieldInsertOp(
Node))
4676 if (trySelectXAR(
Node))
4681 if (trySelectCastScalableToFixedLengthVector(
Node))
4687 if (trySelectCastFixedLengthToScalableVector(
Node))
4696 if (ConstNode->
isZero()) {
4697 if (VT == MVT::i32) {
4699 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::WZR, MVT::i32);
4700 ReplaceNode(
Node,
New.getNode());
4702 }
else if (VT == MVT::i64) {
4704 CurDAG->getEntryNode(),
SDLoc(
Node), AArch64::XZR, MVT::i64);
4705 ReplaceNode(
Node,
New.getNode());
4714 int FI = cast<FrameIndexSDNode>(
Node)->getIndex();
4717 SDValue TFI = CurDAG->getTargetFrameIndex(
4720 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0,
DL, MVT::i32),
4721 CurDAG->getTargetConstant(Shifter,
DL, MVT::i32) };
4722 CurDAG->SelectNodeTo(
Node, AArch64::ADDXri, MVT::i64, Ops);
4726 unsigned IntNo =
Node->getConstantOperandVal(1);
4730 case Intrinsic::aarch64_gcsss: {
4734 SDValue Zero = CurDAG->getCopyFromReg(Chain,
DL, AArch64::XZR, MVT::i64);
4736 CurDAG->getMachineNode(AArch64::GCSSS1,
DL, MVT::Other, Val, Chain);
4737 SDNode *SS2 = CurDAG->getMachineNode(AArch64::GCSSS2,
DL, MVT::i64,
4738 MVT::Other, Zero,
SDValue(SS1, 0));
4739 ReplaceNode(
Node, SS2);
4742 case Intrinsic::aarch64_ldaxp:
4743 case Intrinsic::aarch64_ldxp: {
4745 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
4750 SDNode *Ld = CurDAG->getMachineNode(
Op,
DL, MVT::i64, MVT::i64,
4751 MVT::Other, MemAddr, Chain);
4755 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4756 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {
MemOp});
4757 ReplaceNode(
Node, Ld);
4760 case Intrinsic::aarch64_stlxp:
4761 case Intrinsic::aarch64_stxp: {
4763 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
4771 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
4773 SDNode *St = CurDAG->getMachineNode(
Op,
DL, MVT::i32, MVT::Other, Ops);
4776 cast<MemIntrinsicSDNode>(
Node)->getMemOperand();
4777 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {
MemOp});
4779 ReplaceNode(
Node, St);
4782 case Intrinsic::aarch64_neon_ld1x2:
4783 if (VT == MVT::v8i8) {
4784 SelectLoad(
Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
4786 }
else if (VT == MVT::v16i8) {
4787 SelectLoad(
Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
4789 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4790 SelectLoad(
Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
4792 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4793 SelectLoad(
Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
4795 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4796 SelectLoad(
Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
4798 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4799 SelectLoad(
Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
4801 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4802 SelectLoad(
Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
4804 }
else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4805 SelectLoad(
Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
4809 case Intrinsic::aarch64_neon_ld1x3:
4810 if (VT == MVT::v8i8) {
4811 SelectLoad(
Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
4813 }
else if (VT == MVT::v16i8) {
4814 SelectLoad(
Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
4816 }
else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
4817 SelectLoad(
Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
4819 }
else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
4820 SelectLoad(
Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
4822 }
else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4823 SelectLoad(
Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
4825 }
else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4826 SelectLoad(
Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
4828 }
else if (VT == MVT::v1i64 || VT == MVT::v1f64) {