61#define GET_INSTRINFO_CTOR_DTOR
62#include "AArch64GenInstrInfo.inc"
66 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
70 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
74 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
78 cl::desc(
"Restrict range of B instructions (DEBUG)"));
83 RI(STI.getTargetTriple()), Subtarget(STI) {}
94 auto Op =
MI.getOpcode();
95 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
96 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
100 if (
MI.isMetaInstruction())
105 unsigned NumBytes = 0;
111 switch (
Desc.getOpcode()) {
114 return Desc.getSize();
121 case TargetOpcode::STACKMAP:
124 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
126 case TargetOpcode::PATCHPOINT:
129 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
131 case TargetOpcode::STATEPOINT:
133 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
138 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
143 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
145 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
146 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
150 case TargetOpcode::PATCHABLE_EVENT_CALL:
156 NumBytes =
MI.getOperand(1).getImm();
158 case TargetOpcode::BUNDLE:
159 NumBytes = getInstBundleLength(
MI);
166unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
170 while (++
I != E &&
I->isInsideBundle()) {
171 assert(!
I->isBundle() &&
"No nested bundle!");
230 int64_t BrOffset)
const {
232 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
233 "over conditional branch expansion");
234 return isIntN(Bits, BrOffset / 4);
239 switch (
MI.getOpcode()) {
243 return MI.getOperand(0).getMBB();
248 return MI.getOperand(2).getMBB();
254 return MI.getOperand(1).getMBB();
264 assert(RS &&
"RegScavenger required for long branching");
266 "new block should be inserted for expanding unconditional branch");
269 "restore block should be inserted for restoring clobbered registers");
274 if (!isInt<33>(BrOffset))
276 "Branch offsets outside of the signed 33-bit range not supported");
290 constexpr Register Reg = AArch64::X16;
292 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
300 if (Scavenged != AArch64::NoRegister &&
302 buildIndirectBranch(Scavenged, NewDestBB);
312 "Unable to insert indirect branch inside function that has red zone");
335 bool AllowModify)
const {
342 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
343 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
347 if (!isUnpredicatedTerminator(*
I))
354 unsigned LastOpc = LastInst->
getOpcode();
355 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
370 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
377 LastInst = SecondLastInst;
379 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
384 SecondLastInst = &*
I;
385 SecondLastOpc = SecondLastInst->
getOpcode();
396 LastInst = SecondLastInst;
398 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
400 "unreachable unconditional branches removed above");
409 SecondLastInst = &*
I;
410 SecondLastOpc = SecondLastInst->
getOpcode();
414 if (SecondLastInst &&
I !=
MBB.
begin() && isUnpredicatedTerminator(*--
I))
430 I->eraseFromParent();
439 I->eraseFromParent();
448 MachineBranchPredicate &MBP,
449 bool AllowModify)
const {
459 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
460 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
464 if (!isUnpredicatedTerminator(*
I))
469 unsigned LastOpc = LastInst->
getOpcode();
484 assert(MBP.TrueDest &&
"expected!");
487 MBP.ConditionDef =
nullptr;
488 MBP.SingleUseCondition =
false;
492 MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
493 : MachineBranchPredicate::PRED_EQ;
499 if (
Cond[0].getImm() != -1) {
505 switch (
Cond[1].getImm()) {
509 Cond[1].setImm(AArch64::CBNZW);
512 Cond[1].setImm(AArch64::CBZW);
515 Cond[1].setImm(AArch64::CBNZX);
518 Cond[1].setImm(AArch64::CBZX);
521 Cond[1].setImm(AArch64::TBNZW);
524 Cond[1].setImm(AArch64::TBZW);
527 Cond[1].setImm(AArch64::TBNZX);
530 Cond[1].setImm(AArch64::TBZX);
539 int *BytesRemoved)
const {
549 I->eraseFromParent();
566 I->eraseFromParent();
573void AArch64InstrInfo::instantiateCondBranch(
576 if (
Cond[0].getImm() != -1) {
594 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
633 unsigned *NewVReg =
nullptr) {
638 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
641 unsigned SrcOpNum = 0;
643 case AArch64::ADDSXri:
644 case AArch64::ADDSWri:
651 case AArch64::ADDXri:
652 case AArch64::ADDWri:
658 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
661 case AArch64::ORNXrr:
662 case AArch64::ORNWrr: {
665 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
668 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
672 case AArch64::SUBSXrr:
673 case AArch64::SUBSWrr:
680 case AArch64::SUBXrr:
681 case AArch64::SUBWrr: {
684 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
687 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
693 assert(Opc && SrcOpNum &&
"Missing parameters");
705 int &FalseCycles)
const {
709 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
716 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
720 unsigned ExtraCondLat =
Cond.size() != 1;
724 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
725 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
727 CondCycles = 1 + ExtraCondLat;
728 TrueCycles = FalseCycles = 1;
738 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
739 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
740 CondCycles = 5 + ExtraCondLat;
741 TrueCycles = FalseCycles = 2;
758 switch (
Cond.size()) {
767 switch (
Cond[1].getImm()) {
790 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
796 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
806 switch (
Cond[1].getImm()) {
819 if (
Cond[1].getImm() == AArch64::TBZW ||
Cond[1].getImm() == AArch64::TBNZW)
835 bool TryFold =
false;
836 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
837 RC = &AArch64::GPR64RegClass;
838 Opc = AArch64::CSELXr;
840 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
841 RC = &AArch64::GPR32RegClass;
842 Opc = AArch64::CSELWr;
844 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
845 RC = &AArch64::FPR64RegClass;
846 Opc = AArch64::FCSELDrrr;
847 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
848 RC = &AArch64::FPR32RegClass;
849 Opc = AArch64::FCSELSrrr;
851 assert(RC &&
"Unsupported regclass");
855 unsigned NewVReg = 0;
870 MRI.clearKillFlags(NewVReg);
875 MRI.constrainRegClass(TrueReg, RC);
876 MRI.constrainRegClass(FalseReg, RC);
891 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
896 return Is.
size() <= 2;
902 if (Subtarget.hasExynosCheapAsMoveHandling()) {
903 if (isExynosCheapAsMove(
MI))
905 return MI.isAsCheapAsAMove();
908 switch (
MI.getOpcode()) {
910 return MI.isAsCheapAsAMove();
912 case AArch64::ADDWrs:
913 case AArch64::ADDXrs:
914 case AArch64::SUBWrs:
915 case AArch64::SUBXrs:
916 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
921 case AArch64::MOVi32imm:
923 case AArch64::MOVi64imm:
929 switch (
MI.getOpcode()) {
933 case AArch64::ADDWrs:
934 case AArch64::ADDXrs:
935 case AArch64::ADDSWrs:
936 case AArch64::ADDSXrs: {
937 unsigned Imm =
MI.getOperand(3).getImm();
944 case AArch64::ADDWrx:
945 case AArch64::ADDXrx:
946 case AArch64::ADDXrx64:
947 case AArch64::ADDSWrx:
948 case AArch64::ADDSXrx:
949 case AArch64::ADDSXrx64: {
950 unsigned Imm =
MI.getOperand(3).getImm();
962 case AArch64::SUBWrs:
963 case AArch64::SUBSWrs: {
964 unsigned Imm =
MI.getOperand(3).getImm();
966 return ShiftVal == 0 ||
970 case AArch64::SUBXrs:
971 case AArch64::SUBSXrs: {
972 unsigned Imm =
MI.getOperand(3).getImm();
974 return ShiftVal == 0 ||
978 case AArch64::SUBWrx:
979 case AArch64::SUBXrx:
980 case AArch64::SUBXrx64:
981 case AArch64::SUBSWrx:
982 case AArch64::SUBSXrx:
983 case AArch64::SUBSXrx64: {
984 unsigned Imm =
MI.getOperand(3).getImm();
996 case AArch64::LDRBBroW:
997 case AArch64::LDRBBroX:
998 case AArch64::LDRBroW:
999 case AArch64::LDRBroX:
1000 case AArch64::LDRDroW:
1001 case AArch64::LDRDroX:
1002 case AArch64::LDRHHroW:
1003 case AArch64::LDRHHroX:
1004 case AArch64::LDRHroW:
1005 case AArch64::LDRHroX:
1006 case AArch64::LDRQroW:
1007 case AArch64::LDRQroX:
1008 case AArch64::LDRSBWroW:
1009 case AArch64::LDRSBWroX:
1010 case AArch64::LDRSBXroW:
1011 case AArch64::LDRSBXroX:
1012 case AArch64::LDRSHWroW:
1013 case AArch64::LDRSHWroX:
1014 case AArch64::LDRSHXroW:
1015 case AArch64::LDRSHXroX:
1016 case AArch64::LDRSWroW:
1017 case AArch64::LDRSWroX:
1018 case AArch64::LDRSroW:
1019 case AArch64::LDRSroX:
1020 case AArch64::LDRWroW:
1021 case AArch64::LDRWroX:
1022 case AArch64::LDRXroW:
1023 case AArch64::LDRXroX:
1024 case AArch64::PRFMroW:
1025 case AArch64::PRFMroX:
1026 case AArch64::STRBBroW:
1027 case AArch64::STRBBroX:
1028 case AArch64::STRBroW:
1029 case AArch64::STRBroX:
1030 case AArch64::STRDroW:
1031 case AArch64::STRDroX:
1032 case AArch64::STRHHroW:
1033 case AArch64::STRHHroX:
1034 case AArch64::STRHroW:
1035 case AArch64::STRHroX:
1036 case AArch64::STRQroW:
1037 case AArch64::STRQroX:
1038 case AArch64::STRSroW:
1039 case AArch64::STRSroX:
1040 case AArch64::STRWroW:
1041 case AArch64::STRWroX:
1042 case AArch64::STRXroW:
1043 case AArch64::STRXroX: {
1044 unsigned IsSigned =
MI.getOperand(3).getImm();
1051 unsigned Opc =
MI.getOpcode();
1055 case AArch64::SEH_StackAlloc:
1056 case AArch64::SEH_SaveFPLR:
1057 case AArch64::SEH_SaveFPLR_X:
1058 case AArch64::SEH_SaveReg:
1059 case AArch64::SEH_SaveReg_X:
1060 case AArch64::SEH_SaveRegP:
1061 case AArch64::SEH_SaveRegP_X:
1062 case AArch64::SEH_SaveFReg:
1063 case AArch64::SEH_SaveFReg_X:
1064 case AArch64::SEH_SaveFRegP:
1065 case AArch64::SEH_SaveFRegP_X:
1066 case AArch64::SEH_SetFP:
1067 case AArch64::SEH_AddFP:
1068 case AArch64::SEH_Nop:
1069 case AArch64::SEH_PrologEnd:
1070 case AArch64::SEH_EpilogStart:
1071 case AArch64::SEH_EpilogEnd:
1072 case AArch64::SEH_PACSignLR:
1073 case AArch64::SEH_SaveAnyRegQP:
1074 case AArch64::SEH_SaveAnyRegQPX:
1081 unsigned &SubIdx)
const {
1082 switch (
MI.getOpcode()) {
1085 case AArch64::SBFMXri:
1086 case AArch64::UBFMXri:
1089 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1092 SrcReg =
MI.getOperand(1).getReg();
1093 DstReg =
MI.getOperand(0).getReg();
1094 SubIdx = AArch64::sub_32;
1103 int64_t OffsetA = 0, OffsetB = 0;
1104 TypeSize WidthA(0,
false), WidthB(0,
false);
1105 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1126 OffsetAIsScalable == OffsetBIsScalable) {
1127 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1128 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1129 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1130 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1148 switch (
MI.getOpcode()) {
1151 if (
MI.getOperand(0).getImm() == 0x14)
1158 case AArch64::MSRpstatesvcrImm1:
1165 auto Next = std::next(
MI.getIterator());
1166 return Next !=
MBB->
end() && Next->isCFIInstruction();
1173 Register &SrcReg2, int64_t &CmpMask,
1174 int64_t &CmpValue)
const {
1177 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1178 if (!
MI.getOperand(1).isReg())
1181 switch (
MI.getOpcode()) {
1184 case AArch64::PTEST_PP:
1185 case AArch64::PTEST_PP_ANY:
1186 SrcReg =
MI.getOperand(0).getReg();
1187 SrcReg2 =
MI.getOperand(1).getReg();
1192 case AArch64::SUBSWrr:
1193 case AArch64::SUBSWrs:
1194 case AArch64::SUBSWrx:
1195 case AArch64::SUBSXrr:
1196 case AArch64::SUBSXrs:
1197 case AArch64::SUBSXrx:
1198 case AArch64::ADDSWrr:
1199 case AArch64::ADDSWrs:
1200 case AArch64::ADDSWrx:
1201 case AArch64::ADDSXrr:
1202 case AArch64::ADDSXrs:
1203 case AArch64::ADDSXrx:
1205 SrcReg =
MI.getOperand(1).getReg();
1206 SrcReg2 =
MI.getOperand(2).getReg();
1210 case AArch64::SUBSWri:
1211 case AArch64::ADDSWri:
1212 case AArch64::SUBSXri:
1213 case AArch64::ADDSXri:
1214 SrcReg =
MI.getOperand(1).getReg();
1217 CmpValue =
MI.getOperand(2).getImm();
1219 case AArch64::ANDSWri:
1220 case AArch64::ANDSXri:
1223 SrcReg =
MI.getOperand(1).getReg();
1227 MI.getOperand(2).getImm(),
1228 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1237 assert(
MBB &&
"Can't get MachineBasicBlock here");
1239 assert(MF &&
"Can't get MachineFunction here");
1244 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1248 Instr.getRegClassConstraint(OpIdx,
TII,
TRI);
1251 if (!OpRegCstraints)
1259 "Operand has register constraints without being a register!");
1262 if (Reg.isPhysical()) {
1263 if (!OpRegCstraints->
contains(Reg))
1266 !
MRI->constrainRegClass(Reg, OpRegCstraints))
1279 bool MIDefinesZeroReg =
false;
1280 if (
MI.definesRegister(AArch64::WZR,
nullptr) ||
1281 MI.definesRegister(AArch64::XZR,
nullptr))
1282 MIDefinesZeroReg =
true;
1284 switch (
MI.getOpcode()) {
1286 return MI.getOpcode();
1287 case AArch64::ADDSWrr:
1288 return AArch64::ADDWrr;
1289 case AArch64::ADDSWri:
1290 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1291 case AArch64::ADDSWrs:
1292 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1293 case AArch64::ADDSWrx:
1294 return AArch64::ADDWrx;
1295 case AArch64::ADDSXrr:
1296 return AArch64::ADDXrr;
1297 case AArch64::ADDSXri:
1298 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1299 case AArch64::ADDSXrs:
1300 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1301 case AArch64::ADDSXrx:
1302 return AArch64::ADDXrx;
1303 case AArch64::SUBSWrr:
1304 return AArch64::SUBWrr;
1305 case AArch64::SUBSWri:
1306 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1307 case AArch64::SUBSWrs:
1308 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1309 case AArch64::SUBSWrx:
1310 return AArch64::SUBWrx;
1311 case AArch64::SUBSXrr:
1312 return AArch64::SUBXrr;
1313 case AArch64::SUBSXri:
1314 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1315 case AArch64::SUBSXrs:
1316 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1317 case AArch64::SUBSXrx:
1318 return AArch64::SUBXrx;
1333 if (To == To->getParent()->begin())
1338 if (To->getParent() !=
From->getParent())
1350 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1351 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1359bool AArch64InstrInfo::optimizePTestInstr(
1360 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1362 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1363 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1364 auto NewOp = Pred->getOpcode();
1365 bool OpChanged =
false;
1367 unsigned MaskOpcode =
Mask->getOpcode();
1368 unsigned PredOpcode = Pred->getOpcode();
1372 if (
isPTrueOpcode(MaskOpcode) && (PredIsPTestLike || PredIsWhileLike) &&
1375 Mask->getOperand(1).getImm() == 31) {
1384 if (PredIsPTestLike) {
1385 auto PTestLikeMask =
MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1386 if (Mask != PTestLikeMask && PTest->
getOpcode() != AArch64::PTEST_PP_ANY)
1391 }
else if ((Mask == Pred) && (PredIsPTestLike || PredIsWhileLike) &&
1392 PTest->
getOpcode() == AArch64::PTEST_PP_ANY) {
1398 }
else if (PredIsPTestLike) {
1421 auto PTestLikeMask =
MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1423 if ((Mask != PTestLikeMask) ||
1425 PTest->
getOpcode() != AArch64::PTEST_PP_ANY))
1432 switch (PredOpcode) {
1433 case AArch64::AND_PPzPP:
1434 case AArch64::BIC_PPzPP:
1435 case AArch64::EOR_PPzPP:
1436 case AArch64::NAND_PPzPP:
1437 case AArch64::NOR_PPzPP:
1438 case AArch64::ORN_PPzPP:
1439 case AArch64::ORR_PPzPP:
1440 case AArch64::BRKA_PPzP:
1441 case AArch64::BRKPA_PPzPP:
1442 case AArch64::BRKB_PPzP:
1443 case AArch64::BRKPB_PPzPP:
1444 case AArch64::RDFFR_PPz: {
1447 auto *PredMask =
MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1448 if (Mask != PredMask)
1452 case AArch64::BRKN_PPzP: {
1456 if ((MaskOpcode != AArch64::PTRUE_B) ||
1457 (
Mask->getOperand(1).getImm() != 31))
1461 case AArch64::PTRUE_B:
1484 Pred->setDesc(
get(NewOp));
1489 assert(succeeded &&
"Operands have incompatible register classes!");
1490 Pred->addRegisterDefined(AArch64::NZCV,
TRI);
1494 if (Pred->registerDefIsDead(AArch64::NZCV,
TRI)) {
1495 unsigned i = 0,
e = Pred->getNumOperands();
1496 for (; i !=
e; ++i) {
1527 if (DeadNZCVIdx != -1) {
1542 assert(succeeded &&
"Some operands reg class are incompatible!");
1546 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1547 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY)
1548 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1557 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1559 return (CmpValue == 0 || CmpValue == 1) &&
1560 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1568 switch (Instr.getOpcode()) {
1570 return AArch64::INSTRUCTION_LIST_END;
1572 case AArch64::ADDSWrr:
1573 case AArch64::ADDSWri:
1574 case AArch64::ADDSXrr:
1575 case AArch64::ADDSXri:
1576 case AArch64::SUBSWrr:
1577 case AArch64::SUBSWri:
1578 case AArch64::SUBSXrr:
1579 case AArch64::SUBSXri:
1580 return Instr.getOpcode();
1582 case AArch64::ADDWrr:
1583 return AArch64::ADDSWrr;
1584 case AArch64::ADDWri:
1585 return AArch64::ADDSWri;
1586 case AArch64::ADDXrr:
1587 return AArch64::ADDSXrr;
1588 case AArch64::ADDXri:
1589 return AArch64::ADDSXri;
1590 case AArch64::ADCWr:
1591 return AArch64::ADCSWr;
1592 case AArch64::ADCXr:
1593 return AArch64::ADCSXr;
1594 case AArch64::SUBWrr:
1595 return AArch64::SUBSWrr;
1596 case AArch64::SUBWri:
1597 return AArch64::SUBSWri;
1598 case AArch64::SUBXrr:
1599 return AArch64::SUBSXrr;
1600 case AArch64::SUBXri:
1601 return AArch64::SUBSXri;
1602 case AArch64::SBCWr:
1603 return AArch64::SBCSWr;
1604 case AArch64::SBCXr:
1605 return AArch64::SBCSXr;
1606 case AArch64::ANDWri:
1607 return AArch64::ANDSWri;
1608 case AArch64::ANDXri:
1609 return AArch64::ANDSXri;
1616 if (BB->isLiveIn(AArch64::NZCV))
1625 switch (Instr.getOpcode()) {
1629 case AArch64::Bcc: {
1630 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1635 case AArch64::CSINVWr:
1636 case AArch64::CSINVXr:
1637 case AArch64::CSINCWr:
1638 case AArch64::CSINCXr:
1639 case AArch64::CSELWr:
1640 case AArch64::CSELXr:
1641 case AArch64::CSNEGWr:
1642 case AArch64::CSNEGXr:
1643 case AArch64::FCSELSrrr:
1644 case AArch64::FCSELDrrr: {
1645 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV,
nullptr);
1658 Instr.getOperand(CCIdx).getImm())
1711std::optional<UsedNZCV>
1716 if (
MI.getParent() != CmpParent)
1717 return std::nullopt;
1720 return std::nullopt;
1725 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1728 return std::nullopt;
1733 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1736 return NZCVUsedAfterCmp;
1740 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1744 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1766 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1772 "Caller guarantees that CmpInstr compares with constant 0");
1775 if (!NZVCUsed || NZVCUsed->C)
1797bool AArch64InstrInfo::substituteCmpToZero(
1808 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1815 MI->setDesc(
get(NewOpc));
1819 assert(succeeded &&
"Some operands reg class are incompatible!");
1820 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
1832 assert((CmpValue == 0 || CmpValue == 1) &&
1833 "Only comparisons to 0 or 1 considered for removal!");
1836 unsigned MIOpc =
MI.getOpcode();
1837 if (MIOpc == AArch64::CSINCWr) {
1838 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
1839 MI.getOperand(2).getReg() != AArch64::WZR)
1841 }
else if (MIOpc == AArch64::CSINCXr) {
1842 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
1843 MI.getOperand(2).getReg() != AArch64::XZR)
1853 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) != -1)
1857 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1859 if (CmpValue && !IsSubsRegImm)
1861 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
1866 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
1869 std::optional<UsedNZCV> NZCVUsedAfterCmp =
1873 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
1876 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
1877 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
1880 if (MIUsedNZCV.
N && !CmpValue)
1922bool AArch64InstrInfo::removeCmpToZeroOrOne(
1930 bool IsInvertCC =
false;
1940 assert(
Idx >= 0 &&
"Unexpected instruction using CC.");
1951 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1952 MI.getOpcode() != AArch64::CATCHRET)
1960 if (
MI.getOpcode() == AArch64::CATCHRET) {
1969 FirstEpilogSEH = std::prev(FirstEpilogSEH);
1971 FirstEpilogSEH = std::next(FirstEpilogSEH);
1985 if (M.getStackProtectorGuard() ==
"sysreg") {
1995 int Offset = M.getStackProtectorGuardOffset();
2047 cast<GlobalValue>((*
MI.memoperands_begin())->getValue());
2056 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2098 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2121 switch (
MI.getOpcode()) {
2124 case AArch64::MOVZWi:
2125 case AArch64::MOVZXi:
2126 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2127 assert(
MI.getDesc().getNumOperands() == 3 &&
2128 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2132 case AArch64::ANDWri:
2133 return MI.getOperand(1).getReg() == AArch64::WZR;
2134 case AArch64::ANDXri:
2135 return MI.getOperand(1).getReg() == AArch64::XZR;
2136 case TargetOpcode::COPY:
2137 return MI.getOperand(1).getReg() == AArch64::WZR;
2145 switch (
MI.getOpcode()) {
2148 case TargetOpcode::COPY: {
2151 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2152 AArch64::GPR64RegClass.
contains(DstReg));
2154 case AArch64::ORRXrs:
2155 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2156 assert(
MI.getDesc().getNumOperands() == 4 &&
2157 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2161 case AArch64::ADDXri:
2162 if (
MI.getOperand(2).getImm() == 0) {
2163 assert(
MI.getDesc().getNumOperands() == 4 &&
2164 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2175 switch (
MI.getOpcode()) {
2178 case TargetOpcode::COPY: {
2180 return AArch64::FPR128RegClass.contains(DstReg);
2182 case AArch64::ORRv16i8:
2183 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2184 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2185 "invalid ORRv16i8 operands");
2194 int &FrameIndex)
const {
2195 switch (
MI.getOpcode()) {
2198 case AArch64::LDRWui:
2199 case AArch64::LDRXui:
2200 case AArch64::LDRBui:
2201 case AArch64::LDRHui:
2202 case AArch64::LDRSui:
2203 case AArch64::LDRDui:
2204 case AArch64::LDRQui:
2205 case AArch64::LDR_PXI:
2206 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2207 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2208 FrameIndex =
MI.getOperand(1).getIndex();
2209 return MI.getOperand(0).getReg();
2218 int &FrameIndex)
const {
2219 switch (
MI.getOpcode()) {
2222 case AArch64::STRWui:
2223 case AArch64::STRXui:
2224 case AArch64::STRBui:
2225 case AArch64::STRHui:
2226 case AArch64::STRSui:
2227 case AArch64::STRDui:
2228 case AArch64::STRQui:
2229 case AArch64::STR_PXI:
2230 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2231 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2232 FrameIndex =
MI.getOperand(1).getIndex();
2233 return MI.getOperand(0).getReg();
2243 return MMO->getFlags() & MOSuppressPair;
2249 if (
MI.memoperands_empty())
2257 return MMO->getFlags() & MOStridedAccess;
2265 case AArch64::STURSi:
2266 case AArch64::STRSpre:
2267 case AArch64::STURDi:
2268 case AArch64::STRDpre:
2269 case AArch64::STURQi:
2270 case AArch64::STRQpre:
2271 case AArch64::STURBBi:
2272 case AArch64::STURHHi:
2273 case AArch64::STURWi:
2274 case AArch64::STRWpre:
2275 case AArch64::STURXi:
2276 case AArch64::STRXpre:
2277 case AArch64::LDURSi:
2278 case AArch64::LDRSpre:
2279 case AArch64::LDURDi:
2280 case AArch64::LDRDpre:
2281 case AArch64::LDURQi:
2282 case AArch64::LDRQpre:
2283 case AArch64::LDURWi:
2284 case AArch64::LDRWpre:
2285 case AArch64::LDURXi:
2286 case AArch64::LDRXpre:
2287 case AArch64::LDRSWpre:
2288 case AArch64::LDURSWi:
2289 case AArch64::LDURHHi:
2290 case AArch64::LDURBBi:
2291 case AArch64::LDURSBWi:
2292 case AArch64::LDURSHWi:
2300 case AArch64::PRFMui:
return AArch64::PRFUMi;
2301 case AArch64::LDRXui:
return AArch64::LDURXi;
2302 case AArch64::LDRWui:
return AArch64::LDURWi;
2303 case AArch64::LDRBui:
return AArch64::LDURBi;
2304 case AArch64::LDRHui:
return AArch64::LDURHi;
2305 case AArch64::LDRSui:
return AArch64::LDURSi;
2306 case AArch64::LDRDui:
return AArch64::LDURDi;
2307 case AArch64::LDRQui:
return AArch64::LDURQi;
2308 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2309 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2310 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2311 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2312 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2313 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2314 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2315 case AArch64::STRXui:
return AArch64::STURXi;
2316 case AArch64::STRWui:
return AArch64::STURWi;
2317 case AArch64::STRBui:
return AArch64::STURBi;
2318 case AArch64::STRHui:
return AArch64::STURHi;
2319 case AArch64::STRSui:
return AArch64::STURSi;
2320 case AArch64::STRDui:
return AArch64::STURDi;
2321 case AArch64::STRQui:
return AArch64::STURQi;
2322 case AArch64::STRBBui:
return AArch64::STURBBi;
2323 case AArch64::STRHHui:
return AArch64::STURHHi;
2331 case AArch64::LDPXi:
2332 case AArch64::LDPDi:
2333 case AArch64::STPXi:
2334 case AArch64::STPDi:
2335 case AArch64::LDNPXi:
2336 case AArch64::LDNPDi:
2337 case AArch64::STNPXi:
2338 case AArch64::STNPDi:
2339 case AArch64::LDPQi:
2340 case AArch64::STPQi:
2341 case AArch64::LDNPQi:
2342 case AArch64::STNPQi:
2343 case AArch64::LDPWi:
2344 case AArch64::LDPSi:
2345 case AArch64::STPWi:
2346 case AArch64::STPSi:
2347 case AArch64::LDNPWi:
2348 case AArch64::LDNPSi:
2349 case AArch64::STNPWi:
2350 case AArch64::STNPSi:
2352 case AArch64::STGPi:
2354 case AArch64::LD1B_IMM:
2355 case AArch64::LD1B_H_IMM:
2356 case AArch64::LD1B_S_IMM:
2357 case AArch64::LD1B_D_IMM:
2358 case AArch64::LD1SB_H_IMM:
2359 case AArch64::LD1SB_S_IMM:
2360 case AArch64::LD1SB_D_IMM:
2361 case AArch64::LD1H_IMM:
2362 case AArch64::LD1H_S_IMM:
2363 case AArch64::LD1H_D_IMM:
2364 case AArch64::LD1SH_S_IMM:
2365 case AArch64::LD1SH_D_IMM:
2366 case AArch64::LD1W_IMM:
2367 case AArch64::LD1W_D_IMM:
2368 case AArch64::LD1SW_D_IMM:
2369 case AArch64::LD1D_IMM:
2371 case AArch64::LD2B_IMM:
2372 case AArch64::LD2H_IMM:
2373 case AArch64::LD2W_IMM:
2374 case AArch64::LD2D_IMM:
2375 case AArch64::LD3B_IMM:
2376 case AArch64::LD3H_IMM:
2377 case AArch64::LD3W_IMM:
2378 case AArch64::LD3D_IMM:
2379 case AArch64::LD4B_IMM:
2380 case AArch64::LD4H_IMM:
2381 case AArch64::LD4W_IMM:
2382 case AArch64::LD4D_IMM:
2384 case AArch64::ST1B_IMM:
2385 case AArch64::ST1B_H_IMM:
2386 case AArch64::ST1B_S_IMM:
2387 case AArch64::ST1B_D_IMM:
2388 case AArch64::ST1H_IMM:
2389 case AArch64::ST1H_S_IMM:
2390 case AArch64::ST1H_D_IMM:
2391 case AArch64::ST1W_IMM:
2392 case AArch64::ST1W_D_IMM:
2393 case AArch64::ST1D_IMM:
2395 case AArch64::ST2B_IMM:
2396 case AArch64::ST2H_IMM:
2397 case AArch64::ST2W_IMM:
2398 case AArch64::ST2D_IMM:
2399 case AArch64::ST3B_IMM:
2400 case AArch64::ST3H_IMM:
2401 case AArch64::ST3W_IMM:
2402 case AArch64::ST3D_IMM:
2403 case AArch64::ST4B_IMM:
2404 case AArch64::ST4H_IMM:
2405 case AArch64::ST4W_IMM:
2406 case AArch64::ST4D_IMM:
2408 case AArch64::LD1RB_IMM:
2409 case AArch64::LD1RB_H_IMM:
2410 case AArch64::LD1RB_S_IMM:
2411 case AArch64::LD1RB_D_IMM:
2412 case AArch64::LD1RSB_H_IMM:
2413 case AArch64::LD1RSB_S_IMM:
2414 case AArch64::LD1RSB_D_IMM:
2415 case AArch64::LD1RH_IMM:
2416 case AArch64::LD1RH_S_IMM:
2417 case AArch64::LD1RH_D_IMM:
2418 case AArch64::LD1RSH_S_IMM:
2419 case AArch64::LD1RSH_D_IMM:
2420 case AArch64::LD1RW_IMM:
2421 case AArch64::LD1RW_D_IMM:
2422 case AArch64::LD1RSW_IMM:
2423 case AArch64::LD1RD_IMM:
2425 case AArch64::LDNT1B_ZRI:
2426 case AArch64::LDNT1H_ZRI:
2427 case AArch64::LDNT1W_ZRI:
2428 case AArch64::LDNT1D_ZRI:
2429 case AArch64::STNT1B_ZRI:
2430 case AArch64::STNT1H_ZRI:
2431 case AArch64::STNT1W_ZRI:
2432 case AArch64::STNT1D_ZRI:
2434 case AArch64::LDNF1B_IMM:
2435 case AArch64::LDNF1B_H_IMM:
2436 case AArch64::LDNF1B_S_IMM:
2437 case AArch64::LDNF1B_D_IMM:
2438 case AArch64::LDNF1SB_H_IMM:
2439 case AArch64::LDNF1SB_S_IMM:
2440 case AArch64::LDNF1SB_D_IMM:
2441 case AArch64::LDNF1H_IMM:
2442 case AArch64::LDNF1H_S_IMM:
2443 case AArch64::LDNF1H_D_IMM:
2444 case AArch64::LDNF1SH_S_IMM:
2445 case AArch64::LDNF1SH_D_IMM:
2446 case AArch64::LDNF1W_IMM:
2447 case AArch64::LDNF1W_D_IMM:
2448 case AArch64::LDNF1SW_D_IMM:
2449 case AArch64::LDNF1D_IMM:
2453 case AArch64::LDR_PXI:
2454 case AArch64::STR_PXI:
2460 switch (
MI.getOpcode()) {
2464 case AArch64::STRSui:
2465 case AArch64::STRDui:
2466 case AArch64::STRQui:
2467 case AArch64::STRXui:
2468 case AArch64::STRWui:
2469 case AArch64::LDRSui:
2470 case AArch64::LDRDui:
2471 case AArch64::LDRQui:
2472 case AArch64::LDRXui:
2473 case AArch64::LDRWui:
2474 case AArch64::LDRSWui:
2476 case AArch64::STURSi:
2477 case AArch64::STRSpre:
2478 case AArch64::STURDi:
2479 case AArch64::STRDpre:
2480 case AArch64::STURQi:
2481 case AArch64::STRQpre:
2482 case AArch64::STURWi:
2483 case AArch64::STRWpre:
2484 case AArch64::STURXi:
2485 case AArch64::STRXpre:
2486 case AArch64::LDURSi:
2487 case AArch64::LDRSpre:
2488 case AArch64::LDURDi:
2489 case AArch64::LDRDpre:
2490 case AArch64::LDURQi:
2491 case AArch64::LDRQpre:
2492 case AArch64::LDURWi:
2493 case AArch64::LDRWpre:
2494 case AArch64::LDURXi:
2495 case AArch64::LDRXpre:
2496 case AArch64::LDURSWi:
2497 case AArch64::LDRSWpre:
2503 switch (
MI.getOpcode()) {
2506 "Unexpected instruction - was a new tail call opcode introduced?");
2508 case AArch64::TCRETURNdi:
2509 case AArch64::TCRETURNri:
2510 case AArch64::TCRETURNrix16x17:
2511 case AArch64::TCRETURNrix17:
2512 case AArch64::TCRETURNrinotx16:
2513 case AArch64::TCRETURNriALL:
2523 case AArch64::ADDWri:
2524 return AArch64::ADDSWri;
2525 case AArch64::ADDWrr:
2526 return AArch64::ADDSWrr;
2527 case AArch64::ADDWrs:
2528 return AArch64::ADDSWrs;
2529 case AArch64::ADDWrx:
2530 return AArch64::ADDSWrx;
2531 case AArch64::ANDWri:
2532 return AArch64::ANDSWri;
2533 case AArch64::ANDWrr:
2534 return AArch64::ANDSWrr;
2535 case AArch64::ANDWrs:
2536 return AArch64::ANDSWrs;
2537 case AArch64::BICWrr:
2538 return AArch64::BICSWrr;
2539 case AArch64::BICWrs:
2540 return AArch64::BICSWrs;
2541 case AArch64::SUBWri:
2542 return AArch64::SUBSWri;
2543 case AArch64::SUBWrr:
2544 return AArch64::SUBSWrr;
2545 case AArch64::SUBWrs:
2546 return AArch64::SUBSWrs;
2547 case AArch64::SUBWrx:
2548 return AArch64::SUBSWrx;
2550 case AArch64::ADDXri:
2551 return AArch64::ADDSXri;
2552 case AArch64::ADDXrr:
2553 return AArch64::ADDSXrr;
2554 case AArch64::ADDXrs:
2555 return AArch64::ADDSXrs;
2556 case AArch64::ADDXrx:
2557 return AArch64::ADDSXrx;
2558 case AArch64::ANDXri:
2559 return AArch64::ANDSXri;
2560 case AArch64::ANDXrr:
2561 return AArch64::ANDSXrr;
2562 case AArch64::ANDXrs:
2563 return AArch64::ANDSXrs;
2564 case AArch64::BICXrr:
2565 return AArch64::BICSXrr;
2566 case AArch64::BICXrs:
2567 return AArch64::BICSXrs;
2568 case AArch64::SUBXri:
2569 return AArch64::SUBSXri;
2570 case AArch64::SUBXrr:
2571 return AArch64::SUBSXrr;
2572 case AArch64::SUBXrs:
2573 return AArch64::SUBSXrs;
2574 case AArch64::SUBXrx:
2575 return AArch64::SUBSXrx;
2577 case AArch64::AND_PPzPP:
2578 return AArch64::ANDS_PPzPP;
2579 case AArch64::BIC_PPzPP:
2580 return AArch64::BICS_PPzPP;
2581 case AArch64::EOR_PPzPP:
2582 return AArch64::EORS_PPzPP;
2583 case AArch64::NAND_PPzPP:
2584 return AArch64::NANDS_PPzPP;
2585 case AArch64::NOR_PPzPP:
2586 return AArch64::NORS_PPzPP;
2587 case AArch64::ORN_PPzPP:
2588 return AArch64::ORNS_PPzPP;
2589 case AArch64::ORR_PPzPP:
2590 return AArch64::ORRS_PPzPP;
2591 case AArch64::BRKA_PPzP:
2592 return AArch64::BRKAS_PPzP;
2593 case AArch64::BRKPA_PPzPP:
2594 return AArch64::BRKPAS_PPzPP;
2595 case AArch64::BRKB_PPzP:
2596 return AArch64::BRKBS_PPzP;
2597 case AArch64::BRKPB_PPzPP:
2598 return AArch64::BRKPBS_PPzPP;
2599 case AArch64::BRKN_PPzP:
2600 return AArch64::BRKNS_PPzP;
2601 case AArch64::RDFFR_PPz:
2602 return AArch64::RDFFRS_PPz;
2603 case AArch64::PTRUE_B:
2604 return AArch64::PTRUES_B;
2615 if (
MI.hasOrderedMemoryRef())
2620 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2621 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2622 "Expected a reg or frame index operand.");
2626 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2628 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2641 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2642 Register BaseReg =
MI.getOperand(1).getReg();
2644 if (
MI.modifiesRegister(BaseReg,
TRI))
2657 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
2659 MI.getMF()->getFunction().needsUnwindTableEntry();
2665 if (Subtarget.isPaired128Slow()) {
2666 switch (
MI.getOpcode()) {
2669 case AArch64::LDURQi:
2670 case AArch64::STURQi:
2671 case AArch64::LDRQui:
2672 case AArch64::STRQui:
2699std::optional<ExtAddrMode>
2704 bool OffsetIsScalable;
2705 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
2706 return std::nullopt;
2709 return std::nullopt;
2724 int64_t OffsetScale = 1;
2729 case AArch64::LDURQi:
2730 case AArch64::STURQi:
2734 case AArch64::LDURDi:
2735 case AArch64::STURDi:
2736 case AArch64::LDURXi:
2737 case AArch64::STURXi:
2741 case AArch64::LDURWi:
2742 case AArch64::LDURSWi:
2743 case AArch64::STURWi:
2747 case AArch64::LDURHi:
2748 case AArch64::STURHi:
2749 case AArch64::LDURHHi:
2750 case AArch64::STURHHi:
2751 case AArch64::LDURSHXi:
2752 case AArch64::LDURSHWi:
2756 case AArch64::LDRBroX:
2757 case AArch64::LDRBBroX:
2758 case AArch64::LDRSBXroX:
2759 case AArch64::LDRSBWroX:
2760 case AArch64::STRBroX:
2761 case AArch64::STRBBroX:
2762 case AArch64::LDURBi:
2763 case AArch64::LDURBBi:
2764 case AArch64::LDURSBXi:
2765 case AArch64::LDURSBWi:
2766 case AArch64::STURBi:
2767 case AArch64::STURBBi:
2768 case AArch64::LDRBui:
2769 case AArch64::LDRBBui:
2770 case AArch64::LDRSBXui:
2771 case AArch64::LDRSBWui:
2772 case AArch64::STRBui:
2773 case AArch64::STRBBui:
2777 case AArch64::LDRQroX:
2778 case AArch64::STRQroX:
2779 case AArch64::LDRQui:
2780 case AArch64::STRQui:
2785 case AArch64::LDRDroX:
2786 case AArch64::STRDroX:
2787 case AArch64::LDRXroX:
2788 case AArch64::STRXroX:
2789 case AArch64::LDRDui:
2790 case AArch64::STRDui:
2791 case AArch64::LDRXui:
2792 case AArch64::STRXui:
2797 case AArch64::LDRWroX:
2798 case AArch64::LDRSWroX:
2799 case AArch64::STRWroX:
2800 case AArch64::LDRWui:
2801 case AArch64::LDRSWui:
2802 case AArch64::STRWui:
2807 case AArch64::LDRHroX:
2808 case AArch64::STRHroX:
2809 case AArch64::LDRHHroX:
2810 case AArch64::STRHHroX:
2811 case AArch64::LDRSHXroX:
2812 case AArch64::LDRSHWroX:
2813 case AArch64::LDRHui:
2814 case AArch64::STRHui:
2815 case AArch64::LDRHHui:
2816 case AArch64::STRHHui:
2817 case AArch64::LDRSHXui:
2818 case AArch64::LDRSHWui:
2826 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
2850 case AArch64::SBFMXri:
2863 AM.
Scale = OffsetScale;
2868 case TargetOpcode::SUBREG_TO_REG: {
2881 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
2885 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
2887 DefMI.getOperand(3).getImm() != 0)
2894 AM.
Scale = OffsetScale;
2905 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
2906 int64_t NewOffset) ->
bool {
2907 int64_t MinOffset, MaxOffset;
2924 return OldOffset < MinOffset || OldOffset > MaxOffset ||
2925 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
2927 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
2929 int64_t NewOffset = OldOffset + Disp;
2934 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
2944 auto canFoldAddRegIntoAddrMode =
2961 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
2962 Subtarget.isSTRQroSlow();
2971 case AArch64::ADDXri:
2977 return canFoldAddSubImmIntoAddrMode(Disp);
2979 case AArch64::SUBXri:
2985 return canFoldAddSubImmIntoAddrMode(-Disp);
2987 case AArch64::ADDXrs: {
3000 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3002 if (avoidSlowSTRQ(MemI))
3005 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3008 case AArch64::ADDXrr:
3016 if (!OptSize && avoidSlowSTRQ(MemI))
3018 return canFoldAddRegIntoAddrMode(1);
3020 case AArch64::ADDXrx:
3028 if (!OptSize && avoidSlowSTRQ(MemI))
3037 return canFoldAddRegIntoAddrMode(
3052 case AArch64::LDURQi:
3053 case AArch64::LDRQui:
3054 return AArch64::LDRQroX;
3055 case AArch64::STURQi:
3056 case AArch64::STRQui:
3057 return AArch64::STRQroX;
3058 case AArch64::LDURDi:
3059 case AArch64::LDRDui:
3060 return AArch64::LDRDroX;
3061 case AArch64::STURDi:
3062 case AArch64::STRDui:
3063 return AArch64::STRDroX;
3064 case AArch64::LDURXi:
3065 case AArch64::LDRXui:
3066 return AArch64::LDRXroX;
3067 case AArch64::STURXi:
3068 case AArch64::STRXui:
3069 return AArch64::STRXroX;
3070 case AArch64::LDURWi:
3071 case AArch64::LDRWui:
3072 return AArch64::LDRWroX;
3073 case AArch64::LDURSWi:
3074 case AArch64::LDRSWui:
3075 return AArch64::LDRSWroX;
3076 case AArch64::STURWi:
3077 case AArch64::STRWui:
3078 return AArch64::STRWroX;
3079 case AArch64::LDURHi:
3080 case AArch64::LDRHui:
3081 return AArch64::LDRHroX;
3082 case AArch64::STURHi:
3083 case AArch64::STRHui:
3084 return AArch64::STRHroX;
3085 case AArch64::LDURHHi:
3086 case AArch64::LDRHHui:
3087 return AArch64::LDRHHroX;
3088 case AArch64::STURHHi:
3089 case AArch64::STRHHui:
3090 return AArch64::STRHHroX;
3091 case AArch64::LDURSHXi:
3092 case AArch64::LDRSHXui:
3093 return AArch64::LDRSHXroX;
3094 case AArch64::LDURSHWi:
3095 case AArch64::LDRSHWui:
3096 return AArch64::LDRSHWroX;
3097 case AArch64::LDURBi:
3098 case AArch64::LDRBui:
3099 return AArch64::LDRBroX;
3100 case AArch64::LDURBBi:
3101 case AArch64::LDRBBui:
3102 return AArch64::LDRBBroX;
3103 case AArch64::LDURSBXi:
3104 case AArch64::LDRSBXui:
3105 return AArch64::LDRSBXroX;
3106 case AArch64::LDURSBWi:
3107 case AArch64::LDRSBWui:
3108 return AArch64::LDRSBWroX;
3109 case AArch64::STURBi:
3110 case AArch64::STRBui:
3111 return AArch64::STRBroX;
3112 case AArch64::STURBBi:
3113 case AArch64::STRBBui:
3114 return AArch64::STRBBroX;
3126 case AArch64::LDURQi:
3128 return AArch64::LDRQui;
3129 case AArch64::STURQi:
3131 return AArch64::STRQui;
3132 case AArch64::LDURDi:
3134 return AArch64::LDRDui;
3135 case AArch64::STURDi:
3137 return AArch64::STRDui;
3138 case AArch64::LDURXi:
3140 return AArch64::LDRXui;
3141 case AArch64::STURXi:
3143 return AArch64::STRXui;
3144 case AArch64::LDURWi:
3146 return AArch64::LDRWui;
3147 case AArch64::LDURSWi:
3149 return AArch64::LDRSWui;
3150 case AArch64::STURWi:
3152 return AArch64::STRWui;
3153 case AArch64::LDURHi:
3155 return AArch64::LDRHui;
3156 case AArch64::STURHi:
3158 return AArch64::STRHui;
3159 case AArch64::LDURHHi:
3161 return AArch64::LDRHHui;
3162 case AArch64::STURHHi:
3164 return AArch64::STRHHui;
3165 case AArch64::LDURSHXi:
3167 return AArch64::LDRSHXui;
3168 case AArch64::LDURSHWi:
3170 return AArch64::LDRSHWui;
3171 case AArch64::LDURBi:
3173 return AArch64::LDRBui;
3174 case AArch64::LDURBBi:
3176 return AArch64::LDRBBui;
3177 case AArch64::LDURSBXi:
3179 return AArch64::LDRSBXui;
3180 case AArch64::LDURSBWi:
3182 return AArch64::LDRSBWui;
3183 case AArch64::STURBi:
3185 return AArch64::STRBui;
3186 case AArch64::STURBBi:
3188 return AArch64::STRBBui;
3189 case AArch64::LDRQui:
3190 case AArch64::STRQui:
3193 case AArch64::LDRDui:
3194 case AArch64::STRDui:
3195 case AArch64::LDRXui:
3196 case AArch64::STRXui:
3199 case AArch64::LDRWui:
3200 case AArch64::LDRSWui:
3201 case AArch64::STRWui:
3204 case AArch64::LDRHui:
3205 case AArch64::STRHui:
3206 case AArch64::LDRHHui:
3207 case AArch64::STRHHui:
3208 case AArch64::LDRSHXui:
3209 case AArch64::LDRSHWui:
3212 case AArch64::LDRBui:
3213 case AArch64::LDRBBui:
3214 case AArch64::LDRSBXui:
3215 case AArch64::LDRSBWui:
3216 case AArch64::STRBui:
3217 case AArch64::STRBBui:
3231 case AArch64::LDURQi:
3232 case AArch64::STURQi:
3233 case AArch64::LDURDi:
3234 case AArch64::STURDi:
3235 case AArch64::LDURXi:
3236 case AArch64::STURXi:
3237 case AArch64::LDURWi:
3238 case AArch64::LDURSWi:
3239 case AArch64::STURWi:
3240 case AArch64::LDURHi:
3241 case AArch64::STURHi:
3242 case AArch64::LDURHHi:
3243 case AArch64::STURHHi:
3244 case AArch64::LDURSHXi:
3245 case AArch64::LDURSHWi:
3246 case AArch64::LDURBi:
3247 case AArch64::STURBi:
3248 case AArch64::LDURBBi:
3249 case AArch64::STURBBi:
3250 case AArch64::LDURSBWi:
3251 case AArch64::LDURSBXi:
3253 case AArch64::LDRQui:
3254 return AArch64::LDURQi;
3255 case AArch64::STRQui:
3256 return AArch64::STURQi;
3257 case AArch64::LDRDui:
3258 return AArch64::LDURDi;
3259 case AArch64::STRDui:
3260 return AArch64::STURDi;
3261 case AArch64::LDRXui:
3262 return AArch64::LDURXi;
3263 case AArch64::STRXui:
3264 return AArch64::STURXi;
3265 case AArch64::LDRWui:
3266 return AArch64::LDURWi;
3267 case AArch64::LDRSWui:
3268 return AArch64::LDURSWi;
3269 case AArch64::STRWui:
3270 return AArch64::STURWi;
3271 case AArch64::LDRHui:
3272 return AArch64::LDURHi;
3273 case AArch64::STRHui:
3274 return AArch64::STURHi;
3275 case AArch64::LDRHHui:
3276 return AArch64::LDURHHi;
3277 case AArch64::STRHHui:
3278 return AArch64::STURHHi;
3279 case AArch64::LDRSHXui:
3280 return AArch64::LDURSHXi;
3281 case AArch64::LDRSHWui:
3282 return AArch64::LDURSHWi;
3283 case AArch64::LDRBBui:
3284 return AArch64::LDURBBi;
3285 case AArch64::LDRBui:
3286 return AArch64::LDURBi;
3287 case AArch64::STRBBui:
3288 return AArch64::STURBBi;
3289 case AArch64::STRBui:
3290 return AArch64::STURBi;
3291 case AArch64::LDRSBWui:
3292 return AArch64::LDURSBWi;
3293 case AArch64::LDRSBXui:
3294 return AArch64::LDURSBXi;
3307 case AArch64::LDRQroX:
3308 case AArch64::LDURQi:
3309 case AArch64::LDRQui:
3310 return AArch64::LDRQroW;
3311 case AArch64::STRQroX:
3312 case AArch64::STURQi:
3313 case AArch64::STRQui:
3314 return AArch64::STRQroW;
3315 case AArch64::LDRDroX:
3316 case AArch64::LDURDi:
3317 case AArch64::LDRDui:
3318 return AArch64::LDRDroW;
3319 case AArch64::STRDroX:
3320 case AArch64::STURDi:
3321 case AArch64::STRDui:
3322 return AArch64::STRDroW;
3323 case AArch64::LDRXroX:
3324 case AArch64::LDURXi:
3325 case AArch64::LDRXui:
3326 return AArch64::LDRXroW;
3327 case AArch64::STRXroX:
3328 case AArch64::STURXi:
3329 case AArch64::STRXui:
3330 return AArch64::STRXroW;
3331 case AArch64::LDRWroX:
3332 case AArch64::LDURWi:
3333 case AArch64::LDRWui:
3334 return AArch64::LDRWroW;
3335 case AArch64::LDRSWroX:
3336 case AArch64::LDURSWi:
3337 case AArch64::LDRSWui:
3338 return AArch64::LDRSWroW;
3339 case AArch64::STRWroX:
3340 case AArch64::STURWi:
3341 case AArch64::STRWui:
3342 return AArch64::STRWroW;
3343 case AArch64::LDRHroX:
3344 case AArch64::LDURHi:
3345 case AArch64::LDRHui:
3346 return AArch64::LDRHroW;
3347 case AArch64::STRHroX:
3348 case AArch64::STURHi:
3349 case AArch64::STRHui:
3350 return AArch64::STRHroW;
3351 case AArch64::LDRHHroX:
3352 case AArch64::LDURHHi:
3353 case AArch64::LDRHHui:
3354 return AArch64::LDRHHroW;
3355 case AArch64::STRHHroX:
3356 case AArch64::STURHHi:
3357 case AArch64::STRHHui:
3358 return AArch64::STRHHroW;
3359 case AArch64::LDRSHXroX:
3360 case AArch64::LDURSHXi:
3361 case AArch64::LDRSHXui:
3362 return AArch64::LDRSHXroW;
3363 case AArch64::LDRSHWroX:
3364 case AArch64::LDURSHWi:
3365 case AArch64::LDRSHWui:
3366 return AArch64::LDRSHWroW;
3367 case AArch64::LDRBroX:
3368 case AArch64::LDURBi:
3369 case AArch64::LDRBui:
3370 return AArch64::LDRBroW;
3371 case AArch64::LDRBBroX:
3372 case AArch64::LDURBBi:
3373 case AArch64::LDRBBui:
3374 return AArch64::LDRBBroW;
3375 case AArch64::LDRSBXroX:
3376 case AArch64::LDURSBXi:
3377 case AArch64::LDRSBXui:
3378 return AArch64::LDRSBXroW;
3379 case AArch64::LDRSBWroX:
3380 case AArch64::LDURSBWi:
3381 case AArch64::LDRSBWui:
3382 return AArch64::LDRSBWroW;
3383 case AArch64::STRBroX:
3384 case AArch64::STURBi:
3385 case AArch64::STRBui:
3386 return AArch64::STRBroW;
3387 case AArch64::STRBBroX:
3388 case AArch64::STURBBi:
3389 case AArch64::STRBBui:
3390 return AArch64::STRBBroW;
3405 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3415 return B.getInstr();
3419 "Addressing mode not supported for folding");
3436 return B.getInstr();
3443 "Address offset can be a register or an immediate, but not both");
3445 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3450 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3464 return B.getInstr();
3468 "Function must not be called with an addressing mode it can't handle");
3473 bool &OffsetIsScalable,
TypeSize &Width,
3494 int64_t Dummy1, Dummy2;
3513 if (!BaseOp->
isReg() && !BaseOp->
isFI())
3523 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
3528 TypeSize &Width, int64_t &MinOffset,
3529 int64_t &MaxOffset) {
3535 MinOffset = MaxOffset = 0;
3537 case AArch64::STRWpost:
3538 case AArch64::LDRWpost:
3544 case AArch64::LDURQi:
3545 case AArch64::STURQi:
3551 case AArch64::PRFUMi:
3552 case AArch64::LDURXi:
3553 case AArch64::LDURDi:
3554 case AArch64::LDAPURXi:
3555 case AArch64::STURXi:
3556 case AArch64::STURDi:
3557 case AArch64::STLURXi:
3563 case AArch64::LDURWi:
3564 case AArch64::LDURSi:
3565 case AArch64::LDURSWi:
3566 case AArch64::LDAPURi:
3567 case AArch64::LDAPURSWi:
3568 case AArch64::STURWi:
3569 case AArch64::STURSi:
3570 case AArch64::STLURWi:
3576 case AArch64::LDURHi:
3577 case AArch64::LDURHHi:
3578 case AArch64::LDURSHXi:
3579 case AArch64::LDURSHWi:
3580 case AArch64::LDAPURHi:
3581 case AArch64::LDAPURSHWi:
3582 case AArch64::LDAPURSHXi:
3583 case AArch64::STURHi:
3584 case AArch64::STURHHi:
3585 case AArch64::STLURHi:
3591 case AArch64::LDURBi:
3592 case AArch64::LDURBBi:
3593 case AArch64::LDURSBXi:
3594 case AArch64::LDURSBWi:
3595 case AArch64::LDAPURBi:
3596 case AArch64::LDAPURSBWi:
3597 case AArch64::LDAPURSBXi:
3598 case AArch64::STURBi:
3599 case AArch64::STURBBi:
3600 case AArch64::STLURBi:
3606 case AArch64::LDPQi:
3607 case AArch64::LDNPQi:
3608 case AArch64::STPQi:
3609 case AArch64::STNPQi:
3615 case AArch64::LDRQui:
3616 case AArch64::STRQui:
3622 case AArch64::LDPXi:
3623 case AArch64::LDPDi:
3624 case AArch64::LDNPXi:
3625 case AArch64::LDNPDi:
3626 case AArch64::STPXi:
3627 case AArch64::STPDi:
3628 case AArch64::STNPXi:
3629 case AArch64::STNPDi:
3635 case AArch64::PRFMui:
3636 case AArch64::LDRXui:
3637 case AArch64::LDRDui:
3638 case AArch64::STRXui:
3639 case AArch64::STRDui:
3645 case AArch64::StoreSwiftAsyncContext:
3652 case AArch64::LDPWi:
3653 case AArch64::LDPSi:
3654 case AArch64::LDNPWi:
3655 case AArch64::LDNPSi:
3656 case AArch64::STPWi:
3657 case AArch64::STPSi:
3658 case AArch64::STNPWi:
3659 case AArch64::STNPSi:
3665 case AArch64::LDRWui:
3666 case AArch64::LDRSui:
3667 case AArch64::LDRSWui:
3668 case AArch64::STRWui:
3669 case AArch64::STRSui:
3675 case AArch64::LDRHui:
3676 case AArch64::LDRHHui:
3677 case AArch64::LDRSHWui:
3678 case AArch64::LDRSHXui:
3679 case AArch64::STRHui:
3680 case AArch64::STRHHui:
3686 case AArch64::LDRBui:
3687 case AArch64::LDRBBui:
3688 case AArch64::LDRSBWui:
3689 case AArch64::LDRSBXui:
3690 case AArch64::STRBui:
3691 case AArch64::STRBBui:
3697 case AArch64::STPXpre:
3698 case AArch64::LDPXpost:
3699 case AArch64::STPDpre:
3700 case AArch64::LDPDpost:
3706 case AArch64::STPQpre:
3707 case AArch64::LDPQpost:
3713 case AArch64::STRXpre:
3714 case AArch64::STRDpre:
3715 case AArch64::LDRXpost:
3716 case AArch64::LDRDpost:
3722 case AArch64::STRQpre:
3723 case AArch64::LDRQpost:
3735 case AArch64::TAGPstack:
3745 case AArch64::STZGi:
3751 case AArch64::STR_ZZZZXI:
3752 case AArch64::LDR_ZZZZXI:
3758 case AArch64::STR_ZZZXI:
3759 case AArch64::LDR_ZZZXI:
3765 case AArch64::STR_ZZXI:
3766 case AArch64::LDR_ZZXI:
3772 case AArch64::LDR_PXI:
3773 case AArch64::STR_PXI:
3779 case AArch64::LDR_PPXI:
3780 case AArch64::STR_PPXI:
3786 case AArch64::LDR_ZXI:
3787 case AArch64::STR_ZXI:
3793 case AArch64::LD1B_IMM:
3794 case AArch64::LD1H_IMM:
3795 case AArch64::LD1W_IMM:
3796 case AArch64::LD1D_IMM:
3797 case AArch64::LDNT1B_ZRI:
3798 case AArch64::LDNT1H_ZRI:
3799 case AArch64::LDNT1W_ZRI:
3800 case AArch64::LDNT1D_ZRI:
3801 case AArch64::ST1B_IMM:
3802 case AArch64::ST1H_IMM:
3803 case AArch64::ST1W_IMM:
3804 case AArch64::ST1D_IMM:
3805 case AArch64::STNT1B_ZRI:
3806 case AArch64::STNT1H_ZRI:
3807 case AArch64::STNT1W_ZRI:
3808 case AArch64::STNT1D_ZRI:
3809 case AArch64::LDNF1B_IMM:
3810 case AArch64::LDNF1H_IMM:
3811 case AArch64::LDNF1W_IMM:
3812 case AArch64::LDNF1D_IMM:
3820 case AArch64::LD2B_IMM:
3821 case AArch64::LD2H_IMM:
3822 case AArch64::LD2W_IMM:
3823 case AArch64::LD2D_IMM:
3824 case AArch64::ST2B_IMM:
3825 case AArch64::ST2H_IMM:
3826 case AArch64::ST2W_IMM:
3827 case AArch64::ST2D_IMM:
3833 case AArch64::LD3B_IMM:
3834 case AArch64::LD3H_IMM:
3835 case AArch64::LD3W_IMM:
3836 case AArch64::LD3D_IMM:
3837 case AArch64::ST3B_IMM:
3838 case AArch64::ST3H_IMM:
3839 case AArch64::ST3W_IMM:
3840 case AArch64::ST3D_IMM:
3846 case AArch64::LD4B_IMM:
3847 case AArch64::LD4H_IMM:
3848 case AArch64::LD4W_IMM:
3849 case AArch64::LD4D_IMM:
3850 case AArch64::ST4B_IMM:
3851 case AArch64::ST4H_IMM:
3852 case AArch64::ST4W_IMM:
3853 case AArch64::ST4D_IMM:
3859 case AArch64::LD1B_H_IMM:
3860 case AArch64::LD1SB_H_IMM:
3861 case AArch64::LD1H_S_IMM:
3862 case AArch64::LD1SH_S_IMM:
3863 case AArch64::LD1W_D_IMM:
3864 case AArch64::LD1SW_D_IMM:
3865 case AArch64::ST1B_H_IMM:
3866 case AArch64::ST1H_S_IMM:
3867 case AArch64::ST1W_D_IMM:
3868 case AArch64::LDNF1B_H_IMM:
3869 case AArch64::LDNF1SB_H_IMM:
3870 case AArch64::LDNF1H_S_IMM:
3871 case AArch64::LDNF1SH_S_IMM:
3872 case AArch64::LDNF1W_D_IMM:
3873 case AArch64::LDNF1SW_D_IMM:
3881 case AArch64::LD1B_S_IMM:
3882 case AArch64::LD1SB_S_IMM:
3883 case AArch64::LD1H_D_IMM:
3884 case AArch64::LD1SH_D_IMM:
3885 case AArch64::ST1B_S_IMM:
3886 case AArch64::ST1H_D_IMM:
3887 case AArch64::LDNF1B_S_IMM:
3888 case AArch64::LDNF1SB_S_IMM:
3889 case AArch64::LDNF1H_D_IMM:
3890 case AArch64::LDNF1SH_D_IMM:
3898 case AArch64::LD1B_D_IMM:
3899 case AArch64::LD1SB_D_IMM:
3900 case AArch64::ST1B_D_IMM:
3901 case AArch64::LDNF1B_D_IMM:
3902 case AArch64::LDNF1SB_D_IMM:
3910 case AArch64::ST2Gi:
3911 case AArch64::STZ2Gi:
3917 case AArch64::STGPi:
3923 case AArch64::LD1RB_IMM:
3924 case AArch64::LD1RB_H_IMM:
3925 case AArch64::LD1RB_S_IMM:
3926 case AArch64::LD1RB_D_IMM:
3927 case AArch64::LD1RSB_H_IMM:
3928 case AArch64::LD1RSB_S_IMM:
3929 case AArch64::LD1RSB_D_IMM:
3935 case AArch64::LD1RH_IMM:
3936 case AArch64::LD1RH_S_IMM:
3937 case AArch64::LD1RH_D_IMM:
3938 case AArch64::LD1RSH_S_IMM:
3939 case AArch64::LD1RSH_D_IMM:
3945 case AArch64::LD1RW_IMM:
3946 case AArch64::LD1RW_D_IMM:
3947 case AArch64::LD1RSW_IMM:
3953 case AArch64::LD1RD_IMM:
3969 case AArch64::LDRBBui:
3970 case AArch64::LDURBBi:
3971 case AArch64::LDRSBWui:
3972 case AArch64::LDURSBWi:
3973 case AArch64::STRBBui:
3974 case AArch64::STURBBi:
3976 case AArch64::LDRHHui:
3977 case AArch64::LDURHHi:
3978 case AArch64::LDRSHWui:
3979 case AArch64::LDURSHWi:
3980 case AArch64::STRHHui:
3981 case AArch64::STURHHi:
3983 case AArch64::LDRSui:
3984 case AArch64::LDURSi:
3985 case AArch64::LDRSpre:
3986 case AArch64::LDRSWui:
3987 case AArch64::LDURSWi:
3988 case AArch64::LDRSWpre:
3989 case AArch64::LDRWpre:
3990 case AArch64::LDRWui:
3991 case AArch64::LDURWi:
3992 case AArch64::STRSui:
3993 case AArch64::STURSi:
3994 case AArch64::STRSpre:
3995 case AArch64::STRWui:
3996 case AArch64::STURWi:
3997 case AArch64::STRWpre:
3998 case AArch64::LDPSi:
3999 case AArch64::LDPSWi:
4000 case AArch64::LDPWi:
4001 case AArch64::STPSi:
4002 case AArch64::STPWi:
4004 case AArch64::LDRDui:
4005 case AArch64::LDURDi:
4006 case AArch64::LDRDpre:
4007 case AArch64::LDRXui:
4008 case AArch64::LDURXi:
4009 case AArch64::LDRXpre:
4010 case AArch64::STRDui:
4011 case AArch64::STURDi:
4012 case AArch64::STRDpre:
4013 case AArch64::STRXui:
4014 case AArch64::STURXi:
4015 case AArch64::STRXpre:
4016 case AArch64::LDPDi:
4017 case AArch64::LDPXi:
4018 case AArch64::STPDi:
4019 case AArch64::STPXi:
4021 case AArch64::LDRQui:
4022 case AArch64::LDURQi:
4023 case AArch64::STRQui:
4024 case AArch64::STURQi:
4025 case AArch64::STRQpre:
4026 case AArch64::LDPQi:
4027 case AArch64::LDRQpre:
4028 case AArch64::STPQi:
4030 case AArch64::STZGi:
4031 case AArch64::ST2Gi:
4032 case AArch64::STZ2Gi:
4033 case AArch64::STGPi:
4039 switch (
MI.getOpcode()) {
4042 case AArch64::LDRWpre:
4043 case AArch64::LDRXpre:
4044 case AArch64::LDRSWpre:
4045 case AArch64::LDRSpre:
4046 case AArch64::LDRDpre:
4047 case AArch64::LDRQpre:
4053 switch (
MI.getOpcode()) {
4056 case AArch64::STRWpre:
4057 case AArch64::STRXpre:
4058 case AArch64::STRSpre:
4059 case AArch64::STRDpre:
4060 case AArch64::STRQpre:
4070 switch (
MI.getOpcode()) {
4073 case AArch64::LDPSi:
4074 case AArch64::LDPSWi:
4075 case AArch64::LDPDi:
4076 case AArch64::LDPQi:
4077 case AArch64::LDPWi:
4078 case AArch64::LDPXi:
4079 case AArch64::STPSi:
4080 case AArch64::STPDi:
4081 case AArch64::STPQi:
4082 case AArch64::STPWi:
4083 case AArch64::STPXi:
4084 case AArch64::STGPi:
4093 return MI.getOperand(
Idx);
4101 return MI.getOperand(
Idx);
4106 if (
MI.getParent() ==
nullptr)
4116 auto Reg =
Op.getReg();
4117 if (Reg.isPhysical())
4118 return AArch64::FPR16RegClass.
contains(Reg);
4120 return TRC == &AArch64::FPR16RegClass ||
4121 TRC == &AArch64::FPR16_loRegClass;
4130 auto Reg =
Op.getReg();
4131 if (Reg.isPhysical())
4132 return AArch64::FPR128RegClass.
contains(Reg);
4134 return TRC == &AArch64::FPR128RegClass ||
4135 TRC == &AArch64::FPR128_loRegClass;
4141 switch (
MI.getOpcode()) {
4144 case AArch64::PACIASP:
4145 case AArch64::PACIBSP:
4148 case AArch64::PAUTH_PROLOGUE:
4151 case AArch64::HINT: {
4152 unsigned Imm =
MI.getOperand(0).getImm();
4154 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4157 if (Imm == 25 || Imm == 27)
4170 auto Reg =
Op.getReg();
4171 if (Reg.isPhysical())
4172 return AArch64::FPR128RegClass.
contains(Reg) ||
4173 AArch64::FPR64RegClass.contains(Reg) ||
4174 AArch64::FPR32RegClass.contains(Reg) ||
4175 AArch64::FPR16RegClass.contains(Reg) ||
4176 AArch64::FPR8RegClass.contains(Reg);
4179 return TRC == &AArch64::FPR128RegClass ||
4180 TRC == &AArch64::FPR128_loRegClass ||
4181 TRC == &AArch64::FPR64RegClass ||
4182 TRC == &AArch64::FPR64_loRegClass ||
4183 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4184 TRC == &AArch64::FPR8RegClass;
4206 if (FirstOpc == SecondOpc)
4212 case AArch64::STRSui:
4213 case AArch64::STURSi:
4214 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4215 case AArch64::STRDui:
4216 case AArch64::STURDi:
4217 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4218 case AArch64::STRQui:
4219 case AArch64::STURQi:
4220 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4221 case AArch64::STRWui:
4222 case AArch64::STURWi:
4223 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4224 case AArch64::STRXui:
4225 case AArch64::STURXi:
4226 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4227 case AArch64::LDRSui:
4228 case AArch64::LDURSi:
4229 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4230 case AArch64::LDRDui:
4231 case AArch64::LDURDi:
4232 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4233 case AArch64::LDRQui:
4234 case AArch64::LDURQi:
4235 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4236 case AArch64::LDRWui:
4237 case AArch64::LDURWi:
4238 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4239 case AArch64::LDRSWui:
4240 case AArch64::LDURSWi:
4241 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4242 case AArch64::LDRXui:
4243 case AArch64::LDURXi:
4244 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4251 int64_t Offset1,
unsigned Opcode1,
int FI2,
4252 int64_t Offset2,
unsigned Opcode2) {
4258 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4261 if (ObjectOffset1 % Scale1 != 0)
4263 ObjectOffset1 /= Scale1;
4265 if (ObjectOffset2 % Scale2 != 0)
4267 ObjectOffset2 /= Scale2;
4268 ObjectOffset1 += Offset1;
4269 ObjectOffset2 += Offset2;
4270 return ObjectOffset1 + 1 == ObjectOffset2;
4282 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4283 unsigned NumBytes)
const {
4293 "Only base registers and frame indices are supported.");
4300 if (ClusterSize > 2)
4307 unsigned FirstOpc = FirstLdSt.
getOpcode();
4308 unsigned SecondOpc = SecondLdSt.
getOpcode();
4328 if (Offset1 > 63 || Offset1 < -64)
4333 if (BaseOp1.
isFI()) {
4335 "Caller should have ordered offsets.");
4340 BaseOp2.
getIndex(), Offset2, SecondOpc);
4343 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
4345 return Offset1 + 1 == Offset2;
4349 unsigned Reg,
unsigned SubIdx,
4353 return MIB.
addReg(Reg, State);
4356 return MIB.
addReg(
TRI->getSubReg(Reg, SubIdx), State);
4357 return MIB.
addReg(Reg, State, SubIdx);
4364 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
4373 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
4375 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4376 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4377 unsigned NumRegs = Indices.
size();
4379 int SubReg = 0,
End = NumRegs, Incr = 1;
4397 unsigned SrcReg,
bool KillSrc,
4398 unsigned Opcode,
unsigned ZeroReg,
4401 unsigned NumRegs = Indices.
size();
4404 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4405 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4406 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
4407 "GPR reg sequences should not be able to overlap");
4423 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
4424 (AArch64::GPR32spRegClass.
contains(SrcReg) || SrcReg == AArch64::WZR)) {
4427 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
4429 if (Subtarget.hasZeroCycleRegMove()) {
4432 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4434 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4450 }
else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
4455 if (Subtarget.hasZeroCycleRegMove()) {
4458 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4460 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4480 if (AArch64::PPRRegClass.
contains(DestReg) &&
4481 AArch64::PPRRegClass.
contains(SrcReg)) {
4492 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
4493 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
4494 if (DestIsPNR || SrcIsPNR) {
4495 assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4496 "Unexpected predicate-as-counter register.");
4498 return (R - AArch64::PN0) + AArch64::P0;
4500 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg;
4501 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg;
4503 if (PPRSrcReg != PPRDestReg) {
4515 if (AArch64::ZPRRegClass.
contains(DestReg) &&
4516 AArch64::ZPRRegClass.
contains(SrcReg)) {
4525 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
4526 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
4527 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
4528 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
4530 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
4537 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
4538 AArch64::ZPR3RegClass.
contains(SrcReg)) {
4540 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4548 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
4549 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
4550 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
4551 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
4553 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4554 AArch64::zsub2, AArch64::zsub3};
4560 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
4561 (AArch64::GPR64spRegClass.
contains(SrcReg) || SrcReg == AArch64::XZR)) {
4562 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
4568 }
else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
4582 if (AArch64::DDDDRegClass.
contains(DestReg) &&
4583 AArch64::DDDDRegClass.
contains(SrcReg)) {
4584 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
4585 AArch64::dsub2, AArch64::dsub3};
4592 if (AArch64::DDDRegClass.
contains(DestReg) &&
4593 AArch64::DDDRegClass.
contains(SrcReg)) {
4594 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
4602 if (AArch64::DDRegClass.
contains(DestReg) &&
4603 AArch64::DDRegClass.
contains(SrcReg)) {
4604 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
4611 if (AArch64::QQQQRegClass.
contains(DestReg) &&
4612 AArch64::QQQQRegClass.
contains(SrcReg)) {
4613 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
4614 AArch64::qsub2, AArch64::qsub3};
4621 if (AArch64::QQQRegClass.
contains(DestReg) &&
4622 AArch64::QQQRegClass.
contains(SrcReg)) {
4623 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
4631 if (AArch64::QQRegClass.
contains(DestReg) &&
4632 AArch64::QQRegClass.
contains(SrcReg)) {
4633 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
4639 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
4640 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
4641 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
4643 AArch64::XZR, Indices);
4647 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
4648 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
4649 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
4651 AArch64::WZR, Indices);
4655 if (AArch64::FPR128RegClass.
contains(DestReg) &&
4656 AArch64::FPR128RegClass.
contains(SrcReg)) {
4660 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
4661 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
4662 else if (Subtarget.hasNEON())
4681 if (AArch64::FPR64RegClass.
contains(DestReg) &&
4682 AArch64::FPR64RegClass.
contains(SrcReg)) {
4688 if (AArch64::FPR32RegClass.
contains(DestReg) &&
4689 AArch64::FPR32RegClass.
contains(SrcReg)) {
4695 if (AArch64::FPR16RegClass.
contains(DestReg) &&
4696 AArch64::FPR16RegClass.
contains(SrcReg)) {
4698 RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
4700 RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
4706 if (AArch64::FPR8RegClass.
contains(DestReg) &&
4707 AArch64::FPR8RegClass.
contains(SrcReg)) {
4709 RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
4711 RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
4718 if (AArch64::FPR64RegClass.
contains(DestReg) &&
4719 AArch64::GPR64RegClass.
contains(SrcReg)) {
4724 if (AArch64::GPR64RegClass.
contains(DestReg) &&
4725 AArch64::FPR64RegClass.
contains(SrcReg)) {
4731 if (AArch64::FPR32RegClass.
contains(DestReg) &&
4732 AArch64::GPR32RegClass.
contains(SrcReg)) {
4737 if (AArch64::GPR32RegClass.
contains(DestReg) &&
4738 AArch64::FPR32RegClass.
contains(SrcReg)) {
4744 if (DestReg == AArch64::NZCV) {
4745 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
4747 .
addImm(AArch64SysReg::NZCV)
4753 if (SrcReg == AArch64::NZCV) {
4754 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
4756 .
addImm(AArch64SysReg::NZCV)
4763 errs() <<
TRI.getRegAsmName(DestReg) <<
" = COPY "
4764 <<
TRI.getRegAsmName(SrcReg) <<
"\n";
4774 unsigned SubIdx0,
unsigned SubIdx1,
int FI,
4779 SrcReg0 =
TRI.getSubReg(SrcReg, SubIdx0);
4781 SrcReg1 =
TRI.getSubReg(SrcReg, SubIdx1);
4794 Register SrcReg,
bool isKill,
int FI,
4809 switch (
TRI->getSpillSize(*RC)) {
4811 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
4812 Opc = AArch64::STRBui;
4815 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
4816 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
4817 Opc = AArch64::STRHui;
4818 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
4820 "Unexpected register store without SVE store instructions");
4821 assert((!IsPNR || Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4822 "Unexpected register store without SVE2p1 or SME2");
4823 Opc = AArch64::STR_PXI;
4829 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
4830 Opc = AArch64::STRWui;
4834 assert(SrcReg != AArch64::WSP);
4835 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
4836 Opc = AArch64::STRSui;
4837 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
4838 Opc = AArch64::STR_PPXI;
4843 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
4844 Opc = AArch64::STRXui;
4848 assert(SrcReg != AArch64::SP);
4849 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
4850 Opc = AArch64::STRDui;
4851 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
4853 get(AArch64::STPWi), SrcReg, isKill,
4854 AArch64::sube32, AArch64::subo32, FI, MMO);
4859 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
4860 Opc = AArch64::STRQui;
4861 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
4862 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4863 Opc = AArch64::ST1Twov1d;
4865 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
4867 get(AArch64::STPXi), SrcReg, isKill,
4868 AArch64::sube64, AArch64::subo64, FI, MMO);
4870 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
4872 "Unexpected register store without SVE store instructions");
4873 Opc = AArch64::STR_ZXI;
4878 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
4879 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4880 Opc = AArch64::ST1Threev1d;
4885 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
4886 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4887 Opc = AArch64::ST1Fourv1d;
4889 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
4890 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4891 Opc = AArch64::ST1Twov2d;
4893 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
4894 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4896 "Unexpected register store without SVE store instructions");
4897 Opc = AArch64::STR_ZZXI;
4902 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
4903 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4904 Opc = AArch64::ST1Threev2d;
4906 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
4908 "Unexpected register store without SVE store instructions");
4909 Opc = AArch64::STR_ZZZXI;
4914 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
4915 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4916 Opc = AArch64::ST1Fourv2d;
4918 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
4919 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4921 "Unexpected register store without SVE store instructions");
4922 Opc = AArch64::STR_ZZZZXI;
4927 assert(Opc &&
"Unknown register class");
4938 MI.addMemOperand(MMO);
4945 Register DestReg,
unsigned SubIdx0,
4946 unsigned SubIdx1,
int FI,
4950 bool IsUndef =
true;
4952 DestReg0 =
TRI.getSubReg(DestReg, SubIdx0);
4954 DestReg1 =
TRI.getSubReg(DestReg, SubIdx1);
4983 switch (
TRI->getSpillSize(*RC)) {
4985 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
4986 Opc = AArch64::LDRBui;
4989 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
4990 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
4991 Opc = AArch64::LDRHui;
4992 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
4994 "Unexpected register load without SVE load instructions");
4995 assert((!IsPNR || Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4996 "Unexpected register load without SVE2p1 or SME2");
4999 Opc = AArch64::LDR_PXI;
5005 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5006 Opc = AArch64::LDRWui;
5010 assert(DestReg != AArch64::WSP);
5011 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5012 Opc = AArch64::LDRSui;
5013 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5014 Opc = AArch64::LDR_PPXI;
5019 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5020 Opc = AArch64::LDRXui;
5024 assert(DestReg != AArch64::SP);
5025 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5026 Opc = AArch64::LDRDui;
5027 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5029 get(AArch64::LDPWi), DestReg, AArch64::sube32,
5030 AArch64::subo32, FI, MMO);
5035 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5036 Opc = AArch64::LDRQui;
5037 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5038 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5039 Opc = AArch64::LD1Twov1d;
5041 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5043 get(AArch64::LDPXi), DestReg, AArch64::sube64,
5044 AArch64::subo64, FI, MMO);
5046 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5048 "Unexpected register load without SVE load instructions");
5049 Opc = AArch64::LDR_ZXI;
5054 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5055 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5056 Opc = AArch64::LD1Threev1d;
5061 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5062 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5063 Opc = AArch64::LD1Fourv1d;
5065 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5066 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5067 Opc = AArch64::LD1Twov2d;
5069 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
5070 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5072 "Unexpected register load without SVE load instructions");
5073 Opc = AArch64::LDR_ZZXI;
5078 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5079 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5080 Opc = AArch64::LD1Threev2d;
5082 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5084 "Unexpected register load without SVE load instructions");
5085 Opc = AArch64::LDR_ZZZXI;
5090 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5091 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5092 Opc = AArch64::LD1Fourv2d;
5094 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
5095 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5097 "Unexpected register load without SVE load instructions");
5098 Opc = AArch64::LDR_ZZZZXI;
5104 assert(Opc &&
"Unknown register class");
5114 MI.addMemOperand(MMO);
5125 UseMI.getIterator()),
5127 return I.modifiesRegister(AArch64::NZCV, TRI) ||
5128 I.readsRegister(AArch64::NZCV, TRI);
5137 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5144 ByteSized =
Offset.getFixed();
5145 VGSized =
Offset.getScalable() / 2;
5153 int64_t &NumDataVectors) {
5157 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5159 NumBytes =
Offset.getFixed();
5161 NumPredicateVectors =
Offset.getScalable() / 2;
5166 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
5167 NumPredicateVectors > 62) {
5168 NumDataVectors = NumPredicateVectors / 8;
5169 NumPredicateVectors -= NumDataVectors * 8;
5176 int NumVGScaledBytes,
unsigned VG,
5183 Expr.
push_back((uint8_t)dwarf::DW_OP_plus);
5184 Comment << (NumBytes < 0 ?
" - " :
" + ") << std::abs(NumBytes);
5187 if (NumVGScaledBytes) {
5188 Expr.
push_back((uint8_t)dwarf::DW_OP_consts);
5191 Expr.
push_back((uint8_t)dwarf::DW_OP_bregx);
5195 Expr.
push_back((uint8_t)dwarf::DW_OP_mul);
5196 Expr.
push_back((uint8_t)dwarf::DW_OP_plus);
5198 Comment << (NumVGScaledBytes < 0 ?
" - " :
" + ")
5199 << std::abs(NumVGScaledBytes) <<
" * VG";
5208 int64_t NumBytes, NumVGScaledBytes;
5211 std::string CommentBuffer;
5214 if (Reg == AArch64::SP)
5216 else if (Reg == AArch64::FP)
5223 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5224 Expr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
5227 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5231 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
5240 unsigned FrameReg,
unsigned Reg,
5242 bool LastAdjustmentWasScalable) {
5243 if (
Offset.getScalable())
5246 if (FrameReg == Reg && !LastAdjustmentWasScalable)
5249 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5256 int64_t NumBytes, NumVGScaledBytes;
5258 OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
5260 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5263 if (!NumVGScaledBytes)
5266 std::string CommentBuffer;
5273 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5277 CfaExpr.
push_back(dwarf::DW_CFA_expression);
5293 unsigned SrcReg, int64_t
Offset,
unsigned Opc,
5296 bool *HasWinCFI,
bool EmitCFAOffset,
5299 unsigned MaxEncoding, ShiftSize;
5301 case AArch64::ADDXri:
5302 case AArch64::ADDSXri:
5303 case AArch64::SUBXri:
5304 case AArch64::SUBSXri:
5305 MaxEncoding = 0xfff;
5308 case AArch64::ADDVL_XXI:
5309 case AArch64::ADDPL_XXI:
5310 case AArch64::ADDSVL_XXI:
5311 case AArch64::ADDSPL_XXI:
5326 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
5328 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
5342 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
5344 if (TmpReg == AArch64::XZR)
5346 &AArch64::GPR64RegClass);
5348 uint64_t ThisVal = std::min<uint64_t>(
Offset, MaxEncodableValue);
5349 unsigned LocalShiftSize = 0;
5350 if (ThisVal > MaxEncoding) {
5351 ThisVal = ThisVal >> ShiftSize;
5352 LocalShiftSize = ShiftSize;
5354 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
5355 "Encoding cannot handle value that big");
5357 Offset -= ThisVal << LocalShiftSize;
5362 .
addImm(Sign * (
int)ThisVal);
5372 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
5373 CFAOffset += Change;
5375 CFAOffset -= Change;
5376 if (EmitCFAOffset && DestReg == TmpReg) {
5389 assert(Sign == 1 &&
"SEH directives should always have a positive sign");
5390 int Imm = (int)(ThisVal << LocalShiftSize);
5391 if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
5392 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
5401 assert(
Offset == 0 &&
"Expected remaining offset to be zero to "
5402 "emit a single SEH directive");
5403 }
else if (DestReg == AArch64::SP) {
5406 assert(SrcReg == AArch64::SP &&
"Unexpected SrcReg for SEH_StackAlloc");
5419 unsigned DestReg,
unsigned SrcReg,
5422 bool NeedsWinCFI,
bool *HasWinCFI,
5424 unsigned FrameReg) {
5431 bool UseSVL =
F.hasFnAttribute(
"aarch64_pstate_sm_body");
5433 int64_t Bytes, NumPredicateVectors, NumDataVectors;
5435 Offset, Bytes, NumPredicateVectors, NumDataVectors);
5438 if (Bytes || (!
Offset && SrcReg != DestReg)) {
5439 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
5440 "SP increment/decrement not 8-byte aligned");
5441 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
5444 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
5447 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
5449 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
5456 assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
5457 "SetNZCV not supported with SVE vectors");
5458 assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
5459 "WinCFI not supported with SVE vectors");
5461 if (NumDataVectors) {
5463 UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
5464 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5465 CFAOffset, FrameReg);
5470 if (NumPredicateVectors) {
5471 assert(DestReg != AArch64::SP &&
"Unaligned access to SP");
5473 UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
5474 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5475 CFAOffset, FrameReg);
5493 if (
MI.isFullCopy()) {
5496 if (SrcReg == AArch64::SP && DstReg.
isVirtual()) {
5500 if (DstReg == AArch64::SP && SrcReg.
isVirtual()) {
5505 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
5533 if (
MI.isCopy() && Ops.
size() == 1 &&
5535 (Ops[0] == 0 || Ops[0] == 1)) {
5536 bool IsSpill = Ops[0] == 0;
5537 bool IsFill = !IsSpill;
5549 :
TRI.getMinimalPhysRegClass(Reg);
5555 "Mismatched register size in non subreg COPY");
5562 return &*--InsertPt;
5574 if (IsSpill && DstMO.
isUndef() && SrcReg == AArch64::WZR &&
5577 "Unexpected subreg on physical register");
5579 FrameIndex, &AArch64::GPR64RegClass, &
TRI,
5581 return &*--InsertPt;
5599 case AArch64::sub_32:
5600 FillRC = &AArch64::GPR32RegClass;
5603 FillRC = &AArch64::FPR32RegClass;
5606 FillRC = &AArch64::FPR64RegClass;
5612 TRI.getRegSizeInBits(*FillRC) &&
5613 "Mismatched regclass size on folded subreg COPY");
5632 bool *OutUseUnscaledOp,
5633 unsigned *OutUnscaledOp,
5634 int64_t *EmittableOffset) {
5636 if (EmittableOffset)
5637 *EmittableOffset = 0;
5638 if (OutUseUnscaledOp)
5639 *OutUseUnscaledOp =
false;
5645 switch (
MI.getOpcode()) {
5648 case AArch64::LD1Rv1d:
5649 case AArch64::LD1Rv2s:
5650 case AArch64::LD1Rv2d:
5651 case AArch64::LD1Rv4h:
5652 case AArch64::LD1Rv4s:
5653 case AArch64::LD1Rv8b:
5654 case AArch64::LD1Rv8h:
5655 case AArch64::LD1Rv16b:
5656 case AArch64::LD1Twov2d:
5657 case AArch64::LD1Threev2d:
5658 case AArch64::LD1Fourv2d:
5659 case AArch64::LD1Twov1d:
5660 case AArch64::LD1Threev1d:
5661 case AArch64::LD1Fourv1d:
5662 case AArch64::ST1Twov2d:
5663 case AArch64::ST1Threev2d:
5664 case AArch64::ST1Fourv2d:
5665 case AArch64::ST1Twov1d:
5666 case AArch64::ST1Threev1d:
5667 case AArch64::ST1Fourv1d:
5668 case AArch64::ST1i8:
5669 case AArch64::ST1i16:
5670 case AArch64::ST1i32:
5671 case AArch64::ST1i64:
5673 case AArch64::IRGstack:
5674 case AArch64::STGloop:
5675 case AArch64::STZGloop:
5680 TypeSize ScaleValue(0U,
false), Width(0U,
false);
5681 int64_t MinOff, MaxOff;
5687 bool IsMulVL = ScaleValue.isScalable();
5688 unsigned Scale = ScaleValue.getKnownMinValue();
5698 std::optional<unsigned> UnscaledOp =
5700 bool useUnscaledOp = UnscaledOp && (
Offset % Scale ||
Offset < 0);
5701 if (useUnscaledOp &&
5706 Scale = ScaleValue.getKnownMinValue();
5707 assert(IsMulVL == ScaleValue.isScalable() &&
5708 "Unscaled opcode has different value for scalable");
5710 int64_t Remainder =
Offset % Scale;
5711 assert(!(Remainder && useUnscaledOp) &&
5712 "Cannot have remainder when using unscaled op");
5714 assert(MinOff < MaxOff &&
"Unexpected Min/Max offsets");
5715 int64_t NewOffset =
Offset / Scale;
5716 if (MinOff <= NewOffset && NewOffset <= MaxOff)
5719 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
5723 if (EmittableOffset)
5724 *EmittableOffset = NewOffset;
5725 if (OutUseUnscaledOp)
5726 *OutUseUnscaledOp = useUnscaledOp;
5727 if (OutUnscaledOp && UnscaledOp)
5728 *OutUnscaledOp = *UnscaledOp;
5741 unsigned Opcode =
MI.getOpcode();
5742 unsigned ImmIdx = FrameRegIdx + 1;
5744 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
5749 MI.eraseFromParent();
5755 unsigned UnscaledOp;
5758 &UnscaledOp, &NewOffset);
5762 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg,
false);
5764 MI.setDesc(
TII->get(UnscaledOp));
5766 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
5789 case AArch64::ADDSWrr:
5790 case AArch64::ADDSWri:
5791 case AArch64::ADDSXrr:
5792 case AArch64::ADDSXri:
5793 case AArch64::SUBSWrr:
5794 case AArch64::SUBSXrr:
5796 case AArch64::SUBSWri:
5797 case AArch64::SUBSXri:
5808 case AArch64::ADDWrr:
5809 case AArch64::ADDWri:
5810 case AArch64::SUBWrr:
5811 case AArch64::ADDSWrr:
5812 case AArch64::ADDSWri:
5813 case AArch64::SUBSWrr:
5815 case AArch64::SUBWri:
5816 case AArch64::SUBSWri:
5827 case AArch64::ADDXrr:
5828 case AArch64::ADDXri:
5829 case AArch64::SUBXrr:
5830 case AArch64::ADDSXrr:
5831 case AArch64::ADDSXri:
5832 case AArch64::SUBSXrr:
5834 case AArch64::SUBXri:
5835 case AArch64::SUBSXri:
5836 case AArch64::ADDv8i8:
5837 case AArch64::ADDv16i8:
5838 case AArch64::ADDv4i16:
5839 case AArch64::ADDv8i16:
5840 case AArch64::ADDv2i32:
5841 case AArch64::ADDv4i32:
5842 case AArch64::SUBv8i8:
5843 case AArch64::SUBv16i8:
5844 case AArch64::SUBv4i16:
5845 case AArch64::SUBv8i16:
5846 case AArch64::SUBv2i32:
5847 case AArch64::SUBv4i32:
5860 case AArch64::FADDHrr:
5861 case AArch64::FADDSrr:
5862 case AArch64::FADDDrr:
5863 case AArch64::FADDv4f16:
5864 case AArch64::FADDv8f16:
5865 case AArch64::FADDv2f32:
5866 case AArch64::FADDv2f64:
5867 case AArch64::FADDv4f32:
5868 case AArch64::FSUBHrr:
5869 case AArch64::FSUBSrr:
5870 case AArch64::FSUBDrr:
5871 case AArch64::FSUBv4f16:
5872 case AArch64::FSUBv8f16:
5873 case AArch64::FSUBv2f32:
5874 case AArch64::FSUBv2f64:
5875 case AArch64::FSUBv4f32:
5879 return Options.UnsafeFPMath ||
5896 unsigned CombineOpc,
unsigned ZeroReg = 0,
5897 bool CheckZeroReg =
false) {
5904 if (!
MI ||
MI->getParent() != &
MBB || (
unsigned)
MI->getOpcode() != CombineOpc)
5907 if (!
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
5911 assert(
MI->getNumOperands() >= 4 &&
MI->getOperand(0).isReg() &&
5912 MI->getOperand(1).isReg() &&
MI->getOperand(2).isReg() &&
5913 MI->getOperand(3).isReg() &&
"MAdd/MSub must have a least 4 regs");
5915 if (
MI->getOperand(3).getReg() != ZeroReg)
5920 MI->findRegisterDefOperandIdx(AArch64::NZCV,
nullptr,
true) == -1)
5929 unsigned MulOpc,
unsigned ZeroReg) {
5945 bool Invert)
const {
5951 case AArch64::FADDHrr:
5952 case AArch64::FADDSrr:
5953 case AArch64::FADDDrr:
5954 case AArch64::FMULHrr:
5955 case AArch64::FMULSrr:
5956 case AArch64::FMULDrr:
5957 case AArch64::FMULX16:
5958 case AArch64::FMULX32:
5959 case AArch64::FMULX64:
5961 case AArch64::FADDv4f16:
5962 case AArch64::FADDv8f16:
5963 case AArch64::FADDv2f32:
5964 case AArch64::FADDv4f32:
5965 case AArch64::FADDv2f64:
5966 case AArch64::FMULv4f16:
5967 case AArch64::FMULv8f16:
5968 case AArch64::FMULv2f32:
5969 case AArch64::FMULv4f32:
5970 case AArch64::FMULv2f64:
5971 case AArch64::FMULXv4f16:
5972 case AArch64::FMULXv8f16:
5973 case AArch64::FMULXv2f32:
5974 case AArch64::FMULXv4f32:
5975 case AArch64::FMULXv2f64:
5979 case AArch64::FADD_ZZZ_H:
5980 case AArch64::FADD_ZZZ_S:
5981 case AArch64::FADD_ZZZ_D:
5982 case AArch64::FMUL_ZZZ_H:
5983 case AArch64::FMUL_ZZZ_S:
5984 case AArch64::FMUL_ZZZ_D:
5996 case AArch64::ADDWrr:
5997 case AArch64::ADDXrr:
5998 case AArch64::ANDWrr:
5999 case AArch64::ANDXrr:
6000 case AArch64::ORRWrr:
6001 case AArch64::ORRXrr:
6002 case AArch64::EORWrr:
6003 case AArch64::EORXrr:
6004 case AArch64::EONWrr:
6005 case AArch64::EONXrr:
6009 case AArch64::ADDv8i8:
6010 case AArch64::ADDv16i8:
6011 case AArch64::ADDv4i16:
6012 case AArch64::ADDv8i16:
6013 case AArch64::ADDv2i32:
6014 case AArch64::ADDv4i32:
6015 case AArch64::ADDv1i64:
6016 case AArch64::ADDv2i64:
6017 case AArch64::MULv8i8:
6018 case AArch64::MULv16i8:
6019 case AArch64::MULv4i16:
6020 case AArch64::MULv8i16:
6021 case AArch64::MULv2i32:
6022 case AArch64::MULv4i32:
6023 case AArch64::ANDv8i8:
6024 case AArch64::ANDv16i8:
6025 case AArch64::ORRv8i8:
6026 case AArch64::ORRv16i8:
6027 case AArch64::EORv8i8:
6028 case AArch64::EORv16i8:
6030 case AArch64::ADD_ZZZ_B:
6031 case AArch64::ADD_ZZZ_H:
6032 case AArch64::ADD_ZZZ_S:
6033 case AArch64::ADD_ZZZ_D:
6034 case AArch64::MUL_ZZZ_B:
6035 case AArch64::MUL_ZZZ_H:
6036 case AArch64::MUL_ZZZ_S:
6037 case AArch64::MUL_ZZZ_D:
6038 case AArch64::AND_ZZZ:
6039 case AArch64::ORR_ZZZ:
6040 case AArch64::EOR_ZZZ:
6071 auto setFound = [&](
int Opcode,
int Operand,
unsigned ZeroReg,
6079 auto setVFound = [&](
int Opcode,
int Operand,
unsigned Pattern) {
6091 case AArch64::ADDWrr:
6093 "ADDWrr does not have register operands");
6094 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
6095 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
6097 case AArch64::ADDXrr:
6098 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
6099 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
6101 case AArch64::SUBWrr:
6102 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
6103 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
6105 case AArch64::SUBXrr:
6106 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
6107 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
6109 case AArch64::ADDWri:
6110 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
6112 case AArch64::ADDXri:
6113 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
6115 case AArch64::SUBWri:
6116 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
6118 case AArch64::SUBXri:
6119 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
6121 case AArch64::ADDv8i8:
6122 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
6123 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
6125 case AArch64::ADDv16i8:
6126 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
6127 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
6129 case AArch64::ADDv4i16:
6130 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
6131 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
6132 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
6133 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
6135 case AArch64::ADDv8i16:
6136 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
6137 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
6138 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
6139 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
6141 case AArch64::ADDv2i32:
6142 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
6143 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
6144 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
6145 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
6147 case AArch64::ADDv4i32:
6148 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
6149 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
6150 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
6151 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
6153 case AArch64::SUBv8i8:
6154 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
6155 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
6157 case AArch64::SUBv16i8:
6158 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
6159 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
6161 case AArch64::SUBv4i16:
6162 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
6163 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
6164 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
6165 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
6167 case AArch64::SUBv8i16:
6168 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
6169 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
6170 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
6171 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
6173 case AArch64::SUBv2i32:
6174 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
6175 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
6176 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
6177 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
6179 case AArch64::SUBv4i32:
6180 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
6181 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
6182 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
6183 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
6200 auto Match = [&](
int Opcode,
int Operand,
unsigned Pattern) ->
bool {
6212 assert(
false &&
"Unsupported FP instruction in combiner\n");
6214 case AArch64::FADDHrr:
6216 "FADDHrr does not have register operands");
6218 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
6219 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
6221 case AArch64::FADDSrr:
6223 "FADDSrr does not have register operands");
6225 Found |=
Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
6226 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
6228 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
6229 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
6231 case AArch64::FADDDrr:
6232 Found |=
Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
6233 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
6235 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
6236 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
6238 case AArch64::FADDv4f16:
6239 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
6240 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
6242 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
6243 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
6245 case AArch64::FADDv8f16:
6246 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
6247 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
6249 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
6250 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
6252 case AArch64::FADDv2f32:
6253 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
6254 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
6256 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
6257 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
6259 case AArch64::FADDv2f64:
6260 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
6261 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
6263 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
6264 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
6266 case AArch64::FADDv4f32:
6267 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
6268 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
6270 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
6271 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
6273 case AArch64::FSUBHrr:
6274 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
6275 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
6276 Found |=
Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
6278 case AArch64::FSUBSrr:
6279 Found =
Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
6281 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
6282 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
6284 Found |=
Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
6286 case AArch64::FSUBDrr:
6287 Found =
Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
6289 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
6290 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
6292 Found |=
Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
6294 case AArch64::FSUBv4f16:
6295 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
6296 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
6298 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
6299 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
6301 case AArch64::FSUBv8f16:
6302 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
6303 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
6305 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
6306 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
6308 case AArch64::FSUBv2f32:
6309 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
6310 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
6312 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
6313 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
6315 case AArch64::FSUBv2f64:
6316 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
6317 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
6319 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
6320 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
6322 case AArch64::FSUBv4f32:
6323 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
6324 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
6326 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
6327 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
6338 auto Match = [&](
unsigned Opcode,
int Operand,
unsigned Pattern) ->
bool {
6345 if (
MI &&
MI->getOpcode() == TargetOpcode::COPY &&
6346 MI->getOperand(1).getReg().isVirtual())
6347 MI =
MRI.getUniqueVRegDef(
MI->getOperand(1).getReg());
6348 if (
MI &&
MI->getOpcode() == Opcode) {
6360 case AArch64::FMULv2f32:
6361 Found =
Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
6362 Found |=
Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
6364 case AArch64::FMULv2f64:
6365 Found =
Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
6366 Found |=
Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
6368 case AArch64::FMULv4f16:
6369 Found =
Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
6370 Found |=
Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
6372 case AArch64::FMULv4f32:
6373 Found =
Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
6374 Found |=
Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
6376 case AArch64::FMULv8f16:
6377 Found =
Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
6378 Found |=
Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
6391 auto Match = [&](
unsigned Opcode,
unsigned Pattern) ->
bool {
6394 if (
MI !=
nullptr && (
MI->getOpcode() == Opcode) &&
6395 MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()) &&
6409 case AArch64::FNEGDr:
6411 case AArch64::FNEGSr:
6543 case AArch64::SUBWrr:
6544 case AArch64::SUBSWrr:
6545 case AArch64::SUBXrr:
6546 case AArch64::SUBSXrr:
6588 bool DoRegPressureReduce)
const {
6605 DoRegPressureReduce);
6634 const Register *ReplacedAddend =
nullptr) {
6635 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
6637 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
6640 Register SrcReg0 = MUL->getOperand(1).getReg();
6641 bool Src0IsKill = MUL->getOperand(1).isKill();
6642 Register SrcReg1 = MUL->getOperand(2).getReg();
6643 bool Src1IsKill = MUL->getOperand(2).isKill();
6647 if (ReplacedAddend) {
6649 SrcReg2 = *ReplacedAddend;
6657 MRI.constrainRegClass(ResultReg, RC);
6659 MRI.constrainRegClass(SrcReg0, RC);
6661 MRI.constrainRegClass(SrcReg1, RC);
6663 MRI.constrainRegClass(SrcReg2, RC);
6666 if (kind == FMAInstKind::Default)
6671 else if (kind == FMAInstKind::Indexed)
6676 .
addImm(MUL->getOperand(3).getImm());
6677 else if (kind == FMAInstKind::Accumulator)
6683 assert(
false &&
"Invalid FMA instruction kind \n");
6697 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6698 Opc = AArch64::FNMADDSrrr;
6699 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
6700 Opc = AArch64::FNMADDDrrr;
6712 MRI.constrainRegClass(ResultReg, RC);
6714 MRI.constrainRegClass(SrcReg0, RC);
6716 MRI.constrainRegClass(SrcReg1, RC);
6718 MRI.constrainRegClass(SrcReg2, RC);
6734 unsigned IdxDupOp,
unsigned MulOpc,
6736 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
6737 "Invalid index of FMUL operand");
6745 if (Dup->
getOpcode() == TargetOpcode::COPY)
6749 MRI.clearKillFlags(DupSrcReg);
6750 MRI.constrainRegClass(DupSrcReg, RC);
6754 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
6778 FMAInstKind::Accumulator);
6795 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
6810 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
6812 FMAInstKind::Accumulator, &NewVR);
6824 FMAInstKind::Indexed);
6837 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
6840 FMAInstKind::Indexed, &NewVR);
6865 unsigned IdxMulOpd,
unsigned MaddOpc,
unsigned VR,
6867 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
6871 Register SrcReg0 = MUL->getOperand(1).getReg();
6872 bool Src0IsKill = MUL->getOperand(1).isKill();
6873 Register SrcReg1 = MUL->getOperand(2).getReg();
6874 bool Src1IsKill = MUL->getOperand(2).isKill();
6877 MRI.constrainRegClass(ResultReg, RC);
6879 MRI.constrainRegClass(SrcReg0, RC);
6881 MRI.constrainRegClass(SrcReg1, RC);
6883 MRI.constrainRegClass(VR, RC);
6905 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
6906 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
6916 Register NewVR =
MRI.createVirtualRegister(
MRI.getRegClass(RegA));
6919 if (Opcode == AArch64::SUBSWrr)
6920 Opcode = AArch64::SUBWrr;
6921 else if (Opcode == AArch64::SUBSXrr)
6922 Opcode = AArch64::SUBXrr;
6924 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
6925 "Unexpected instruction opcode.");
6936 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
6962 DelInstrs, InstrIdxForVirtReg);
6968 InstrIdxForVirtReg);
6974 InstrIdxForVirtReg);
6983 Opc = AArch64::MADDWrrr;
6984 RC = &AArch64::GPR32RegClass;
6986 Opc = AArch64::MADDXrrr;
6987 RC = &AArch64::GPR64RegClass;
6998 Opc = AArch64::MADDWrrr;
6999 RC = &AArch64::GPR32RegClass;
7001 Opc = AArch64::MADDXrrr;
7002 RC = &AArch64::GPR64RegClass;
7014 unsigned BitSize, OrrOpc, ZeroReg;
7016 OrrOpc = AArch64::ORRWri;
7017 OrrRC = &AArch64::GPR32spRegClass;
7019 ZeroReg = AArch64::WZR;
7020 Opc = AArch64::MADDWrrr;
7021 RC = &AArch64::GPR32RegClass;
7023 OrrOpc = AArch64::ORRXri;
7024 OrrRC = &AArch64::GPR64spRegClass;
7026 ZeroReg = AArch64::XZR;
7027 Opc = AArch64::MADDXrrr;
7028 RC = &AArch64::GPR64RegClass;
7030 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7041 if (
Insn.size() != 1)
7043 auto MovI =
Insn.begin();
7046 if (MovI->Opcode == OrrOpc)
7052 assert((MovI->Opcode == AArch64::MOVNWi ||
7053 MovI->Opcode == AArch64::MOVZWi) &&
7056 assert((MovI->Opcode == AArch64::MOVNXi ||
7057 MovI->Opcode == AArch64::MOVZXi) &&
7064 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7065 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7076 unsigned SubOpc, ZeroReg;
7078 SubOpc = AArch64::SUBWrr;
7079 SubRC = &AArch64::GPR32spRegClass;
7080 ZeroReg = AArch64::WZR;
7081 Opc = AArch64::MADDWrrr;
7082 RC = &AArch64::GPR32RegClass;
7084 SubOpc = AArch64::SUBXrr;
7085 SubRC = &AArch64::GPR64spRegClass;
7086 ZeroReg = AArch64::XZR;
7087 Opc = AArch64::MADDXrrr;
7088 RC = &AArch64::GPR64RegClass;
7090 Register NewVR =
MRI.createVirtualRegister(SubRC);
7097 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7098 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7108 Opc = AArch64::MSUBWrrr;
7109 RC = &AArch64::GPR32RegClass;
7111 Opc = AArch64::MSUBXrrr;
7112 RC = &AArch64::GPR64RegClass;
7124 unsigned BitSize, OrrOpc, ZeroReg;
7126 OrrOpc = AArch64::ORRWri;
7127 OrrRC = &AArch64::GPR32spRegClass;
7129 ZeroReg = AArch64::WZR;
7130 Opc = AArch64::MADDWrrr;
7131 RC = &AArch64::GPR32RegClass;
7133 OrrOpc = AArch64::ORRXri;
7134 OrrRC = &AArch64::GPR64spRegClass;
7136 ZeroReg = AArch64::XZR;
7137 Opc = AArch64::MADDXrrr;
7138 RC = &AArch64::GPR64RegClass;
7140 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7150 if (
Insn.size() != 1)
7152 auto MovI =
Insn.begin();
7155 if (MovI->Opcode == OrrOpc)
7161 assert((MovI->Opcode == AArch64::MOVNWi ||
7162 MovI->Opcode == AArch64::MOVZWi) &&
7165 assert((MovI->Opcode == AArch64::MOVNXi ||
7166 MovI->Opcode == AArch64::MOVZXi) &&
7173 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7174 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7179 Opc = AArch64::MLAv8i8;
7180 RC = &AArch64::FPR64RegClass;
7184 Opc = AArch64::MLAv8i8;
7185 RC = &AArch64::FPR64RegClass;
7189 Opc = AArch64::MLAv16i8;
7190 RC = &AArch64::FPR128RegClass;
7194 Opc = AArch64::MLAv16i8;
7195 RC = &AArch64::FPR128RegClass;
7199 Opc = AArch64::MLAv4i16;
7200 RC = &AArch64::FPR64RegClass;
7204 Opc = AArch64::MLAv4i16;
7205 RC = &AArch64::FPR64RegClass;
7209 Opc = AArch64::MLAv8i16;
7210 RC = &AArch64::FPR128RegClass;
7214 Opc = AArch64::MLAv8i16;
7215 RC = &AArch64::FPR128RegClass;
7219 Opc = AArch64::MLAv2i32;
7220 RC = &AArch64::FPR64RegClass;
7224 Opc = AArch64::MLAv2i32;
7225 RC = &AArch64::FPR64RegClass;
7229 Opc = AArch64::MLAv4i32;
7230 RC = &AArch64::FPR128RegClass;
7234 Opc = AArch64::MLAv4i32;
7235 RC = &AArch64::FPR128RegClass;
7240 Opc = AArch64::MLAv8i8;
7241 RC = &AArch64::FPR64RegClass;
7243 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
7247 Opc = AArch64::MLSv8i8;
7248 RC = &AArch64::FPR64RegClass;
7252 Opc = AArch64::MLAv16i8;
7253 RC = &AArch64::FPR128RegClass;
7255 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
7259 Opc = AArch64::MLSv16i8;
7260 RC = &AArch64::FPR128RegClass;
7264 Opc = AArch64::MLAv4i16;
7265 RC = &AArch64::FPR64RegClass;
7267 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7271 Opc = AArch64::MLSv4i16;
7272 RC = &AArch64::FPR64RegClass;
7276 Opc = AArch64::MLAv8i16;
7277 RC = &AArch64::FPR128RegClass;
7279 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7283 Opc = AArch64::MLSv8i16;
7284 RC = &AArch64::FPR128RegClass;
7288 Opc = AArch64::MLAv2i32;
7289 RC = &AArch64::FPR64RegClass;
7291 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7295 Opc = AArch64::MLSv2i32;
7296 RC = &AArch64::FPR64RegClass;
7300 Opc = AArch64::MLAv4i32;
7301 RC = &AArch64::FPR128RegClass;
7303 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7307 Opc = AArch64::MLSv4i32;
7308 RC = &AArch64::FPR128RegClass;
7313 Opc = AArch64::MLAv4i16_indexed;
7314 RC = &AArch64::FPR64RegClass;
7318 Opc = AArch64::MLAv4i16_indexed;
7319 RC = &AArch64::FPR64RegClass;
7323 Opc = AArch64::MLAv8i16_indexed;
7324 RC = &AArch64::FPR128RegClass;
7328 Opc = AArch64::MLAv8i16_indexed;
7329 RC = &AArch64::FPR128RegClass;
7333 Opc = AArch64::MLAv2i32_indexed;
7334 RC = &AArch64::FPR64RegClass;
7338 Opc = AArch64::MLAv2i32_indexed;
7339 RC = &AArch64::FPR64RegClass;
7343 Opc = AArch64::MLAv4i32_indexed;
7344 RC = &AArch64::FPR128RegClass;
7348 Opc = AArch64::MLAv4i32_indexed;
7349 RC = &AArch64::FPR128RegClass;
7354 Opc = AArch64::MLAv4i16_indexed;
7355 RC = &AArch64::FPR64RegClass;
7357 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7361 Opc = AArch64::MLSv4i16_indexed;
7362 RC = &AArch64::FPR64RegClass;
7366 Opc = AArch64::MLAv8i16_indexed;
7367 RC = &AArch64::FPR128RegClass;
7369 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7373 Opc = AArch64::MLSv8i16_indexed;
7374 RC = &AArch64::FPR128RegClass;
7378 Opc = AArch64::MLAv2i32_indexed;
7379 RC = &AArch64::FPR64RegClass;
7381 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7385 Opc = AArch64::MLSv2i32_indexed;
7386 RC = &AArch64::FPR64RegClass;
7390 Opc = AArch64::MLAv4i32_indexed;
7391 RC = &AArch64::FPR128RegClass;
7393 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7397 Opc = AArch64::MLSv4i32_indexed;
7398 RC = &AArch64::FPR128RegClass;
7404 Opc = AArch64::FMADDHrrr;
7405 RC = &AArch64::FPR16RegClass;
7409 Opc = AArch64::FMADDSrrr;
7410 RC = &AArch64::FPR32RegClass;
7414 Opc = AArch64::FMADDDrrr;
7415 RC = &AArch64::FPR64RegClass;
7420 Opc = AArch64::FMADDHrrr;
7421 RC = &AArch64::FPR16RegClass;
7425 Opc = AArch64::FMADDSrrr;
7426 RC = &AArch64::FPR32RegClass;
7430 Opc = AArch64::FMADDDrrr;
7431 RC = &AArch64::FPR64RegClass;
7436 Opc = AArch64::FMLAv1i32_indexed;
7437 RC = &AArch64::FPR32RegClass;
7439 FMAInstKind::Indexed);
7442 Opc = AArch64::FMLAv1i32_indexed;
7443 RC = &AArch64::FPR32RegClass;
7445 FMAInstKind::Indexed);
7449 Opc = AArch64::FMLAv1i64_indexed;
7450 RC = &AArch64::FPR64RegClass;
7452 FMAInstKind::Indexed);
7455 Opc = AArch64::FMLAv1i64_indexed;
7456 RC = &AArch64::FPR64RegClass;
7458 FMAInstKind::Indexed);
7462 RC = &AArch64::FPR64RegClass;
7463 Opc = AArch64::FMLAv4i16_indexed;
7465 FMAInstKind::Indexed);
7468 RC = &AArch64::FPR64RegClass;
7469 Opc = AArch64::FMLAv4f16;
7471 FMAInstKind::Accumulator);
7474 RC = &AArch64::FPR64RegClass;
7475 Opc = AArch64::FMLAv4i16_indexed;
7477 FMAInstKind::Indexed);
7480 RC = &AArch64::FPR64RegClass;
7481 Opc = AArch64::FMLAv4f16;
7483 FMAInstKind::Accumulator);
7488 RC = &AArch64::FPR64RegClass;
7490 Opc = AArch64::FMLAv2i32_indexed;
7492 FMAInstKind::Indexed);
7494 Opc = AArch64::FMLAv2f32;
7496 FMAInstKind::Accumulator);
7501 RC = &AArch64::FPR64RegClass;
7503 Opc = AArch64::FMLAv2i32_indexed;
7505 FMAInstKind::Indexed);
7507 Opc = AArch64::FMLAv2f32;
7509 FMAInstKind::Accumulator);
7514 RC = &AArch64::FPR128RegClass;
7515 Opc = AArch64::FMLAv8i16_indexed;
7517 FMAInstKind::Indexed);
7520 RC = &AArch64::FPR128RegClass;
7521 Opc = AArch64::FMLAv8f16;
7523 FMAInstKind::Accumulator);
7526 RC = &AArch64::FPR128RegClass;
7527 Opc = AArch64::FMLAv8i16_indexed;
7529 FMAInstKind::Indexed);
7532 RC = &AArch64::FPR128RegClass;
7533 Opc = AArch64::FMLAv8f16;
7535 FMAInstKind::Accumulator);
7540 RC = &AArch64::FPR128RegClass;
7542 Opc = AArch64::FMLAv2i64_indexed;
7544 FMAInstKind::Indexed);
7546 Opc = AArch64::FMLAv2f64;
7548 FMAInstKind::Accumulator);
7553 RC = &AArch64::FPR128RegClass;
7555 Opc = AArch64::FMLAv2i64_indexed;
7557 FMAInstKind::Indexed);
7559 Opc = AArch64::FMLAv2f64;
7561 FMAInstKind::Accumulator);
7567 RC = &AArch64::FPR128RegClass;
7569 Opc = AArch64::FMLAv4i32_indexed;
7571 FMAInstKind::Indexed);
7573 Opc = AArch64::FMLAv4f32;
7575 FMAInstKind::Accumulator);
7581 RC = &AArch64::FPR128RegClass;
7583 Opc = AArch64::FMLAv4i32_indexed;
7585 FMAInstKind::Indexed);
7587 Opc = AArch64::FMLAv4f32;
7589 FMAInstKind::Accumulator);
7594 Opc = AArch64::FNMSUBHrrr;
7595 RC = &AArch64::FPR16RegClass;
7599 Opc = AArch64::FNMSUBSrrr;
7600 RC = &AArch64::FPR32RegClass;
7604 Opc = AArch64::FNMSUBDrrr;
7605 RC = &AArch64::FPR64RegClass;
7610 Opc = AArch64::FNMADDHrrr;
7611 RC = &AArch64::FPR16RegClass;
7615 Opc = AArch64::FNMADDSrrr;
7616 RC = &AArch64::FPR32RegClass;
7620 Opc = AArch64::FNMADDDrrr;
7621 RC = &AArch64::FPR64RegClass;
7626 Opc = AArch64::FMSUBHrrr;
7627 RC = &AArch64::FPR16RegClass;
7631 Opc = AArch64::FMSUBSrrr;
7632 RC = &AArch64::FPR32RegClass;
7636 Opc = AArch64::FMSUBDrrr;
7637 RC = &AArch64::FPR64RegClass;
7642 Opc = AArch64::FMLSv1i32_indexed;
7643 RC = &AArch64::FPR32RegClass;
7645 FMAInstKind::Indexed);
7649 Opc = AArch64::FMLSv1i64_indexed;
7650 RC = &AArch64::FPR64RegClass;
7652 FMAInstKind::Indexed);
7657 RC = &AArch64::FPR64RegClass;
7663 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7665 Opc = AArch64::FMLAv4f16;
7667 FMAInstKind::Accumulator, &NewVR);
7669 Opc = AArch64::FMLAv4i16_indexed;
7671 FMAInstKind::Indexed, &NewVR);
7676 RC = &AArch64::FPR64RegClass;
7677 Opc = AArch64::FMLSv4f16;
7679 FMAInstKind::Accumulator);
7682 RC = &AArch64::FPR64RegClass;
7683 Opc = AArch64::FMLSv4i16_indexed;
7685 FMAInstKind::Indexed);
7690 RC = &AArch64::FPR64RegClass;
7692 Opc = AArch64::FMLSv2i32_indexed;
7694 FMAInstKind::Indexed);
7696 Opc = AArch64::FMLSv2f32;
7698 FMAInstKind::Accumulator);
7704 RC = &AArch64::FPR128RegClass;
7710 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7712 Opc = AArch64::FMLAv8f16;
7714 FMAInstKind::Accumulator, &NewVR);
7716 Opc = AArch64::FMLAv8i16_indexed;
7718 FMAInstKind::Indexed, &NewVR);
7723 RC = &AArch64::FPR128RegClass;
7724 Opc = AArch64::FMLSv8f16;
7726 FMAInstKind::Accumulator);
7729 RC = &AArch64::FPR128RegClass;
7730 Opc = AArch64::FMLSv8i16_indexed;
7732 FMAInstKind::Indexed);
7737 RC = &AArch64::FPR128RegClass;
7739 Opc = AArch64::FMLSv2i64_indexed;
7741 FMAInstKind::Indexed);
7743 Opc = AArch64::FMLSv2f64;
7745 FMAInstKind::Accumulator);
7751 RC = &AArch64::FPR128RegClass;
7753 Opc = AArch64::FMLSv4i32_indexed;
7755 FMAInstKind::Indexed);
7757 Opc = AArch64::FMLSv4f32;
7759 FMAInstKind::Accumulator);
7764 RC = &AArch64::FPR64RegClass;
7770 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7772 Opc = AArch64::FMLAv2i32_indexed;
7774 FMAInstKind::Indexed, &NewVR);
7776 Opc = AArch64::FMLAv2f32;
7778 FMAInstKind::Accumulator, &NewVR);
7784 RC = &AArch64::FPR128RegClass;
7790 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7792 Opc = AArch64::FMLAv4i32_indexed;
7794 FMAInstKind::Indexed, &NewVR);
7796 Opc = AArch64::FMLAv4f32;
7798 FMAInstKind::Accumulator, &NewVR);
7804 RC = &AArch64::FPR128RegClass;
7810 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7812 Opc = AArch64::FMLAv2i64_indexed;
7814 FMAInstKind::Indexed, &NewVR);
7816 Opc = AArch64::FMLAv2f64;
7818 FMAInstKind::Accumulator, &NewVR);
7828 &AArch64::FPR128RegClass,
MRI);
7837 &AArch64::FPR128RegClass,
MRI);
7846 &AArch64::FPR128_loRegClass,
MRI);
7855 &AArch64::FPR128RegClass,
MRI);
7864 &AArch64::FPR128_loRegClass,
MRI);
7883 for (
auto *
MI : InsInstrs)
7884 MI->setFlags(Flags);
7925 bool IsNegativeBranch =
false;
7926 bool IsTestAndBranch =
false;
7927 unsigned TargetBBInMI = 0;
7928 switch (
MI.getOpcode()) {
7937 case AArch64::CBNZW:
7938 case AArch64::CBNZX:
7940 IsNegativeBranch =
true;
7945 IsTestAndBranch =
true;
7947 case AArch64::TBNZW:
7948 case AArch64::TBNZX:
7950 IsNegativeBranch =
true;
7951 IsTestAndBranch =
true;
7957 if (IsTestAndBranch &&
MI.getOperand(1).getImm())
7961 assert(
MI.getParent() &&
"Incomplete machine instruciton\n");
7974 if (!
MRI->hasOneNonDBGUse(CopyVReg))
7976 if (!
MRI->hasOneDef(CopyVReg))
7985 case AArch64::ANDWri:
7986 case AArch64::ANDXri: {
7987 if (IsTestAndBranch)
7991 if (!
MRI->hasOneNonDBGUse(VReg))
8005 assert(!
MRI->def_empty(NewReg) &&
"Register must be defined.");
8011 unsigned Opc = (Imm < 32)
8012 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
8013 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
8026 if (!Is32Bit && Imm < 32)
8028 MI.eraseFromParent();
8032 case AArch64::CSINCWr:
8033 case AArch64::CSINCXr: {
8053 if (IsNegativeBranch)
8056 MI.eraseFromParent();
8062std::pair<unsigned, unsigned>
8065 return std::make_pair(TF & Mask, TF & ~Mask);
8070 using namespace AArch64II;
8072 static const std::pair<unsigned, const char *> TargetFlags[] = {
8073 {MO_PAGE,
"aarch64-page"}, {MO_PAGEOFF,
"aarch64-pageoff"},
8074 {MO_G3,
"aarch64-g3"}, {MO_G2,
"aarch64-g2"},
8075 {MO_G1,
"aarch64-g1"}, {MO_G0,
"aarch64-g0"},
8076 {MO_HI12,
"aarch64-hi12"}};
8082 using namespace AArch64II;
8084 static const std::pair<unsigned, const char *> TargetFlags[] = {
8085 {MO_COFFSTUB,
"aarch64-coffstub"},
8086 {MO_GOT,
"aarch64-got"},
8087 {MO_NC,
"aarch64-nc"},
8088 {MO_S,
"aarch64-s"},
8089 {MO_TLS,
"aarch64-tls"},
8090 {MO_DLLIMPORT,
"aarch64-dllimport"},
8091 {MO_PREL,
"aarch64-prel"},
8092 {MO_TAGGED,
"aarch64-tagged"},
8093 {MO_ARM64EC_CALLMANGLE,
"aarch64-arm64ec-callmangle"},
8100 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
8208 for (
unsigned Reg : AArch64::GPR64RegClass) {
8210 Reg != AArch64::LR &&
8211 Reg != AArch64::X16 &&
8212 Reg != AArch64::X17 &&
8213 C.isAvailableAcrossAndOutOfSeq(
Reg,
TRI) &&
8214 C.isAvailableInsideSeq(
Reg,
TRI))
8245 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
8248std::optional<outliner::OutlinedFunction>
8250 std::vector<outliner::Candidate> &RepeatedSequenceLocs)
const {
8253 unsigned SequenceSize = 0;
8254 for (
auto &
MI : FirstCand)
8257 unsigned NumBytesToCreateFrame = 0;
8267 if (std::adjacent_find(
8268 RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
8272 if (outliningCandidatesSigningScopeConsensus(a, b) &&
8273 outliningCandidatesSigningKeyConsensus(a, b) &&
8274 outliningCandidatesV8_3OpsConsensus(a, b)) {
8278 }) != RepeatedSequenceLocs.end()) {
8279 return std::nullopt;
8296 unsigned NumBytesToCheckLRInTCEpilogue = 0;
8297 if (FirstCand.getMF()
8301 NumBytesToCreateFrame += 8;
8304 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod();
8305 NumBytesToCheckLRInTCEpilogue =
8309 if (isTailCallReturnInst(RepeatedSequenceLocs[0].back()))
8310 SequenceSize += NumBytesToCheckLRInTCEpilogue;
8318 for (
auto &
MI :
C) {
8319 if (
MI.modifiesRegister(AArch64::SP, &
TRI)) {
8320 switch (
MI.getOpcode()) {
8321 case AArch64::ADDXri:
8322 case AArch64::ADDWri:
8323 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8325 "Expected operand to be immediate");
8327 "Expected operand to be a register");
8331 if (
MI.getOperand(1).getReg() == AArch64::SP)
8332 SPValue +=
MI.getOperand(2).getImm();
8336 case AArch64::SUBXri:
8337 case AArch64::SUBWri:
8338 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8340 "Expected operand to be immediate");
8342 "Expected operand to be a register");
8346 if (
MI.getOperand(1).getReg() == AArch64::SP)
8347 SPValue -=
MI.getOperand(2).getImm();
8364 if (RepeatedSequenceLocs.size() < 2)
8365 return std::nullopt;
8369 unsigned FlagsSetInAll = 0xF;
8373 FlagsSetInAll &=
C.Flags;
8375 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
8378 auto SetCandidateCallInfo =
8379 [&RepeatedSequenceLocs](
unsigned CallID,
unsigned NumBytesForCall) {
8381 C.setCallInfo(CallID, NumBytesForCall);
8385 NumBytesToCreateFrame += 4;
8388 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
8393 unsigned CFICount = 0;
8394 for (
auto &
I : RepeatedSequenceLocs[0]) {
8395 if (
I.isCFIInstruction())
8405 std::vector<MCCFIInstruction> CFIInstructions =
8406 C.getMF()->getFrameInstructions();
8408 if (CFICount > 0 && CFICount != CFIInstructions.size())
8409 return std::nullopt;
8417 if (!
MI.modifiesRegister(AArch64::SP, &
TRI) &&
8418 !
MI.readsRegister(AArch64::SP, &
TRI))
8424 if (
MI.modifiesRegister(AArch64::SP, &
TRI))
8429 if (
MI.mayLoadOrStore()) {
8432 bool OffsetIsScalable;
8436 if (!getMemOperandWithOffset(
MI,
Base,
Offset, OffsetIsScalable, &
TRI) ||
8437 !
Base->isReg() ||
Base->getReg() != AArch64::SP)
8441 if (OffsetIsScalable)
8449 TypeSize Scale(0U,
false), DummyWidth(0U,
false);
8450 getMemOpInfo(
MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
8453 if (
Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
8454 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
8469 bool AllStackInstrsSafe =
llvm::all_of(FirstCand, IsSafeToFixup);
8473 if (RepeatedSequenceLocs[0].back().isTerminator()) {
8475 NumBytesToCreateFrame = 0;
8476 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
8480 else if (LastInstrOpcode == AArch64::BL ||
8481 ((LastInstrOpcode == AArch64::BLR ||
8482 LastInstrOpcode == AArch64::BLRNoIP) &&
8486 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
8494 unsigned NumBytesNoStackCalls = 0;
8495 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
8500 (
C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
8501 ?
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI)
8510 C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
8513 if (LRAvailable && !IsNoReturn) {
8514 NumBytesNoStackCalls += 4;
8516 CandidatesWithoutStackFixups.push_back(
C);
8521 else if (findRegisterToSaveLRTo(
C)) {
8522 NumBytesNoStackCalls += 12;
8524 CandidatesWithoutStackFixups.push_back(
C);
8529 else if (
C.isAvailableInsideSeq(AArch64::SP,
TRI)) {
8530 NumBytesNoStackCalls += 12;
8532 CandidatesWithoutStackFixups.push_back(
C);
8538 NumBytesNoStackCalls += SequenceSize;
8545 if (!AllStackInstrsSafe ||
8546 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
8547 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
8597 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
8601 (!
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI) ||
8602 !findRegisterToSaveLRTo(
C));
8608 if (RepeatedSequenceLocs.size() < 2) {
8609 RepeatedSequenceLocs.clear();
8610 return std::nullopt;
8616 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
8619 bool ModStackToSaveLR =
false;
8620 if (std::any_of(FirstCand.begin(), std::prev(FirstCand.end()),
8622 ModStackToSaveLR =
true;
8631 ModStackToSaveLR =
true;
8633 if (ModStackToSaveLR) {
8635 if (!AllStackInstrsSafe) {
8636 RepeatedSequenceLocs.clear();
8637 return std::nullopt;
8641 NumBytesToCreateFrame += 8;
8648 return std::nullopt;
8651 NumBytesToCreateFrame, FrameID);
8655 Function &
F, std::vector<outliner::Candidate> &Candidates)
const {
8659 const auto &CFn = Candidates.front().getMF()->getFunction();
8663 if (CFn.hasFnAttribute(
"sign-return-address"))
8664 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address"));
8665 if (CFn.hasFnAttribute(
"sign-return-address-key"))
8666 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address-key"));
8668 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(
F, Candidates);
8676 if (!OutlineFromLinkOnceODRs &&
F.hasLinkOnceODRLinkage())
8690 if (!AFI || AFI->
hasRedZone().value_or(
true))
8703 unsigned &Flags)
const {
8705 "Must track liveness!");
8707 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
8722 auto AreAllUnsafeRegsDead = [&LRU]() {
8739 bool LRAvailableEverywhere =
true;
8744 if (
MI.isCall() && !
MI.isTerminator())
8745 Flags |= MachineOutlinerMBBFlags::HasCalls;
8750 auto CreateNewRangeStartingAt =
8751 [&RangeBegin, &RangeEnd,
8753 RangeBegin = NewBegin;
8754 RangeEnd = std::next(RangeBegin);
8757 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
8762 Ranges.push_back(std::make_pair(RangeBegin, RangeEnd));
8770 for (; FirstPossibleEndPt !=
MBB.
instr_rend(); ++FirstPossibleEndPt) {
8774 UpdateWholeMBBFlags(*FirstPossibleEndPt);
8775 if (AreAllUnsafeRegsDead())
8782 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
8788 UpdateWholeMBBFlags(
MI);
8789 if (!AreAllUnsafeRegsDead()) {
8790 SaveRangeIfNonEmpty();
8791 CreateNewRangeStartingAt(
MI.getIterator());
8794 LRAvailableEverywhere &= LRU.
available(AArch64::LR);
8795 RangeBegin =
MI.getIterator();
8800 if (AreAllUnsafeRegsDead())
8801 SaveRangeIfNonEmpty();
8806 std::reverse(Ranges.begin(), Ranges.end());
8809 if (!LRAvailableEverywhere)
8810 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
8816 unsigned Flags)
const {
8824 switch (
MI.getOpcode()) {
8826 case AArch64::PACIASP:
8827 case AArch64::PACIBSP:
8828 case AArch64::PACIASPPC:
8829 case AArch64::PACIBSPPC:
8830 case AArch64::AUTIASP:
8831 case AArch64::AUTIBSP:
8832 case AArch64::AUTIASPPCi:
8833 case AArch64::AUTIASPPCr:
8834 case AArch64::AUTIBSPPCi:
8835 case AArch64::AUTIBSPPCr:
8836 case AArch64::RETAA:
8837 case AArch64::RETAB:
8838 case AArch64::RETAASPPCi:
8839 case AArch64::RETAASPPCr:
8840 case AArch64::RETABSPPCi:
8841 case AArch64::RETABSPPCr:
8842 case AArch64::EMITBKEY:
8843 case AArch64::PAUTH_PROLOGUE:
8844 case AArch64::PAUTH_EPILOGUE:
8858 if (
MI.isCFIInstruction())
8862 if (
MI.isTerminator())
8871 assert(!MOP.isCFIIndex());
8874 if (MOP.isReg() && !MOP.isImplicit() &&
8875 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
8882 if (
MI.getOpcode() == AArch64::ADRP)
8903 if (MOP.isGlobal()) {
8904 Callee = dyn_cast<Function>(MOP.getGlobal());
8911 if (Callee && Callee->getName() ==
"\01_mcount")
8919 if (
MI.getOpcode() == AArch64::BLR ||
8920 MI.getOpcode() == AArch64::BLRNoIP ||
MI.getOpcode() == AArch64::BL)
8924 return UnknownCallOutlineType;
8932 return UnknownCallOutlineType;
8940 return UnknownCallOutlineType;
8965 bool OffsetIsScalable;
8968 if (!
MI.mayLoadOrStore() ||
8971 (
Base->isReg() &&
Base->getReg() != AArch64::SP))
8976 int64_t Dummy1, Dummy2;
8979 assert(StackOffsetOperand.
isImm() &&
"Stack offset wasn't immediate!");
8981 assert(Scale != 0 &&
"Unexpected opcode!");
8982 assert(!OffsetIsScalable &&
"Expected offset to be a byte offset");
8987 int64_t NewImm = (
Offset + 16) / (int64_t)Scale.getFixedValue();
8988 StackOffsetOperand.
setImm(NewImm);
8994 bool ShouldSignReturnAddr) {
8995 if (!ShouldSignReturnAddr)
9001 TII->get(AArch64::PAUTH_EPILOGUE))
9017 unsigned TailOpcode;
9018 if (Call->getOpcode() == AArch64::BL) {
9019 TailOpcode = AArch64::TCRETURNdi;
9021 assert(Call->getOpcode() == AArch64::BLR ||
9022 Call->getOpcode() == AArch64::BLRNoIP);
9023 TailOpcode = AArch64::TCRETURNriALL;
9026 .
add(Call->getOperand(0))
9029 Call->eraseFromParent();
9034 bool IsLeafFunction =
true;
9038 return MI.isCall() && !
MI.isReturn();
9048 "Can only fix up stack references once");
9049 fixupPostOutline(
MBB);
9051 IsLeafFunction =
false;
9062 Et = std::prev(
MBB.
end());
9075 unsigned DwarfReg =
MRI->getDwarfRegNum(AArch64::LR,
true);
9078 int64_t StackPosEntry =
9131 fixupPostOutline(
MBB);
9142 .addGlobalAddress(M.getNamedValue(MF.
getName()))
9152 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9167 Register Reg = findRegisterToSaveLRTo(
C);
9168 assert(Reg &&
"No callee-saved register available?");
9202 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9218 bool AllowSideEffects)
const {
9223 if (
TRI.isGeneralPurposeRegister(MF, Reg)) {
9225 }
else if (STI.hasSVE()) {
9235std::optional<DestSourcePair>
9240 if (
MI.getOpcode() == AArch64::ORRWrs &&
9241 MI.getOperand(1).getReg() == AArch64::WZR &&
9242 MI.getOperand(3).getImm() == 0x0 &&
9244 (!
MI.getOperand(0).getReg().isVirtual() ||
9245 MI.getOperand(0).getSubReg() == 0) &&
9246 (!
MI.getOperand(0).getReg().isPhysical() ||
9247 MI.findRegisterDefOperandIdx(
MI.getOperand(0).getReg() - AArch64::W0 +
9252 if (
MI.getOpcode() == AArch64::ORRXrs &&
9253 MI.getOperand(1).getReg() == AArch64::XZR &&
9254 MI.getOperand(3).getImm() == 0x0)
9257 return std::nullopt;
9260std::optional<DestSourcePair>
9262 if (
MI.getOpcode() == AArch64::ORRWrs &&
9263 MI.getOperand(1).getReg() == AArch64::WZR &&
9264 MI.getOperand(3).getImm() == 0x0)
9266 return std::nullopt;
9269std::optional<RegImmPair>
9278 return std::nullopt;
9280 switch (
MI.getOpcode()) {
9282 return std::nullopt;
9283 case AArch64::SUBWri:
9284 case AArch64::SUBXri:
9285 case AArch64::SUBSWri:
9286 case AArch64::SUBSXri:
9289 case AArch64::ADDSWri:
9290 case AArch64::ADDSXri:
9291 case AArch64::ADDWri:
9292 case AArch64::ADDXri: {
9294 if (!
MI.getOperand(0).isReg() || !
MI.getOperand(1).isReg() ||
9295 !
MI.getOperand(2).isImm())
9296 return std::nullopt;
9297 int Shift =
MI.getOperand(3).getImm();
9298 assert((Shift == 0 || Shift == 12) &&
"Shift can be either 0 or 12");
9299 Offset = Sign * (
MI.getOperand(2).getImm() << Shift);
9308static std::optional<ParamLoadedValue>
9312 auto DestSrc =
TII->isCopyLikeInstr(
MI);
9314 return std::nullopt;
9316 Register DestReg = DestSrc->Destination->getReg();
9317 Register SrcReg = DestSrc->Source->getReg();
9322 if (DestReg == DescribedReg)
9326 if (
MI.getOpcode() == AArch64::ORRWrs &&
9327 TRI->isSuperRegister(DestReg, DescribedReg))
9331 if (
MI.getOpcode() == AArch64::ORRXrs &&
9332 TRI->isSubRegister(DestReg, DescribedReg)) {
9333 Register SrcSubReg =
TRI->getSubReg(SrcReg, AArch64::sub_32);
9337 assert(!
TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
9338 "Unhandled ORR[XW]rs copy case");
9340 return std::nullopt;
9359 return MI.getOpcode() == AArch64::INLINEASM_BR;
9377 switch (
MI.getOpcode()) {
9378 case TargetOpcode::G_BRJT:
9379 case AArch64::JumpTableDest32:
9380 case AArch64::JumpTableDest16:
9381 case AArch64::JumpTableDest8:
9392std::optional<ParamLoadedValue>
9397 switch (
MI.getOpcode()) {
9398 case AArch64::MOVZWi:
9399 case AArch64::MOVZXi: {
9402 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(), Reg))
9403 return std::nullopt;
9405 if (!
MI.getOperand(1).isImm())
9406 return std::nullopt;
9407 int64_t Immediate =
MI.getOperand(1).getImm();
9408 int Shift =
MI.getOperand(2).getImm();
9412 case AArch64::ORRWrs:
9413 case AArch64::ORRXrs:
9423 ExtMI.
getOpcode() == TargetOpcode::G_ZEXT ||
9424 ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT);
9427 if (ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT)
9431 if (!
MRI.hasOneNonDBGUse(DefReg))
9436 auto *UserMI = &*
MRI.use_instr_nodbg_begin(DefReg);
9437 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
9458 unsigned Scale)
const {
9469 unsigned Shift =
Log2_64(NumBytes);
9470 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
9478 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
9483 return AArch64::BLRNoIP;
9485 return AArch64::BLR;
9490 Register TargetReg,
bool FrameSetup)
const {
9491 assert(TargetReg != AArch64::SP &&
"New top of stack cannot aleady be in SP");
9503 MF.
insert(MBBInsertPoint, LoopTestMBB);
9506 MF.
insert(MBBInsertPoint, LoopBodyMBB);
9508 MF.
insert(MBBInsertPoint, ExitMBB);
9518 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::SUBSXrx64),
9532 BuildMI(*LoopBodyMBB, LoopBodyMBB->
end(),
DL,
TII->get(AArch64::STRXui))
9545 BuildMI(*ExitMBB, ExitMBB->
end(),
DL,
TII->get(AArch64::ADDXri), AArch64::SP)
9570 return ExitMBB->
begin();
9581 : PredBranch(PredBranch),
Cond(
Cond.begin(),
Cond.end()) {}
9583 bool shouldIgnoreForPipelining(
const MachineInstr *
MI)
const override {
9586 return MI == PredBranch;
9589 std::optional<bool> createTripCountGreaterCondition(
9601 void adjustTripCount(
int TripCountAdjust)
override {}
9603 void disposed()
override {}
9611 case AArch64::CBNZW:
9612 case AArch64::CBNZX:
9615 case AArch64::TBNZW:
9616 case AArch64::TBNZX:
9622std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
9630 if (
TBB == LoopBB && FBB == LoopBB)
9637 assert((
TBB == LoopBB || FBB == LoopBB) &&
9638 "The Loop must be a single-basic-block loop");
9649 if (CondBranch->
getOpcode() == AArch64::Bcc) {
9651 if (
MI.modifiesRegister(AArch64::NZCV, &
TRI)) {
9661 if (!Reg.isVirtual())
9663 PredBranch =
MRI.getVRegDef(Reg);
9666 if (PredBranch->
isPHI())
9675 return std::make_unique<AArch64PipelinerLoopInfo>(PredBranch,
Cond);
9678#define GET_INSTRINFO_HELPERS
9679#define GET_INSTRMAP_INFO
9680#include "AArch64GenInstrInfo.inc"
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static cl::opt< unsigned > BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg)
Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
static bool getMaddPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find instructions that can be turned into madd.
static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr)
Find a condition code used by the instruction.
static MachineInstr * genFusedMultiplyAcc(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
static bool isCombineInstrCandidate64(unsigned Opc)
static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg)
static bool areCFlagsAccessedBetweenInstrs(MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
True when condition flags are accessed (either by writing or reading) on the instruction trace starti...
static bool getFMAPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Floating-Point Support.
static bool isADDSRegImm(unsigned Opcode)
static MachineInstr * genFusedMultiplyIdxNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static bool isCompareAndBranch(unsigned Opcode)
static unsigned sForm(MachineInstr &Instr)
Get opcode of S version of Instr.
static bool isCombineInstrSettingFlag(unsigned Opc)
static bool getFNEGPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc)
static int findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr)
static unsigned getBranchDisplacementBits(unsigned Opc)
static std::optional< ParamLoadedValue > describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then,...
static bool getFMULPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static MachineInstr * genFusedMultiplyAccNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static void appendVGScaledOffsetExpr(SmallVectorImpl< char > &Expr, int NumBytes, int NumVGScaledBytes, unsigned VG, llvm::raw_string_ostream &Comment)
static MachineInstr * genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example ...
static bool scaleOffset(unsigned Opc, int64_t &Offset)
static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale)
static MachineInstr * genFusedMultiplyIdx(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genIndexedMultiply(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static bool isSUBSRegImm(unsigned Opcode)
static bool UpdateOperandRegClass(MachineInstr &Instr)
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
unsigned unscaledOffsetOpcode(unsigned Opcode)
static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
Check if CmpInstr can be substituted by MI.
static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC)
static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
genNeg - Helper to generate an intermediate negation of the second operand of Root
static bool isCombineInstrCandidateFP(const MachineInstr &Inst)
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI)
Return the opcode that does not set flags when possible - otherwise return the original opcode.
static const MachineInstrBuilder & AddSubReg(const MachineInstrBuilder &MIB, unsigned Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool isCombineInstrCandidate32(unsigned Opc)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
static unsigned offsetExtendOpcode(unsigned Opcode)
static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static bool getMiscPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
Find other MI combine patterns.
static bool outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
static cl::opt< unsigned > TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
static MachineInstr * genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC, FMAInstKind kind=FMAInstKind::Default, const Register *ReplacedAddend=nullptr)
genFusedMultiply - Generate fused multiply instructions.
static bool isCombineInstrCandidate(unsigned Opc)
static unsigned regOffsetOpcode(unsigned Opcode)
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerRegSave
Emit a call and tail-call.
@ MachineOutlinerNoLRSave
Only emit a branch.
@ MachineOutlinerThunk
Emit a call and return.
static cl::opt< unsigned > BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB)
Check if AArch64::NZCV should be alive in successors of MBB.
static void emitFrameOffsetAdj(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize)
static cl::opt< unsigned > CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewVReg=nullptr)
static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
static MachineInstr * genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs)
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool shouldSignReturnAddress(const MachineFunction &MF) const
const SetOfInstructions & getLOHRelated() const
bool needsDwarfUnwindInfo(const MachineFunction &MF) const
void setOutliningStyle(std::string Style)
std::optional< bool > hasRedZone() const
bool shouldSignWithBKey() const
static bool isHForm(const MachineInstr &MI)
Returns whether the instruction is in H form (16 bit operands)
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
static bool hasBTISemantics(const MachineInstr &MI)
Returns whether the instruction can be compatible with non-zero BTYPE.
static bool isQForm(const MachineInstr &MI)
Returns whether the instruction is in Q form (128 bit operands)
static void decomposeStackOffsetForFrameOffsets(const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors, int64_t &NumDataVectors)
Returns the offset in parts to which this frame offset can be decomposed for the purpose of describin...
static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width, int64_t &MinOffset, int64_t &MaxOffset)
Returns true if opcode Opc is a memory operation.
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isFPRCopy(const MachineInstr &MI)
Does this instruction rename an FPR without modifying bits?
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is an instruction that moves/copies value from one register to an...
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
GetInstSize - Return the number of bytes of code the specified instruction may be.
uint64_t getElementSizeForOpcode(unsigned Opc) const
Returns the vector element size (B, H, S or D) of an SVE opcode.
outliner::InstrType getOutliningTypeImpl(MachineBasicBlock::iterator &MIT, unsigned Flags) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
static bool isGPRCopy(const MachineInstr &MI)
Does this instruction rename a GPR without modifying bits?
static unsigned convertToFlagSettingOpc(unsigned Opc)
Return the opcode that set flags when possible.
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, int &, int &, int &) const override
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
bool isWhileOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE WHILE## instruction.
static std::optional< unsigned > getUnscaledLdSt(unsigned Opc)
Returns the unscaled load/store for the scaled load/store opcode, if there is a corresponding unscale...
static bool hasUnscaledLdStOffset(unsigned Opc)
Return true if it has an unscaled load/store offset.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify) const override
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
SmallVector< std::pair< MachineBasicBlock::iterator, MachineBasicBlock::iterator > > getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override
static bool isPairableLdStInst(const MachineInstr &MI)
Return true if pairing the given load or store may be paired with another.
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const override
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
const AArch64RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
static bool isPreSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed store.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
AArch64InstrInfo(const AArch64Subtarget &STI)
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
bool useMachineCombiner() const override
AArch64 supports MachineCombiner.
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool isExtendLikelyToBeFolded(MachineInstr &ExtMI, MachineRegisterInfo &MRI) const override
static bool isFalkorShiftExtFast(const MachineInstr &MI)
Returns true if the instruction has a shift by immediate that can be executed in one cycle less.
std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, TypeSize &Width, const TargetRegisterInfo *TRI) const
If OffsetIsScalable is set to 'true', the offset is scaled by vscale.
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isStridedAccess(const MachineInstr &MI)
Return true if the given load or store is a strided memory access.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
Detect opportunities for ldp/stp formation.
bool expandPostRAPseudo(MachineInstr &MI) const override
unsigned int getTailDuplicateSize(CodeGenOptLevel OptLevel) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
static bool isFpOrNEON(const MachineInstr &MI)
Returns whether the instruction is FP or NEON.
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
bool isThroughputPattern(unsigned Pattern) const override
Return true when a code sequence can improve throughput.
void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL, unsigned DestReg, unsigned SrcReg, bool KillSrc, unsigned Opcode, unsigned ZeroReg, llvm::ArrayRef< unsigned > Indices) const
void buildClearRegister(Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator Iter, DebugLoc &DL, bool AllowSideEffects=true) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
MachineOperand & getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const
Return the immediate offset of the base register in a load/store LdSt.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
static bool isLdStPairSuppressed(const MachineInstr &MI)
Return true if pairing the given load or store is hinted to be unprofitable.
bool isFunctionSafeToSplit(const MachineFunction &MF) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
Return true when Inst is associative and commutative so that it can be reassociated.
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI, Register TargetReg, bool FrameSetup) const
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
optimizeCompareInstr - Convert the instruction supplying the argument to the comparison into one that...
std::optional< outliner::OutlinedFunction > getOutliningCandidateInfo(std::vector< outliner::Candidate > &RepeatedSequenceLocs) const override
static unsigned getLoadStoreImmIdx(unsigned Opc)
Returns the index for the immediate for a given instruction.
static bool isGPRZero(const MachineInstr &MI)
Does this instruction set its full destination register to zero?
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
analyzeCompare - For a comparison instruction, return the source registers in SrcReg and SrcReg2,...
CombinerObjective getCombinerObjective(unsigned Pattern) const override
bool isMBBSafeToSplitToCold(const MachineBasicBlock &MBB) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool isLegalAddressingMode(unsigned NumBytes, int64_t Offset, unsigned Scale) const
std::optional< DestSourcePair > isCopyLikeInstrImpl(const MachineInstr &MI) const override
static void suppressLdStPair(MachineInstr &MI)
Hint that pairing the given load or store is unprofitable.
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isPreLd(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load.
ArrayRef< std::pair< unsigned, const char * > > getSerializableBitmaskMachineOperandTargetFlags() const override
void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, llvm::ArrayRef< unsigned > Indices) const
bool optimizeCondBranch(MachineInstr &MI) const override
Replace csincr-branch sequence by simple conditional branch.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
bool isCandidateToMergeOrPair(const MachineInstr &MI) const
Return true if this is a load/store that can be potentially paired/merged.
MCInst getNop() const override
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
bool isPTestLikeOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE instruction that sets the condition codes as if it's results...
void mergeOutliningCandidateAttributes(Function &F, std::vector< outliner::Candidate > &Candidates) const override
static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized)
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
bool isNeonAvailable() const
Returns true if the target has NEON and the function at runtime is known to have NEON enabled (e....
const AArch64RegisterInfo * getRegisterInfo() const override
bool isTargetILP32() const
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Module * getParent()
Get the module that this global value is contained inside of...
A set of register units used to track register liveness.
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
void stepBackward(const MachineInstr &MI)
Updates liveness when stepping backwards over the instruction MI.
void addLiveOuts(const MachineBasicBlock &MBB)
Adds registers living out of block MBB.
static LocationSize precise(uint64_t Value)
This class is intended to be used as a base class for asm properties and features specific to the tar...
bool usesWindowsCFI() const
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
static constexpr unsigned NoRegister
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
reverse_instr_iterator instr_rbegin()
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MBBSectionID getSectionID() const
Returns the section ID of this basic block.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
Instructions::iterator instr_iterator
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
instr_iterator instr_end()
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
unsigned getNumObjects() const
Return the number of objects.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
uint32_t mergeFlagsWith(const MachineInstr &Other) const
Return the MIFlags which represent both MachineInstrs.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
const std::vector< MachineJumpTableEntry > & getJumpTables() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineFunction * getMachineFunction(const Function &F) const
Returns the MachineFunction associated to IR function F if there is one, otherwise nullptr.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
bool isRegUsed(Register Reg, bool includeReserved=true) const
Return if a specific register is currently used.
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Represents a location in source code.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual bool isFunctionSafeToSplit(const MachineFunction &MF) const
Return true if the function is a viable candidate for machine function splitting.
Primary interface to the complete machine description for the target machine.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
unsigned getCheckerSizeInBytes(AuthCheckMethod Method)
Returns the number of bytes added by checkAuthenticatedRegister.
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getArithShiftValue(unsigned Imm)
getArithShiftValue - get the arithmetic shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static AArch64_AM::ShiftExtendType getArithExtendType(unsigned Imm)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static const uint64_t InstrFlagIsWhile
static const uint64_t InstrFlagIsPTestLike
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static bool isCondBranchOpcode(int Opc)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
static bool isPTrueOpcode(unsigned Opc)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
static bool isIndirectBranchOpcode(int Opc)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetIsLegal
Offset is legal.
@ AArch64FrameOffsetCanUpdate
Offset can apply, at least partly.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
AArch64MachineCombinerPattern
@ MULSUBv2i32_indexed_OP1
@ MULADDv4i16_indexed_OP2
@ MULSUBv8i16_indexed_OP2
@ MULSUBv4i16_indexed_OP2
@ MULSUBv4i32_indexed_OP2
@ MULADDv2i32_indexed_OP1
@ MULADDv4i32_indexed_OP1
@ MULADDv2i32_indexed_OP2
@ MULSUBv4i16_indexed_OP1
@ MULADDv4i32_indexed_OP2
@ MULSUBv8i16_indexed_OP1
@ MULSUBv2i32_indexed_OP2
@ MULADDv8i16_indexed_OP1
@ MULSUBv4i32_indexed_OP1
@ MULADDv8i16_indexed_OP2
@ MULADDv4i16_indexed_OP1
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
std::optional< UsedNZCV > examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > *CCUseInstrs=nullptr)
CodeGenOptLevel
Code generation optimization level.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
unsigned getUndefRegState(bool B)
unsigned getDefRegState(bool B)
unsigned getKillRegState(bool B)
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
DWARFExpression::Operation Op
static bool isUncondBranchOpcode(int Opc)
unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a SLEB128 value to an output stream.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx, unsigned FrameReg, StackOffset &Offset, const AArch64InstrInfo *TII)
rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the FP.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
static const MachineMemOperand::Flags MOSuppressPair
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI, const MachineInstr &UseMI, const TargetRegisterInfo *TRI)
Return true if there is an instruction /after/ DefMI and before UseMI which either reads or clobbers ...
static const MachineMemOperand::Flags MOStridedAccess
@ Default
The result values are uniform if and only if all operands are uniform.
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Description of the encoding of one expression Op.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
static const MBBSectionID ColdSectionID
MachineJumpTableEntry - One jump table in the jump table info.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Used to describe a register and immediate addition.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.
unsigned FrameConstructionID
Target-defined identifier for constructing a frame for this function.