61#define GET_INSTRINFO_CTOR_DTOR
62#include "AArch64GenInstrInfo.inc"
66 cl::desc(
"Restrict range of TB[N]Z instructions (DEBUG)"));
70 cl::desc(
"Restrict range of CB[N]Z instructions (DEBUG)"));
74 cl::desc(
"Restrict range of Bcc instructions (DEBUG)"));
78 cl::desc(
"Restrict range of B instructions (DEBUG)"));
83 RI(STI.getTargetTriple()), Subtarget(STI) {}
94 auto Op =
MI.getOpcode();
95 if (
Op == AArch64::INLINEASM ||
Op == AArch64::INLINEASM_BR)
96 return getInlineAsmLength(
MI.getOperand(0).getSymbolName(), *MAI);
100 if (
MI.isMetaInstruction())
105 unsigned NumBytes = 0;
111 switch (
Desc.getOpcode()) {
114 return Desc.getSize();
121 case TargetOpcode::STACKMAP:
124 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
126 case TargetOpcode::PATCHPOINT:
129 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
131 case TargetOpcode::STATEPOINT:
133 assert(NumBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
138 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
143 F.getFnAttributeAsParsedInteger(
"patchable-function-entry", 9) * 4;
145 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
146 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
150 case TargetOpcode::PATCHABLE_EVENT_CALL:
156 NumBytes =
MI.getOperand(1).getImm();
158 case TargetOpcode::BUNDLE:
159 NumBytes = getInstBundleLength(
MI);
166unsigned AArch64InstrInfo::getInstBundleLength(
const MachineInstr &
MI)
const {
170 while (++
I !=
E &&
I->isInsideBundle()) {
171 assert(!
I->isBundle() &&
"No nested bundle!");
230 int64_t BrOffset)
const {
232 assert(Bits >= 3 &&
"max branch displacement must be enough to jump"
233 "over conditional branch expansion");
234 return isIntN(Bits, BrOffset / 4);
239 switch (
MI.getOpcode()) {
243 return MI.getOperand(0).getMBB();
248 return MI.getOperand(2).getMBB();
254 return MI.getOperand(1).getMBB();
264 assert(RS &&
"RegScavenger required for long branching");
266 "new block should be inserted for expanding unconditional branch");
269 "restore block should be inserted for restoring clobbered registers");
274 if (!isInt<33>(BrOffset))
276 "Branch offsets outside of the signed 33-bit range not supported");
290 constexpr Register Reg = AArch64::X16;
292 insertUnconditionalBranch(
MBB, &NewDestBB,
DL);
300 if (Scavenged != AArch64::NoRegister &&
302 buildIndirectBranch(Scavenged, NewDestBB);
312 "Unable to insert indirect branch inside function that has red zone");
335 bool AllowModify)
const {
342 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
343 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
347 if (!isUnpredicatedTerminator(*
I))
354 unsigned LastOpc = LastInst->
getOpcode();
355 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
370 unsigned SecondLastOpc = SecondLastInst->
getOpcode();
377 LastInst = SecondLastInst;
379 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
384 SecondLastInst = &*
I;
385 SecondLastOpc = SecondLastInst->
getOpcode();
396 LastInst = SecondLastInst;
398 if (
I ==
MBB.
begin() || !isUnpredicatedTerminator(*--
I)) {
400 "unreachable unconditional branches removed above");
409 SecondLastInst = &*
I;
410 SecondLastOpc = SecondLastInst->
getOpcode();
414 if (SecondLastInst &&
I !=
MBB.
begin() && isUnpredicatedTerminator(*--
I))
430 I->eraseFromParent();
439 I->eraseFromParent();
448 MachineBranchPredicate &MBP,
449 bool AllowModify)
const {
459 if (
I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
460 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
464 if (!isUnpredicatedTerminator(*
I))
469 unsigned LastOpc = LastInst->
getOpcode();
484 assert(MBP.TrueDest &&
"expected!");
487 MBP.ConditionDef =
nullptr;
488 MBP.SingleUseCondition =
false;
492 MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
493 : MachineBranchPredicate::PRED_EQ;
499 if (
Cond[0].getImm() != -1) {
505 switch (
Cond[1].getImm()) {
509 Cond[1].setImm(AArch64::CBNZW);
512 Cond[1].setImm(AArch64::CBZW);
515 Cond[1].setImm(AArch64::CBNZX);
518 Cond[1].setImm(AArch64::CBZX);
521 Cond[1].setImm(AArch64::TBNZW);
524 Cond[1].setImm(AArch64::TBZW);
527 Cond[1].setImm(AArch64::TBNZX);
530 Cond[1].setImm(AArch64::TBZX);
539 int *BytesRemoved)
const {
549 I->eraseFromParent();
566 I->eraseFromParent();
573void AArch64InstrInfo::instantiateCondBranch(
576 if (
Cond[0].getImm() != -1) {
594 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
633 unsigned *NewVReg =
nullptr) {
638 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(
MRI.getRegClass(VReg));
641 unsigned SrcOpNum = 0;
643 case AArch64::ADDSXri:
644 case AArch64::ADDSWri:
650 case AArch64::ADDXri:
651 case AArch64::ADDWri:
657 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
660 case AArch64::ORNXrr:
661 case AArch64::ORNWrr: {
664 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
667 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
671 case AArch64::SUBSXrr:
672 case AArch64::SUBSWrr:
678 case AArch64::SUBXrr:
679 case AArch64::SUBWrr: {
682 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
685 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
691 assert(Opc && SrcOpNum &&
"Missing parameters");
703 int &FalseCycles)
const {
707 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
714 if (!RI.getCommonSubClass(RC,
MRI.getRegClass(DstReg)))
718 unsigned ExtraCondLat =
Cond.size() != 1;
722 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
723 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
725 CondCycles = 1 + ExtraCondLat;
726 TrueCycles = FalseCycles = 1;
736 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
737 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
738 CondCycles = 5 + ExtraCondLat;
739 TrueCycles = FalseCycles = 2;
756 switch (
Cond.size()) {
765 switch (
Cond[1].getImm()) {
788 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
794 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
804 switch (
Cond[1].getImm()) {
817 if (
Cond[1].getImm() == AArch64::TBZW ||
Cond[1].getImm() == AArch64::TBNZW)
833 bool TryFold =
false;
834 if (
MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
835 RC = &AArch64::GPR64RegClass;
836 Opc = AArch64::CSELXr;
838 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
839 RC = &AArch64::GPR32RegClass;
840 Opc = AArch64::CSELWr;
842 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
843 RC = &AArch64::FPR64RegClass;
844 Opc = AArch64::FCSELDrrr;
845 }
else if (
MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
846 RC = &AArch64::FPR32RegClass;
847 Opc = AArch64::FCSELSrrr;
849 assert(RC &&
"Unsupported regclass");
853 unsigned NewVReg = 0;
868 MRI.clearKillFlags(NewVReg);
873 MRI.constrainRegClass(TrueReg, RC);
874 MRI.constrainRegClass(FalseReg, RC);
889 assert(BitSize == 64 &&
"Only bit sizes of 32 or 64 allowed");
894 return Is.
size() <= 2;
900 if (Subtarget.hasExynosCheapAsMoveHandling()) {
901 if (isExynosCheapAsMove(
MI))
903 return MI.isAsCheapAsAMove();
906 switch (
MI.getOpcode()) {
908 return MI.isAsCheapAsAMove();
910 case AArch64::ADDWrs:
911 case AArch64::ADDXrs:
912 case AArch64::SUBWrs:
913 case AArch64::SUBXrs:
914 return Subtarget.hasALULSLFast() &&
MI.getOperand(3).getImm() <= 4;
919 case AArch64::MOVi32imm:
921 case AArch64::MOVi64imm:
927 switch (
MI.getOpcode()) {
931 case AArch64::ADDWrs:
932 case AArch64::ADDXrs:
933 case AArch64::ADDSWrs:
934 case AArch64::ADDSXrs: {
935 unsigned Imm =
MI.getOperand(3).getImm();
942 case AArch64::ADDWrx:
943 case AArch64::ADDXrx:
944 case AArch64::ADDXrx64:
945 case AArch64::ADDSWrx:
946 case AArch64::ADDSXrx:
947 case AArch64::ADDSXrx64: {
948 unsigned Imm =
MI.getOperand(3).getImm();
960 case AArch64::SUBWrs:
961 case AArch64::SUBSWrs: {
962 unsigned Imm =
MI.getOperand(3).getImm();
964 return ShiftVal == 0 ||
968 case AArch64::SUBXrs:
969 case AArch64::SUBSXrs: {
970 unsigned Imm =
MI.getOperand(3).getImm();
972 return ShiftVal == 0 ||
976 case AArch64::SUBWrx:
977 case AArch64::SUBXrx:
978 case AArch64::SUBXrx64:
979 case AArch64::SUBSWrx:
980 case AArch64::SUBSXrx:
981 case AArch64::SUBSXrx64: {
982 unsigned Imm =
MI.getOperand(3).getImm();
994 case AArch64::LDRBBroW:
995 case AArch64::LDRBBroX:
996 case AArch64::LDRBroW:
997 case AArch64::LDRBroX:
998 case AArch64::LDRDroW:
999 case AArch64::LDRDroX:
1000 case AArch64::LDRHHroW:
1001 case AArch64::LDRHHroX:
1002 case AArch64::LDRHroW:
1003 case AArch64::LDRHroX:
1004 case AArch64::LDRQroW:
1005 case AArch64::LDRQroX:
1006 case AArch64::LDRSBWroW:
1007 case AArch64::LDRSBWroX:
1008 case AArch64::LDRSBXroW:
1009 case AArch64::LDRSBXroX:
1010 case AArch64::LDRSHWroW:
1011 case AArch64::LDRSHWroX:
1012 case AArch64::LDRSHXroW:
1013 case AArch64::LDRSHXroX:
1014 case AArch64::LDRSWroW:
1015 case AArch64::LDRSWroX:
1016 case AArch64::LDRSroW:
1017 case AArch64::LDRSroX:
1018 case AArch64::LDRWroW:
1019 case AArch64::LDRWroX:
1020 case AArch64::LDRXroW:
1021 case AArch64::LDRXroX:
1022 case AArch64::PRFMroW:
1023 case AArch64::PRFMroX:
1024 case AArch64::STRBBroW:
1025 case AArch64::STRBBroX:
1026 case AArch64::STRBroW:
1027 case AArch64::STRBroX:
1028 case AArch64::STRDroW:
1029 case AArch64::STRDroX:
1030 case AArch64::STRHHroW:
1031 case AArch64::STRHHroX:
1032 case AArch64::STRHroW:
1033 case AArch64::STRHroX:
1034 case AArch64::STRQroW:
1035 case AArch64::STRQroX:
1036 case AArch64::STRSroW:
1037 case AArch64::STRSroX:
1038 case AArch64::STRWroW:
1039 case AArch64::STRWroX:
1040 case AArch64::STRXroW:
1041 case AArch64::STRXroX: {
1042 unsigned IsSigned =
MI.getOperand(3).getImm();
1049 unsigned Opc =
MI.getOpcode();
1053 case AArch64::SEH_StackAlloc:
1054 case AArch64::SEH_SaveFPLR:
1055 case AArch64::SEH_SaveFPLR_X:
1056 case AArch64::SEH_SaveReg:
1057 case AArch64::SEH_SaveReg_X:
1058 case AArch64::SEH_SaveRegP:
1059 case AArch64::SEH_SaveRegP_X:
1060 case AArch64::SEH_SaveFReg:
1061 case AArch64::SEH_SaveFReg_X:
1062 case AArch64::SEH_SaveFRegP:
1063 case AArch64::SEH_SaveFRegP_X:
1064 case AArch64::SEH_SetFP:
1065 case AArch64::SEH_AddFP:
1066 case AArch64::SEH_Nop:
1067 case AArch64::SEH_PrologEnd:
1068 case AArch64::SEH_EpilogStart:
1069 case AArch64::SEH_EpilogEnd:
1070 case AArch64::SEH_PACSignLR:
1071 case AArch64::SEH_SaveAnyRegQP:
1072 case AArch64::SEH_SaveAnyRegQPX:
1079 unsigned &SubIdx)
const {
1080 switch (
MI.getOpcode()) {
1083 case AArch64::SBFMXri:
1084 case AArch64::UBFMXri:
1087 if (
MI.getOperand(2).getImm() != 0 ||
MI.getOperand(3).getImm() != 31)
1090 SrcReg =
MI.getOperand(1).getReg();
1091 DstReg =
MI.getOperand(0).getReg();
1092 SubIdx = AArch64::sub_32;
1101 int64_t OffsetA = 0, OffsetB = 0;
1102 TypeSize WidthA(0,
false), WidthB(0,
false);
1103 bool OffsetAIsScalable =
false, OffsetBIsScalable =
false;
1124 OffsetAIsScalable == OffsetBIsScalable) {
1125 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1126 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1127 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1128 if (LowWidth.
isScalable() == OffsetAIsScalable &&
1146 switch (
MI.getOpcode()) {
1149 if (
MI.getOperand(0).getImm() == 0x14)
1156 case AArch64::MSRpstatesvcrImm1:
1163 auto Next = std::next(
MI.getIterator());
1164 return Next !=
MBB->
end() && Next->isCFIInstruction();
1171 Register &SrcReg2, int64_t &CmpMask,
1172 int64_t &CmpValue)
const {
1175 assert(
MI.getNumOperands() >= 2 &&
"All AArch64 cmps should have 2 operands");
1176 if (!
MI.getOperand(1).isReg())
1179 switch (
MI.getOpcode()) {
1182 case AArch64::PTEST_PP:
1183 case AArch64::PTEST_PP_ANY:
1184 SrcReg =
MI.getOperand(0).getReg();
1185 SrcReg2 =
MI.getOperand(1).getReg();
1190 case AArch64::SUBSWrr:
1191 case AArch64::SUBSWrs:
1192 case AArch64::SUBSWrx:
1193 case AArch64::SUBSXrr:
1194 case AArch64::SUBSXrs:
1195 case AArch64::SUBSXrx:
1196 case AArch64::ADDSWrr:
1197 case AArch64::ADDSWrs:
1198 case AArch64::ADDSWrx:
1199 case AArch64::ADDSXrr:
1200 case AArch64::ADDSXrs:
1201 case AArch64::ADDSXrx:
1203 SrcReg =
MI.getOperand(1).getReg();
1204 SrcReg2 =
MI.getOperand(2).getReg();
1208 case AArch64::SUBSWri:
1209 case AArch64::ADDSWri:
1210 case AArch64::SUBSXri:
1211 case AArch64::ADDSXri:
1212 SrcReg =
MI.getOperand(1).getReg();
1215 CmpValue =
MI.getOperand(2).getImm();
1217 case AArch64::ANDSWri:
1218 case AArch64::ANDSXri:
1221 SrcReg =
MI.getOperand(1).getReg();
1225 MI.getOperand(2).getImm(),
1226 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1235 assert(
MBB &&
"Can't get MachineBasicBlock here");
1237 assert(MF &&
"Can't get MachineFunction here");
1242 for (
unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1246 Instr.getRegClassConstraint(OpIdx,
TII,
TRI);
1249 if (!OpRegCstraints)
1257 "Operand has register constraints without being a register!");
1260 if (Reg.isPhysical()) {
1261 if (!OpRegCstraints->
contains(Reg))
1264 !
MRI->constrainRegClass(Reg, OpRegCstraints))
1277 bool MIDefinesZeroReg =
false;
1278 if (
MI.definesRegister(AArch64::WZR) ||
MI.definesRegister(AArch64::XZR))
1279 MIDefinesZeroReg =
true;
1281 switch (
MI.getOpcode()) {
1283 return MI.getOpcode();
1284 case AArch64::ADDSWrr:
1285 return AArch64::ADDWrr;
1286 case AArch64::ADDSWri:
1287 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1288 case AArch64::ADDSWrs:
1289 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1290 case AArch64::ADDSWrx:
1291 return AArch64::ADDWrx;
1292 case AArch64::ADDSXrr:
1293 return AArch64::ADDXrr;
1294 case AArch64::ADDSXri:
1295 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1296 case AArch64::ADDSXrs:
1297 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1298 case AArch64::ADDSXrx:
1299 return AArch64::ADDXrx;
1300 case AArch64::SUBSWrr:
1301 return AArch64::SUBWrr;
1302 case AArch64::SUBSWri:
1303 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1304 case AArch64::SUBSWrs:
1305 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1306 case AArch64::SUBSWrx:
1307 return AArch64::SUBWrx;
1308 case AArch64::SUBSXrr:
1309 return AArch64::SUBXrr;
1310 case AArch64::SUBSXri:
1311 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1312 case AArch64::SUBSXrs:
1313 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1314 case AArch64::SUBSXrx:
1315 return AArch64::SUBXrx;
1330 if (To == To->getParent()->begin())
1335 if (To->getParent() !=
From->getParent())
1347 Instr.modifiesRegister(AArch64::NZCV,
TRI)) ||
1348 ((AccessToCheck &
AK_Read) && Instr.readsRegister(AArch64::NZCV,
TRI)))
1356bool AArch64InstrInfo::optimizePTestInstr(
1357 MachineInstr *PTest,
unsigned MaskReg,
unsigned PredReg,
1359 auto *
Mask =
MRI->getUniqueVRegDef(MaskReg);
1360 auto *Pred =
MRI->getUniqueVRegDef(PredReg);
1361 auto NewOp = Pred->getOpcode();
1362 bool OpChanged =
false;
1364 unsigned MaskOpcode =
Mask->getOpcode();
1365 unsigned PredOpcode = Pred->getOpcode();
1369 if (
isPTrueOpcode(MaskOpcode) && (PredIsPTestLike || PredIsWhileLike) &&
1372 Mask->getOperand(1).getImm() == 31) {
1381 if (PredIsPTestLike) {
1382 auto PTestLikeMask =
MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1383 if (Mask != PTestLikeMask && PTest->
getOpcode() != AArch64::PTEST_PP_ANY)
1388 }
else if ((Mask == Pred) && (PredIsPTestLike || PredIsWhileLike) &&
1389 PTest->
getOpcode() == AArch64::PTEST_PP_ANY) {
1395 }
else if (PredIsPTestLike) {
1418 auto PTestLikeMask =
MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1420 if ((Mask != PTestLikeMask) ||
1422 PTest->
getOpcode() != AArch64::PTEST_PP_ANY))
1429 switch (PredOpcode) {
1430 case AArch64::AND_PPzPP:
1431 case AArch64::BIC_PPzPP:
1432 case AArch64::EOR_PPzPP:
1433 case AArch64::NAND_PPzPP:
1434 case AArch64::NOR_PPzPP:
1435 case AArch64::ORN_PPzPP:
1436 case AArch64::ORR_PPzPP:
1437 case AArch64::BRKA_PPzP:
1438 case AArch64::BRKPA_PPzPP:
1439 case AArch64::BRKB_PPzP:
1440 case AArch64::BRKPB_PPzPP:
1441 case AArch64::RDFFR_PPz: {
1444 auto *PredMask =
MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1445 if (Mask != PredMask)
1449 case AArch64::BRKN_PPzP: {
1453 if ((MaskOpcode != AArch64::PTRUE_B) ||
1454 (
Mask->getOperand(1).getImm() != 31))
1458 case AArch64::PTRUE_B:
1481 Pred->setDesc(
get(NewOp));
1486 assert(succeeded &&
"Operands have incompatible register classes!");
1487 Pred->addRegisterDefined(AArch64::NZCV,
TRI);
1491 if (Pred->registerDefIsDead(AArch64::NZCV,
TRI)) {
1492 unsigned i = 0,
e = Pred->getNumOperands();
1493 for (; i !=
e; ++i) {
1523 if (DeadNZCVIdx != -1) {
1538 assert(succeeded &&
"Some operands reg class are incompatible!");
1542 if (CmpInstr.
getOpcode() == AArch64::PTEST_PP ||
1543 CmpInstr.
getOpcode() == AArch64::PTEST_PP_ANY)
1544 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2,
MRI);
1553 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *
MRI))
1555 return (CmpValue == 0 || CmpValue == 1) &&
1556 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *
MRI);
1564 switch (Instr.getOpcode()) {
1566 return AArch64::INSTRUCTION_LIST_END;
1568 case AArch64::ADDSWrr:
1569 case AArch64::ADDSWri:
1570 case AArch64::ADDSXrr:
1571 case AArch64::ADDSXri:
1572 case AArch64::SUBSWrr:
1573 case AArch64::SUBSWri:
1574 case AArch64::SUBSXrr:
1575 case AArch64::SUBSXri:
1576 return Instr.getOpcode();
1578 case AArch64::ADDWrr:
1579 return AArch64::ADDSWrr;
1580 case AArch64::ADDWri:
1581 return AArch64::ADDSWri;
1582 case AArch64::ADDXrr:
1583 return AArch64::ADDSXrr;
1584 case AArch64::ADDXri:
1585 return AArch64::ADDSXri;
1586 case AArch64::ADCWr:
1587 return AArch64::ADCSWr;
1588 case AArch64::ADCXr:
1589 return AArch64::ADCSXr;
1590 case AArch64::SUBWrr:
1591 return AArch64::SUBSWrr;
1592 case AArch64::SUBWri:
1593 return AArch64::SUBSWri;
1594 case AArch64::SUBXrr:
1595 return AArch64::SUBSXrr;
1596 case AArch64::SUBXri:
1597 return AArch64::SUBSXri;
1598 case AArch64::SBCWr:
1599 return AArch64::SBCSWr;
1600 case AArch64::SBCXr:
1601 return AArch64::SBCSXr;
1602 case AArch64::ANDWri:
1603 return AArch64::ANDSWri;
1604 case AArch64::ANDXri:
1605 return AArch64::ANDSXri;
1612 if (BB->isLiveIn(AArch64::NZCV))
1621 switch (Instr.getOpcode()) {
1625 case AArch64::Bcc: {
1626 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1631 case AArch64::CSINVWr:
1632 case AArch64::CSINVXr:
1633 case AArch64::CSINCWr:
1634 case AArch64::CSINCXr:
1635 case AArch64::CSELWr:
1636 case AArch64::CSELXr:
1637 case AArch64::CSNEGWr:
1638 case AArch64::CSNEGXr:
1639 case AArch64::FCSELSrrr:
1640 case AArch64::FCSELDrrr: {
1641 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1654 Instr.getOperand(CCIdx).getImm())
1707std::optional<UsedNZCV>
1712 if (
MI.getParent() != CmpParent)
1713 return std::nullopt;
1716 return std::nullopt;
1721 if (Instr.readsRegister(AArch64::NZCV, &
TRI)) {
1724 return std::nullopt;
1729 if (Instr.modifiesRegister(AArch64::NZCV, &
TRI))
1732 return NZCVUsedAfterCmp;
1736 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1740 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1762 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1768 "Caller guarantees that CmpInstr compares with constant 0");
1771 if (!NZVCUsed || NZVCUsed->C)
1793bool AArch64InstrInfo::substituteCmpToZero(
1804 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1811 MI->setDesc(
get(NewOpc));
1815 assert(succeeded &&
"Some operands reg class are incompatible!");
1816 MI->addRegisterDefined(AArch64::NZCV, &
TRI);
1828 assert((CmpValue == 0 || CmpValue == 1) &&
1829 "Only comparisons to 0 or 1 considered for removal!");
1832 unsigned MIOpc =
MI.getOpcode();
1833 if (MIOpc == AArch64::CSINCWr) {
1834 if (
MI.getOperand(1).getReg() != AArch64::WZR ||
1835 MI.getOperand(2).getReg() != AArch64::WZR)
1837 }
else if (MIOpc == AArch64::CSINCXr) {
1838 if (
MI.getOperand(1).getReg() != AArch64::XZR ||
1839 MI.getOperand(2).getReg() != AArch64::XZR)
1849 if (
MI.findRegisterDefOperandIdx(AArch64::NZCV,
true) != -1)
1853 const unsigned CmpOpcode = CmpInstr.
getOpcode();
1855 if (CmpValue && !IsSubsRegImm)
1857 if (!CmpValue && !IsSubsRegImm && !
isADDSRegImm(CmpOpcode))
1862 if (MIUsedNZCV.
C || MIUsedNZCV.
V)
1865 std::optional<UsedNZCV> NZCVUsedAfterCmp =
1869 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
1872 if ((MIUsedNZCV.
Z && NZCVUsedAfterCmp->N) ||
1873 (MIUsedNZCV.
N && NZCVUsedAfterCmp->Z))
1876 if (MIUsedNZCV.
N && !CmpValue)
1918bool AArch64InstrInfo::removeCmpToZeroOrOne(
1926 bool IsInvertCC =
false;
1936 assert(
Idx >= 0 &&
"Unexpected instruction using CC.");
1947 if (
MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1948 MI.getOpcode() != AArch64::CATCHRET)
1956 if (
MI.getOpcode() == AArch64::CATCHRET) {
1965 FirstEpilogSEH = std::prev(FirstEpilogSEH);
1967 FirstEpilogSEH = std::next(FirstEpilogSEH);
1981 if (M.getStackProtectorGuard() ==
"sysreg") {
1991 int Offset = M.getStackProtectorGuardOffset();
2043 cast<GlobalValue>((*
MI.memoperands_begin())->getValue());
2052 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2094 unsigned Reg32 =
TRI->getSubReg(Reg, AArch64::sub_32);
2117 switch (
MI.getOpcode()) {
2120 case AArch64::MOVZWi:
2121 case AArch64::MOVZXi:
2122 if (
MI.getOperand(1).isImm() &&
MI.getOperand(1).getImm() == 0) {
2123 assert(
MI.getDesc().getNumOperands() == 3 &&
2124 MI.getOperand(2).getImm() == 0 &&
"invalid MOVZi operands");
2128 case AArch64::ANDWri:
2129 return MI.getOperand(1).getReg() == AArch64::WZR;
2130 case AArch64::ANDXri:
2131 return MI.getOperand(1).getReg() == AArch64::XZR;
2132 case TargetOpcode::COPY:
2133 return MI.getOperand(1).getReg() == AArch64::WZR;
2141 switch (
MI.getOpcode()) {
2144 case TargetOpcode::COPY: {
2147 return (AArch64::GPR32RegClass.
contains(DstReg) ||
2148 AArch64::GPR64RegClass.
contains(DstReg));
2150 case AArch64::ORRXrs:
2151 if (
MI.getOperand(1).getReg() == AArch64::XZR) {
2152 assert(
MI.getDesc().getNumOperands() == 4 &&
2153 MI.getOperand(3).getImm() == 0 &&
"invalid ORRrs operands");
2157 case AArch64::ADDXri:
2158 if (
MI.getOperand(2).getImm() == 0) {
2159 assert(
MI.getDesc().getNumOperands() == 4 &&
2160 MI.getOperand(3).getImm() == 0 &&
"invalid ADDXri operands");
2171 switch (
MI.getOpcode()) {
2174 case TargetOpcode::COPY: {
2176 return AArch64::FPR128RegClass.contains(DstReg);
2178 case AArch64::ORRv16i8:
2179 if (
MI.getOperand(1).getReg() ==
MI.getOperand(2).getReg()) {
2180 assert(
MI.getDesc().getNumOperands() == 3 &&
MI.getOperand(0).isReg() &&
2181 "invalid ORRv16i8 operands");
2190 int &FrameIndex)
const {
2191 switch (
MI.getOpcode()) {
2194 case AArch64::LDRWui:
2195 case AArch64::LDRXui:
2196 case AArch64::LDRBui:
2197 case AArch64::LDRHui:
2198 case AArch64::LDRSui:
2199 case AArch64::LDRDui:
2200 case AArch64::LDRQui:
2201 case AArch64::LDR_PXI:
2202 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2203 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2204 FrameIndex =
MI.getOperand(1).getIndex();
2205 return MI.getOperand(0).getReg();
2214 int &FrameIndex)
const {
2215 switch (
MI.getOpcode()) {
2218 case AArch64::STRWui:
2219 case AArch64::STRXui:
2220 case AArch64::STRBui:
2221 case AArch64::STRHui:
2222 case AArch64::STRSui:
2223 case AArch64::STRDui:
2224 case AArch64::STRQui:
2225 case AArch64::STR_PXI:
2226 if (
MI.getOperand(0).getSubReg() == 0 &&
MI.getOperand(1).isFI() &&
2227 MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
2228 FrameIndex =
MI.getOperand(1).getIndex();
2229 return MI.getOperand(0).getReg();
2239 return MMO->getFlags() & MOSuppressPair;
2245 if (
MI.memoperands_empty())
2253 return MMO->getFlags() & MOStridedAccess;
2261 case AArch64::STURSi:
2262 case AArch64::STRSpre:
2263 case AArch64::STURDi:
2264 case AArch64::STRDpre:
2265 case AArch64::STURQi:
2266 case AArch64::STRQpre:
2267 case AArch64::STURBBi:
2268 case AArch64::STURHHi:
2269 case AArch64::STURWi:
2270 case AArch64::STRWpre:
2271 case AArch64::STURXi:
2272 case AArch64::STRXpre:
2273 case AArch64::LDURSi:
2274 case AArch64::LDRSpre:
2275 case AArch64::LDURDi:
2276 case AArch64::LDRDpre:
2277 case AArch64::LDURQi:
2278 case AArch64::LDRQpre:
2279 case AArch64::LDURWi:
2280 case AArch64::LDRWpre:
2281 case AArch64::LDURXi:
2282 case AArch64::LDRXpre:
2283 case AArch64::LDRSWpre:
2284 case AArch64::LDURSWi:
2285 case AArch64::LDURHHi:
2286 case AArch64::LDURBBi:
2287 case AArch64::LDURSBWi:
2288 case AArch64::LDURSHWi:
2296 case AArch64::PRFMui:
return AArch64::PRFUMi;
2297 case AArch64::LDRXui:
return AArch64::LDURXi;
2298 case AArch64::LDRWui:
return AArch64::LDURWi;
2299 case AArch64::LDRBui:
return AArch64::LDURBi;
2300 case AArch64::LDRHui:
return AArch64::LDURHi;
2301 case AArch64::LDRSui:
return AArch64::LDURSi;
2302 case AArch64::LDRDui:
return AArch64::LDURDi;
2303 case AArch64::LDRQui:
return AArch64::LDURQi;
2304 case AArch64::LDRBBui:
return AArch64::LDURBBi;
2305 case AArch64::LDRHHui:
return AArch64::LDURHHi;
2306 case AArch64::LDRSBXui:
return AArch64::LDURSBXi;
2307 case AArch64::LDRSBWui:
return AArch64::LDURSBWi;
2308 case AArch64::LDRSHXui:
return AArch64::LDURSHXi;
2309 case AArch64::LDRSHWui:
return AArch64::LDURSHWi;
2310 case AArch64::LDRSWui:
return AArch64::LDURSWi;
2311 case AArch64::STRXui:
return AArch64::STURXi;
2312 case AArch64::STRWui:
return AArch64::STURWi;
2313 case AArch64::STRBui:
return AArch64::STURBi;
2314 case AArch64::STRHui:
return AArch64::STURHi;
2315 case AArch64::STRSui:
return AArch64::STURSi;
2316 case AArch64::STRDui:
return AArch64::STURDi;
2317 case AArch64::STRQui:
return AArch64::STURQi;
2318 case AArch64::STRBBui:
return AArch64::STURBBi;
2319 case AArch64::STRHHui:
return AArch64::STURHHi;
2327 case AArch64::LDPXi:
2328 case AArch64::LDPDi:
2329 case AArch64::STPXi:
2330 case AArch64::STPDi:
2331 case AArch64::LDNPXi:
2332 case AArch64::LDNPDi:
2333 case AArch64::STNPXi:
2334 case AArch64::STNPDi:
2335 case AArch64::LDPQi:
2336 case AArch64::STPQi:
2337 case AArch64::LDNPQi:
2338 case AArch64::STNPQi:
2339 case AArch64::LDPWi:
2340 case AArch64::LDPSi:
2341 case AArch64::STPWi:
2342 case AArch64::STPSi:
2343 case AArch64::LDNPWi:
2344 case AArch64::LDNPSi:
2345 case AArch64::STNPWi:
2346 case AArch64::STNPSi:
2348 case AArch64::STGPi:
2350 case AArch64::LD1B_IMM:
2351 case AArch64::LD1B_H_IMM:
2352 case AArch64::LD1B_S_IMM:
2353 case AArch64::LD1B_D_IMM:
2354 case AArch64::LD1SB_H_IMM:
2355 case AArch64::LD1SB_S_IMM:
2356 case AArch64::LD1SB_D_IMM:
2357 case AArch64::LD1H_IMM:
2358 case AArch64::LD1H_S_IMM:
2359 case AArch64::LD1H_D_IMM:
2360 case AArch64::LD1SH_S_IMM:
2361 case AArch64::LD1SH_D_IMM:
2362 case AArch64::LD1W_IMM:
2363 case AArch64::LD1W_D_IMM:
2364 case AArch64::LD1SW_D_IMM:
2365 case AArch64::LD1D_IMM:
2367 case AArch64::LD2B_IMM:
2368 case AArch64::LD2H_IMM:
2369 case AArch64::LD2W_IMM:
2370 case AArch64::LD2D_IMM:
2371 case AArch64::LD3B_IMM:
2372 case AArch64::LD3H_IMM:
2373 case AArch64::LD3W_IMM:
2374 case AArch64::LD3D_IMM:
2375 case AArch64::LD4B_IMM:
2376 case AArch64::LD4H_IMM:
2377 case AArch64::LD4W_IMM:
2378 case AArch64::LD4D_IMM:
2380 case AArch64::ST1B_IMM:
2381 case AArch64::ST1B_H_IMM:
2382 case AArch64::ST1B_S_IMM:
2383 case AArch64::ST1B_D_IMM:
2384 case AArch64::ST1H_IMM:
2385 case AArch64::ST1H_S_IMM:
2386 case AArch64::ST1H_D_IMM:
2387 case AArch64::ST1W_IMM:
2388 case AArch64::ST1W_D_IMM:
2389 case AArch64::ST1D_IMM:
2391 case AArch64::ST2B_IMM:
2392 case AArch64::ST2H_IMM:
2393 case AArch64::ST2W_IMM:
2394 case AArch64::ST2D_IMM:
2395 case AArch64::ST3B_IMM:
2396 case AArch64::ST3H_IMM:
2397 case AArch64::ST3W_IMM:
2398 case AArch64::ST3D_IMM:
2399 case AArch64::ST4B_IMM:
2400 case AArch64::ST4H_IMM:
2401 case AArch64::ST4W_IMM:
2402 case AArch64::ST4D_IMM:
2404 case AArch64::LD1RB_IMM:
2405 case AArch64::LD1RB_H_IMM:
2406 case AArch64::LD1RB_S_IMM:
2407 case AArch64::LD1RB_D_IMM:
2408 case AArch64::LD1RSB_H_IMM:
2409 case AArch64::LD1RSB_S_IMM:
2410 case AArch64::LD1RSB_D_IMM:
2411 case AArch64::LD1RH_IMM:
2412 case AArch64::LD1RH_S_IMM:
2413 case AArch64::LD1RH_D_IMM:
2414 case AArch64::LD1RSH_S_IMM:
2415 case AArch64::LD1RSH_D_IMM:
2416 case AArch64::LD1RW_IMM:
2417 case AArch64::LD1RW_D_IMM:
2418 case AArch64::LD1RSW_IMM:
2419 case AArch64::LD1RD_IMM:
2421 case AArch64::LDNT1B_ZRI:
2422 case AArch64::LDNT1H_ZRI:
2423 case AArch64::LDNT1W_ZRI:
2424 case AArch64::LDNT1D_ZRI:
2425 case AArch64::STNT1B_ZRI:
2426 case AArch64::STNT1H_ZRI:
2427 case AArch64::STNT1W_ZRI:
2428 case AArch64::STNT1D_ZRI:
2430 case AArch64::LDNF1B_IMM:
2431 case AArch64::LDNF1B_H_IMM:
2432 case AArch64::LDNF1B_S_IMM:
2433 case AArch64::LDNF1B_D_IMM:
2434 case AArch64::LDNF1SB_H_IMM:
2435 case AArch64::LDNF1SB_S_IMM:
2436 case AArch64::LDNF1SB_D_IMM:
2437 case AArch64::LDNF1H_IMM:
2438 case AArch64::LDNF1H_S_IMM:
2439 case AArch64::LDNF1H_D_IMM:
2440 case AArch64::LDNF1SH_S_IMM:
2441 case AArch64::LDNF1SH_D_IMM:
2442 case AArch64::LDNF1W_IMM:
2443 case AArch64::LDNF1W_D_IMM:
2444 case AArch64::LDNF1SW_D_IMM:
2445 case AArch64::LDNF1D_IMM:
2449 case AArch64::LDR_PXI:
2450 case AArch64::STR_PXI:
2456 switch (
MI.getOpcode()) {
2460 case AArch64::STRSui:
2461 case AArch64::STRDui:
2462 case AArch64::STRQui:
2463 case AArch64::STRXui:
2464 case AArch64::STRWui:
2465 case AArch64::LDRSui:
2466 case AArch64::LDRDui:
2467 case AArch64::LDRQui:
2468 case AArch64::LDRXui:
2469 case AArch64::LDRWui:
2470 case AArch64::LDRSWui:
2472 case AArch64::STURSi:
2473 case AArch64::STRSpre:
2474 case AArch64::STURDi:
2475 case AArch64::STRDpre:
2476 case AArch64::STURQi:
2477 case AArch64::STRQpre:
2478 case AArch64::STURWi:
2479 case AArch64::STRWpre:
2480 case AArch64::STURXi:
2481 case AArch64::STRXpre:
2482 case AArch64::LDURSi:
2483 case AArch64::LDRSpre:
2484 case AArch64::LDURDi:
2485 case AArch64::LDRDpre:
2486 case AArch64::LDURQi:
2487 case AArch64::LDRQpre:
2488 case AArch64::LDURWi:
2489 case AArch64::LDRWpre:
2490 case AArch64::LDURXi:
2491 case AArch64::LDRXpre:
2492 case AArch64::LDURSWi:
2493 case AArch64::LDRSWpre:
2499 switch (
MI.getOpcode()) {
2502 "Unexpected instruction - was a new tail call opcode introduced?");
2504 case AArch64::TCRETURNdi:
2505 case AArch64::TCRETURNri:
2506 case AArch64::TCRETURNrix16x17:
2507 case AArch64::TCRETURNrix17:
2508 case AArch64::TCRETURNrinotx16:
2509 case AArch64::TCRETURNriALL:
2519 case AArch64::ADDWri:
2520 return AArch64::ADDSWri;
2521 case AArch64::ADDWrr:
2522 return AArch64::ADDSWrr;
2523 case AArch64::ADDWrs:
2524 return AArch64::ADDSWrs;
2525 case AArch64::ADDWrx:
2526 return AArch64::ADDSWrx;
2527 case AArch64::ANDWri:
2528 return AArch64::ANDSWri;
2529 case AArch64::ANDWrr:
2530 return AArch64::ANDSWrr;
2531 case AArch64::ANDWrs:
2532 return AArch64::ANDSWrs;
2533 case AArch64::BICWrr:
2534 return AArch64::BICSWrr;
2535 case AArch64::BICWrs:
2536 return AArch64::BICSWrs;
2537 case AArch64::SUBWri:
2538 return AArch64::SUBSWri;
2539 case AArch64::SUBWrr:
2540 return AArch64::SUBSWrr;
2541 case AArch64::SUBWrs:
2542 return AArch64::SUBSWrs;
2543 case AArch64::SUBWrx:
2544 return AArch64::SUBSWrx;
2546 case AArch64::ADDXri:
2547 return AArch64::ADDSXri;
2548 case AArch64::ADDXrr:
2549 return AArch64::ADDSXrr;
2550 case AArch64::ADDXrs:
2551 return AArch64::ADDSXrs;
2552 case AArch64::ADDXrx:
2553 return AArch64::ADDSXrx;
2554 case AArch64::ANDXri:
2555 return AArch64::ANDSXri;
2556 case AArch64::ANDXrr:
2557 return AArch64::ANDSXrr;
2558 case AArch64::ANDXrs:
2559 return AArch64::ANDSXrs;
2560 case AArch64::BICXrr:
2561 return AArch64::BICSXrr;
2562 case AArch64::BICXrs:
2563 return AArch64::BICSXrs;
2564 case AArch64::SUBXri:
2565 return AArch64::SUBSXri;
2566 case AArch64::SUBXrr:
2567 return AArch64::SUBSXrr;
2568 case AArch64::SUBXrs:
2569 return AArch64::SUBSXrs;
2570 case AArch64::SUBXrx:
2571 return AArch64::SUBSXrx;
2573 case AArch64::AND_PPzPP:
2574 return AArch64::ANDS_PPzPP;
2575 case AArch64::BIC_PPzPP:
2576 return AArch64::BICS_PPzPP;
2577 case AArch64::EOR_PPzPP:
2578 return AArch64::EORS_PPzPP;
2579 case AArch64::NAND_PPzPP:
2580 return AArch64::NANDS_PPzPP;
2581 case AArch64::NOR_PPzPP:
2582 return AArch64::NORS_PPzPP;
2583 case AArch64::ORN_PPzPP:
2584 return AArch64::ORNS_PPzPP;
2585 case AArch64::ORR_PPzPP:
2586 return AArch64::ORRS_PPzPP;
2587 case AArch64::BRKA_PPzP:
2588 return AArch64::BRKAS_PPzP;
2589 case AArch64::BRKPA_PPzPP:
2590 return AArch64::BRKPAS_PPzPP;
2591 case AArch64::BRKB_PPzP:
2592 return AArch64::BRKBS_PPzP;
2593 case AArch64::BRKPB_PPzPP:
2594 return AArch64::BRKPBS_PPzPP;
2595 case AArch64::BRKN_PPzP:
2596 return AArch64::BRKNS_PPzP;
2597 case AArch64::RDFFR_PPz:
2598 return AArch64::RDFFRS_PPz;
2599 case AArch64::PTRUE_B:
2600 return AArch64::PTRUES_B;
2611 if (
MI.hasOrderedMemoryRef())
2616 assert((
MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2617 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2618 "Expected a reg or frame index operand.");
2622 bool IsImmPreLdSt = IsPreLdSt &&
MI.getOperand(3).isImm();
2624 if (!
MI.getOperand(2).isImm() && !IsImmPreLdSt)
2637 if (
MI.getOperand(1).isReg() && !IsPreLdSt) {
2638 Register BaseReg =
MI.getOperand(1).getReg();
2640 if (
MI.modifiesRegister(BaseReg,
TRI))
2653 const MCAsmInfo *MAI =
MI.getMF()->getTarget().getMCAsmInfo();
2655 MI.getMF()->getFunction().needsUnwindTableEntry();
2661 if (Subtarget.isPaired128Slow()) {
2662 switch (
MI.getOpcode()) {
2665 case AArch64::LDURQi:
2666 case AArch64::STURQi:
2667 case AArch64::LDRQui:
2668 case AArch64::STRQui:
2698std::optional<ExtAddrMode>
2703 bool OffsetIsScalable;
2704 if (!getMemOperandWithOffset(MemI,
Base,
Offset, OffsetIsScalable,
TRI))
2705 return std::nullopt;
2708 return std::nullopt;
2723 int64_t OffsetScale = 1;
2728 case AArch64::LDURQi:
2729 case AArch64::STURQi:
2733 case AArch64::LDURDi:
2734 case AArch64::STURDi:
2735 case AArch64::LDURXi:
2736 case AArch64::STURXi:
2740 case AArch64::LDURWi:
2741 case AArch64::LDURSWi:
2742 case AArch64::STURWi:
2746 case AArch64::LDURHi:
2747 case AArch64::STURHi:
2748 case AArch64::LDURHHi:
2749 case AArch64::STURHHi:
2750 case AArch64::LDURSHXi:
2751 case AArch64::LDURSHWi:
2755 case AArch64::LDRBroX:
2756 case AArch64::LDRBBroX:
2757 case AArch64::LDRSBXroX:
2758 case AArch64::LDRSBWroX:
2759 case AArch64::STRBroX:
2760 case AArch64::STRBBroX:
2761 case AArch64::LDURBi:
2762 case AArch64::LDURBBi:
2763 case AArch64::LDURSBXi:
2764 case AArch64::LDURSBWi:
2765 case AArch64::STURBi:
2766 case AArch64::STURBBi:
2767 case AArch64::LDRBui:
2768 case AArch64::LDRBBui:
2769 case AArch64::LDRSBXui:
2770 case AArch64::LDRSBWui:
2771 case AArch64::STRBui:
2772 case AArch64::STRBBui:
2776 case AArch64::LDRQroX:
2777 case AArch64::STRQroX:
2778 case AArch64::LDRQui:
2779 case AArch64::STRQui:
2784 case AArch64::LDRDroX:
2785 case AArch64::STRDroX:
2786 case AArch64::LDRXroX:
2787 case AArch64::STRXroX:
2788 case AArch64::LDRDui:
2789 case AArch64::STRDui:
2790 case AArch64::LDRXui:
2791 case AArch64::STRXui:
2796 case AArch64::LDRWroX:
2797 case AArch64::LDRSWroX:
2798 case AArch64::STRWroX:
2799 case AArch64::LDRWui:
2800 case AArch64::LDRSWui:
2801 case AArch64::STRWui:
2806 case AArch64::LDRHroX:
2807 case AArch64::STRHroX:
2808 case AArch64::LDRHHroX:
2809 case AArch64::STRHHroX:
2810 case AArch64::LDRSHXroX:
2811 case AArch64::LDRSHWroX:
2812 case AArch64::LDRHui:
2813 case AArch64::STRHui:
2814 case AArch64::LDRHHui:
2815 case AArch64::STRHHui:
2816 case AArch64::LDRSHXui:
2817 case AArch64::LDRSHWui:
2825 if (BaseRegOp.
isReg() && BaseRegOp.
getReg() == Reg)
2849 case AArch64::SBFMXri:
2862 AM.
Scale = OffsetScale;
2867 case TargetOpcode::SUBREG_TO_REG: {
2880 if (!OffsetReg.
isVirtual() || !
MRI.hasOneNonDBGUse(OffsetReg))
2884 if (
DefMI.getOpcode() != AArch64::ORRWrs ||
2886 DefMI.getOperand(3).getImm() != 0)
2893 AM.
Scale = OffsetScale;
2904 auto validateOffsetForLDP = [](
unsigned NumBytes, int64_t OldOffset,
2905 int64_t NewOffset) ->
bool {
2906 int64_t MinOffset, MaxOffset;
2923 return OldOffset < MinOffset || OldOffset > MaxOffset ||
2924 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
2926 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) ->
bool {
2928 int64_t NewOffset = OldOffset + Disp;
2933 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
2943 auto canFoldAddRegIntoAddrMode =
2960 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
2961 Subtarget.isSTRQroSlow();
2970 case AArch64::ADDXri:
2976 return canFoldAddSubImmIntoAddrMode(Disp);
2978 case AArch64::SUBXri:
2984 return canFoldAddSubImmIntoAddrMode(-Disp);
2986 case AArch64::ADDXrs: {
2999 if ((Shift != 2 && Shift != 3) || !Subtarget.hasAddrLSLFast())
3001 if (avoidSlowSTRQ(MemI))
3004 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3007 case AArch64::ADDXrr:
3015 if (!OptSize && avoidSlowSTRQ(MemI))
3017 return canFoldAddRegIntoAddrMode(1);
3019 case AArch64::ADDXrx:
3027 if (!OptSize && avoidSlowSTRQ(MemI))
3036 return canFoldAddRegIntoAddrMode(
3051 case AArch64::LDURQi:
3052 case AArch64::LDRQui:
3053 return AArch64::LDRQroX;
3054 case AArch64::STURQi:
3055 case AArch64::STRQui:
3056 return AArch64::STRQroX;
3057 case AArch64::LDURDi:
3058 case AArch64::LDRDui:
3059 return AArch64::LDRDroX;
3060 case AArch64::STURDi:
3061 case AArch64::STRDui:
3062 return AArch64::STRDroX;
3063 case AArch64::LDURXi:
3064 case AArch64::LDRXui:
3065 return AArch64::LDRXroX;
3066 case AArch64::STURXi:
3067 case AArch64::STRXui:
3068 return AArch64::STRXroX;
3069 case AArch64::LDURWi:
3070 case AArch64::LDRWui:
3071 return AArch64::LDRWroX;
3072 case AArch64::LDURSWi:
3073 case AArch64::LDRSWui:
3074 return AArch64::LDRSWroX;
3075 case AArch64::STURWi:
3076 case AArch64::STRWui:
3077 return AArch64::STRWroX;
3078 case AArch64::LDURHi:
3079 case AArch64::LDRHui:
3080 return AArch64::LDRHroX;
3081 case AArch64::STURHi:
3082 case AArch64::STRHui:
3083 return AArch64::STRHroX;
3084 case AArch64::LDURHHi:
3085 case AArch64::LDRHHui:
3086 return AArch64::LDRHHroX;
3087 case AArch64::STURHHi:
3088 case AArch64::STRHHui:
3089 return AArch64::STRHHroX;
3090 case AArch64::LDURSHXi:
3091 case AArch64::LDRSHXui:
3092 return AArch64::LDRSHXroX;
3093 case AArch64::LDURSHWi:
3094 case AArch64::LDRSHWui:
3095 return AArch64::LDRSHWroX;
3096 case AArch64::LDURBi:
3097 case AArch64::LDRBui:
3098 return AArch64::LDRBroX;
3099 case AArch64::LDURBBi:
3100 case AArch64::LDRBBui:
3101 return AArch64::LDRBBroX;
3102 case AArch64::LDURSBXi:
3103 case AArch64::LDRSBXui:
3104 return AArch64::LDRSBXroX;
3105 case AArch64::LDURSBWi:
3106 case AArch64::LDRSBWui:
3107 return AArch64::LDRSBWroX;
3108 case AArch64::STURBi:
3109 case AArch64::STRBui:
3110 return AArch64::STRBroX;
3111 case AArch64::STURBBi:
3112 case AArch64::STRBBui:
3113 return AArch64::STRBBroX;
3125 case AArch64::LDURQi:
3127 return AArch64::LDRQui;
3128 case AArch64::STURQi:
3130 return AArch64::STRQui;
3131 case AArch64::LDURDi:
3133 return AArch64::LDRDui;
3134 case AArch64::STURDi:
3136 return AArch64::STRDui;
3137 case AArch64::LDURXi:
3139 return AArch64::LDRXui;
3140 case AArch64::STURXi:
3142 return AArch64::STRXui;
3143 case AArch64::LDURWi:
3145 return AArch64::LDRWui;
3146 case AArch64::LDURSWi:
3148 return AArch64::LDRSWui;
3149 case AArch64::STURWi:
3151 return AArch64::STRWui;
3152 case AArch64::LDURHi:
3154 return AArch64::LDRHui;
3155 case AArch64::STURHi:
3157 return AArch64::STRHui;
3158 case AArch64::LDURHHi:
3160 return AArch64::LDRHHui;
3161 case AArch64::STURHHi:
3163 return AArch64::STRHHui;
3164 case AArch64::LDURSHXi:
3166 return AArch64::LDRSHXui;
3167 case AArch64::LDURSHWi:
3169 return AArch64::LDRSHWui;
3170 case AArch64::LDURBi:
3172 return AArch64::LDRBui;
3173 case AArch64::LDURBBi:
3175 return AArch64::LDRBBui;
3176 case AArch64::LDURSBXi:
3178 return AArch64::LDRSBXui;
3179 case AArch64::LDURSBWi:
3181 return AArch64::LDRSBWui;
3182 case AArch64::STURBi:
3184 return AArch64::STRBui;
3185 case AArch64::STURBBi:
3187 return AArch64::STRBBui;
3188 case AArch64::LDRQui:
3189 case AArch64::STRQui:
3192 case AArch64::LDRDui:
3193 case AArch64::STRDui:
3194 case AArch64::LDRXui:
3195 case AArch64::STRXui:
3198 case AArch64::LDRWui:
3199 case AArch64::LDRSWui:
3200 case AArch64::STRWui:
3203 case AArch64::LDRHui:
3204 case AArch64::STRHui:
3205 case AArch64::LDRHHui:
3206 case AArch64::STRHHui:
3207 case AArch64::LDRSHXui:
3208 case AArch64::LDRSHWui:
3211 case AArch64::LDRBui:
3212 case AArch64::LDRBBui:
3213 case AArch64::LDRSBXui:
3214 case AArch64::LDRSBWui:
3215 case AArch64::STRBui:
3216 case AArch64::STRBBui:
3230 case AArch64::LDURQi:
3231 case AArch64::STURQi:
3232 case AArch64::LDURDi:
3233 case AArch64::STURDi:
3234 case AArch64::LDURXi:
3235 case AArch64::STURXi:
3236 case AArch64::LDURWi:
3237 case AArch64::LDURSWi:
3238 case AArch64::STURWi:
3239 case AArch64::LDURHi:
3240 case AArch64::STURHi:
3241 case AArch64::LDURHHi:
3242 case AArch64::STURHHi:
3243 case AArch64::LDURSHXi:
3244 case AArch64::LDURSHWi:
3245 case AArch64::LDURBi:
3246 case AArch64::STURBi:
3247 case AArch64::LDURBBi:
3248 case AArch64::STURBBi:
3249 case AArch64::LDURSBWi:
3250 case AArch64::LDURSBXi:
3252 case AArch64::LDRQui:
3253 return AArch64::LDURQi;
3254 case AArch64::STRQui:
3255 return AArch64::STURQi;
3256 case AArch64::LDRDui:
3257 return AArch64::LDURDi;
3258 case AArch64::STRDui:
3259 return AArch64::STURDi;
3260 case AArch64::LDRXui:
3261 return AArch64::LDURXi;
3262 case AArch64::STRXui:
3263 return AArch64::STURXi;
3264 case AArch64::LDRWui:
3265 return AArch64::LDURWi;
3266 case AArch64::LDRSWui:
3267 return AArch64::LDURSWi;
3268 case AArch64::STRWui:
3269 return AArch64::STURWi;
3270 case AArch64::LDRHui:
3271 return AArch64::LDURHi;
3272 case AArch64::STRHui:
3273 return AArch64::STURHi;
3274 case AArch64::LDRHHui:
3275 return AArch64::LDURHHi;
3276 case AArch64::STRHHui:
3277 return AArch64::STURHHi;
3278 case AArch64::LDRSHXui:
3279 return AArch64::LDURSHXi;
3280 case AArch64::LDRSHWui:
3281 return AArch64::LDURSHWi;
3282 case AArch64::LDRBBui:
3283 return AArch64::LDURBBi;
3284 case AArch64::LDRBui:
3285 return AArch64::LDURBi;
3286 case AArch64::STRBBui:
3287 return AArch64::STURBBi;
3288 case AArch64::STRBui:
3289 return AArch64::STURBi;
3290 case AArch64::LDRSBWui:
3291 return AArch64::LDURSBWi;
3292 case AArch64::LDRSBXui:
3293 return AArch64::LDURSBXi;
3306 case AArch64::LDRQroX:
3307 case AArch64::LDURQi:
3308 case AArch64::LDRQui:
3309 return AArch64::LDRQroW;
3310 case AArch64::STRQroX:
3311 case AArch64::STURQi:
3312 case AArch64::STRQui:
3313 return AArch64::STRQroW;
3314 case AArch64::LDRDroX:
3315 case AArch64::LDURDi:
3316 case AArch64::LDRDui:
3317 return AArch64::LDRDroW;
3318 case AArch64::STRDroX:
3319 case AArch64::STURDi:
3320 case AArch64::STRDui:
3321 return AArch64::STRDroW;
3322 case AArch64::LDRXroX:
3323 case AArch64::LDURXi:
3324 case AArch64::LDRXui:
3325 return AArch64::LDRXroW;
3326 case AArch64::STRXroX:
3327 case AArch64::STURXi:
3328 case AArch64::STRXui:
3329 return AArch64::STRXroW;
3330 case AArch64::LDRWroX:
3331 case AArch64::LDURWi:
3332 case AArch64::LDRWui:
3333 return AArch64::LDRWroW;
3334 case AArch64::LDRSWroX:
3335 case AArch64::LDURSWi:
3336 case AArch64::LDRSWui:
3337 return AArch64::LDRSWroW;
3338 case AArch64::STRWroX:
3339 case AArch64::STURWi:
3340 case AArch64::STRWui:
3341 return AArch64::STRWroW;
3342 case AArch64::LDRHroX:
3343 case AArch64::LDURHi:
3344 case AArch64::LDRHui:
3345 return AArch64::LDRHroW;
3346 case AArch64::STRHroX:
3347 case AArch64::STURHi:
3348 case AArch64::STRHui:
3349 return AArch64::STRHroW;
3350 case AArch64::LDRHHroX:
3351 case AArch64::LDURHHi:
3352 case AArch64::LDRHHui:
3353 return AArch64::LDRHHroW;
3354 case AArch64::STRHHroX:
3355 case AArch64::STURHHi:
3356 case AArch64::STRHHui:
3357 return AArch64::STRHHroW;
3358 case AArch64::LDRSHXroX:
3359 case AArch64::LDURSHXi:
3360 case AArch64::LDRSHXui:
3361 return AArch64::LDRSHXroW;
3362 case AArch64::LDRSHWroX:
3363 case AArch64::LDURSHWi:
3364 case AArch64::LDRSHWui:
3365 return AArch64::LDRSHWroW;
3366 case AArch64::LDRBroX:
3367 case AArch64::LDURBi:
3368 case AArch64::LDRBui:
3369 return AArch64::LDRBroW;
3370 case AArch64::LDRBBroX:
3371 case AArch64::LDURBBi:
3372 case AArch64::LDRBBui:
3373 return AArch64::LDRBBroW;
3374 case AArch64::LDRSBXroX:
3375 case AArch64::LDURSBXi:
3376 case AArch64::LDRSBXui:
3377 return AArch64::LDRSBXroW;
3378 case AArch64::LDRSBWroX:
3379 case AArch64::LDURSBWi:
3380 case AArch64::LDRSBWui:
3381 return AArch64::LDRSBWroW;
3382 case AArch64::STRBroX:
3383 case AArch64::STURBi:
3384 case AArch64::STRBui:
3385 return AArch64::STRBroW;
3386 case AArch64::STRBBroX:
3387 case AArch64::STURBBi:
3388 case AArch64::STRBBui:
3389 return AArch64::STRBBroW;
3404 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3414 return B.getInstr();
3418 "Addressing mode not supported for folding");
3435 return B.getInstr();
3442 "Address offset can be a register or an immediate, but not both");
3444 MRI.constrainRegClass(AM.
BaseReg, &AArch64::GPR64spRegClass);
3449 OffsetReg =
MRI.createVirtualRegister(&AArch64::GPR32RegClass);
3463 return B.getInstr();
3467 "Function must not be called with an addressing mode it can't handle");
3472 bool &OffsetIsScalable,
TypeSize &Width,
3493 int64_t Dummy1, Dummy2;
3512 if (!BaseOp->
isReg() && !BaseOp->
isFI())
3522 assert(OfsOp.
isImm() &&
"Offset operand wasn't immediate.");
3527 TypeSize &Width, int64_t &MinOffset,
3528 int64_t &MaxOffset) {
3534 MinOffset = MaxOffset = 0;
3536 case AArch64::STRWpost:
3537 case AArch64::LDRWpost:
3543 case AArch64::LDURQi:
3544 case AArch64::STURQi:
3550 case AArch64::PRFUMi:
3551 case AArch64::LDURXi:
3552 case AArch64::LDURDi:
3553 case AArch64::LDAPURXi:
3554 case AArch64::STURXi:
3555 case AArch64::STURDi:
3556 case AArch64::STLURXi:
3562 case AArch64::LDURWi:
3563 case AArch64::LDURSi:
3564 case AArch64::LDURSWi:
3565 case AArch64::LDAPURi:
3566 case AArch64::LDAPURSWi:
3567 case AArch64::STURWi:
3568 case AArch64::STURSi:
3569 case AArch64::STLURWi:
3575 case AArch64::LDURHi:
3576 case AArch64::LDURHHi:
3577 case AArch64::LDURSHXi:
3578 case AArch64::LDURSHWi:
3579 case AArch64::LDAPURHi:
3580 case AArch64::LDAPURSHWi:
3581 case AArch64::LDAPURSHXi:
3582 case AArch64::STURHi:
3583 case AArch64::STURHHi:
3584 case AArch64::STLURHi:
3590 case AArch64::LDURBi:
3591 case AArch64::LDURBBi:
3592 case AArch64::LDURSBXi:
3593 case AArch64::LDURSBWi:
3594 case AArch64::LDAPURBi:
3595 case AArch64::LDAPURSBWi:
3596 case AArch64::LDAPURSBXi:
3597 case AArch64::STURBi:
3598 case AArch64::STURBBi:
3599 case AArch64::STLURBi:
3605 case AArch64::LDPQi:
3606 case AArch64::LDNPQi:
3607 case AArch64::STPQi:
3608 case AArch64::STNPQi:
3614 case AArch64::LDRQui:
3615 case AArch64::STRQui:
3621 case AArch64::LDPXi:
3622 case AArch64::LDPDi:
3623 case AArch64::LDNPXi:
3624 case AArch64::LDNPDi:
3625 case AArch64::STPXi:
3626 case AArch64::STPDi:
3627 case AArch64::STNPXi:
3628 case AArch64::STNPDi:
3634 case AArch64::PRFMui:
3635 case AArch64::LDRXui:
3636 case AArch64::LDRDui:
3637 case AArch64::STRXui:
3638 case AArch64::STRDui:
3644 case AArch64::StoreSwiftAsyncContext:
3651 case AArch64::LDPWi:
3652 case AArch64::LDPSi:
3653 case AArch64::LDNPWi:
3654 case AArch64::LDNPSi:
3655 case AArch64::STPWi:
3656 case AArch64::STPSi:
3657 case AArch64::STNPWi:
3658 case AArch64::STNPSi:
3664 case AArch64::LDRWui:
3665 case AArch64::LDRSui:
3666 case AArch64::LDRSWui:
3667 case AArch64::STRWui:
3668 case AArch64::STRSui:
3674 case AArch64::LDRHui:
3675 case AArch64::LDRHHui:
3676 case AArch64::LDRSHWui:
3677 case AArch64::LDRSHXui:
3678 case AArch64::STRHui:
3679 case AArch64::STRHHui:
3685 case AArch64::LDRBui:
3686 case AArch64::LDRBBui:
3687 case AArch64::LDRSBWui:
3688 case AArch64::LDRSBXui:
3689 case AArch64::STRBui:
3690 case AArch64::STRBBui:
3696 case AArch64::STPXpre:
3697 case AArch64::LDPXpost:
3698 case AArch64::STPDpre:
3699 case AArch64::LDPDpost:
3705 case AArch64::STPQpre:
3706 case AArch64::LDPQpost:
3712 case AArch64::STRXpre:
3713 case AArch64::STRDpre:
3714 case AArch64::LDRXpost:
3715 case AArch64::LDRDpost:
3721 case AArch64::STRQpre:
3722 case AArch64::LDRQpost:
3734 case AArch64::TAGPstack:
3744 case AArch64::STZGi:
3750 case AArch64::STR_ZZZZXI:
3751 case AArch64::LDR_ZZZZXI:
3757 case AArch64::STR_ZZZXI:
3758 case AArch64::LDR_ZZZXI:
3764 case AArch64::STR_ZZXI:
3765 case AArch64::LDR_ZZXI:
3771 case AArch64::LDR_PXI:
3772 case AArch64::STR_PXI:
3778 case AArch64::LDR_PPXI:
3779 case AArch64::STR_PPXI:
3785 case AArch64::LDR_ZXI:
3786 case AArch64::STR_ZXI:
3792 case AArch64::LD1B_IMM:
3793 case AArch64::LD1H_IMM:
3794 case AArch64::LD1W_IMM:
3795 case AArch64::LD1D_IMM:
3796 case AArch64::LDNT1B_ZRI:
3797 case AArch64::LDNT1H_ZRI:
3798 case AArch64::LDNT1W_ZRI:
3799 case AArch64::LDNT1D_ZRI:
3800 case AArch64::ST1B_IMM:
3801 case AArch64::ST1H_IMM:
3802 case AArch64::ST1W_IMM:
3803 case AArch64::ST1D_IMM:
3804 case AArch64::STNT1B_ZRI:
3805 case AArch64::STNT1H_ZRI:
3806 case AArch64::STNT1W_ZRI:
3807 case AArch64::STNT1D_ZRI:
3808 case AArch64::LDNF1B_IMM:
3809 case AArch64::LDNF1H_IMM:
3810 case AArch64::LDNF1W_IMM:
3811 case AArch64::LDNF1D_IMM:
3819 case AArch64::LD2B_IMM:
3820 case AArch64::LD2H_IMM:
3821 case AArch64::LD2W_IMM:
3822 case AArch64::LD2D_IMM:
3823 case AArch64::ST2B_IMM:
3824 case AArch64::ST2H_IMM:
3825 case AArch64::ST2W_IMM:
3826 case AArch64::ST2D_IMM:
3832 case AArch64::LD3B_IMM:
3833 case AArch64::LD3H_IMM:
3834 case AArch64::LD3W_IMM:
3835 case AArch64::LD3D_IMM:
3836 case AArch64::ST3B_IMM:
3837 case AArch64::ST3H_IMM:
3838 case AArch64::ST3W_IMM:
3839 case AArch64::ST3D_IMM:
3845 case AArch64::LD4B_IMM:
3846 case AArch64::LD4H_IMM:
3847 case AArch64::LD4W_IMM:
3848 case AArch64::LD4D_IMM:
3849 case AArch64::ST4B_IMM:
3850 case AArch64::ST4H_IMM:
3851 case AArch64::ST4W_IMM:
3852 case AArch64::ST4D_IMM:
3858 case AArch64::LD1B_H_IMM:
3859 case AArch64::LD1SB_H_IMM:
3860 case AArch64::LD1H_S_IMM:
3861 case AArch64::LD1SH_S_IMM:
3862 case AArch64::LD1W_D_IMM:
3863 case AArch64::LD1SW_D_IMM:
3864 case AArch64::ST1B_H_IMM:
3865 case AArch64::ST1H_S_IMM:
3866 case AArch64::ST1W_D_IMM:
3867 case AArch64::LDNF1B_H_IMM:
3868 case AArch64::LDNF1SB_H_IMM:
3869 case AArch64::LDNF1H_S_IMM:
3870 case AArch64::LDNF1SH_S_IMM:
3871 case AArch64::LDNF1W_D_IMM:
3872 case AArch64::LDNF1SW_D_IMM:
3880 case AArch64::LD1B_S_IMM:
3881 case AArch64::LD1SB_S_IMM:
3882 case AArch64::LD1H_D_IMM:
3883 case AArch64::LD1SH_D_IMM:
3884 case AArch64::ST1B_S_IMM:
3885 case AArch64::ST1H_D_IMM:
3886 case AArch64::LDNF1B_S_IMM:
3887 case AArch64::LDNF1SB_S_IMM:
3888 case AArch64::LDNF1H_D_IMM:
3889 case AArch64::LDNF1SH_D_IMM:
3897 case AArch64::LD1B_D_IMM:
3898 case AArch64::LD1SB_D_IMM:
3899 case AArch64::ST1B_D_IMM:
3900 case AArch64::LDNF1B_D_IMM:
3901 case AArch64::LDNF1SB_D_IMM:
3909 case AArch64::ST2Gi:
3910 case AArch64::STZ2Gi:
3916 case AArch64::STGPi:
3922 case AArch64::LD1RB_IMM:
3923 case AArch64::LD1RB_H_IMM:
3924 case AArch64::LD1RB_S_IMM:
3925 case AArch64::LD1RB_D_IMM:
3926 case AArch64::LD1RSB_H_IMM:
3927 case AArch64::LD1RSB_S_IMM:
3928 case AArch64::LD1RSB_D_IMM:
3934 case AArch64::LD1RH_IMM:
3935 case AArch64::LD1RH_S_IMM:
3936 case AArch64::LD1RH_D_IMM:
3937 case AArch64::LD1RSH_S_IMM:
3938 case AArch64::LD1RSH_D_IMM:
3944 case AArch64::LD1RW_IMM:
3945 case AArch64::LD1RW_D_IMM:
3946 case AArch64::LD1RSW_IMM:
3952 case AArch64::LD1RD_IMM:
3968 case AArch64::LDRBBui:
3969 case AArch64::LDURBBi:
3970 case AArch64::LDRSBWui:
3971 case AArch64::LDURSBWi:
3972 case AArch64::STRBBui:
3973 case AArch64::STURBBi:
3975 case AArch64::LDRHHui:
3976 case AArch64::LDURHHi:
3977 case AArch64::LDRSHWui:
3978 case AArch64::LDURSHWi:
3979 case AArch64::STRHHui:
3980 case AArch64::STURHHi:
3982 case AArch64::LDRSui:
3983 case AArch64::LDURSi:
3984 case AArch64::LDRSpre:
3985 case AArch64::LDRSWui:
3986 case AArch64::LDURSWi:
3987 case AArch64::LDRSWpre:
3988 case AArch64::LDRWpre:
3989 case AArch64::LDRWui:
3990 case AArch64::LDURWi:
3991 case AArch64::STRSui:
3992 case AArch64::STURSi:
3993 case AArch64::STRSpre:
3994 case AArch64::STRWui:
3995 case AArch64::STURWi:
3996 case AArch64::STRWpre:
3997 case AArch64::LDPSi:
3998 case AArch64::LDPSWi:
3999 case AArch64::LDPWi:
4000 case AArch64::STPSi:
4001 case AArch64::STPWi:
4003 case AArch64::LDRDui:
4004 case AArch64::LDURDi:
4005 case AArch64::LDRDpre:
4006 case AArch64::LDRXui:
4007 case AArch64::LDURXi:
4008 case AArch64::LDRXpre:
4009 case AArch64::STRDui:
4010 case AArch64::STURDi:
4011 case AArch64::STRDpre:
4012 case AArch64::STRXui:
4013 case AArch64::STURXi:
4014 case AArch64::STRXpre:
4015 case AArch64::LDPDi:
4016 case AArch64::LDPXi:
4017 case AArch64::STPDi:
4018 case AArch64::STPXi:
4020 case AArch64::LDRQui:
4021 case AArch64::LDURQi:
4022 case AArch64::STRQui:
4023 case AArch64::STURQi:
4024 case AArch64::STRQpre:
4025 case AArch64::LDPQi:
4026 case AArch64::LDRQpre:
4027 case AArch64::STPQi:
4029 case AArch64::STZGi:
4030 case AArch64::ST2Gi:
4031 case AArch64::STZ2Gi:
4032 case AArch64::STGPi:
4038 switch (
MI.getOpcode()) {
4041 case AArch64::LDRWpre:
4042 case AArch64::LDRXpre:
4043 case AArch64::LDRSWpre:
4044 case AArch64::LDRSpre:
4045 case AArch64::LDRDpre:
4046 case AArch64::LDRQpre:
4052 switch (
MI.getOpcode()) {
4055 case AArch64::STRWpre:
4056 case AArch64::STRXpre:
4057 case AArch64::STRSpre:
4058 case AArch64::STRDpre:
4059 case AArch64::STRQpre:
4069 switch (
MI.getOpcode()) {
4072 case AArch64::LDPSi:
4073 case AArch64::LDPSWi:
4074 case AArch64::LDPDi:
4075 case AArch64::LDPQi:
4076 case AArch64::LDPWi:
4077 case AArch64::LDPXi:
4078 case AArch64::STPSi:
4079 case AArch64::STPDi:
4080 case AArch64::STPQi:
4081 case AArch64::STPWi:
4082 case AArch64::STPXi:
4083 case AArch64::STGPi:
4092 return MI.getOperand(
Idx);
4100 return MI.getOperand(
Idx);
4105 if (
MI.getParent() ==
nullptr)
4115 auto Reg =
Op.getReg();
4116 if (Reg.isPhysical())
4117 return AArch64::FPR16RegClass.
contains(Reg);
4119 return TRC == &AArch64::FPR16RegClass ||
4120 TRC == &AArch64::FPR16_loRegClass;
4129 auto Reg =
Op.getReg();
4130 if (Reg.isPhysical())
4131 return AArch64::FPR128RegClass.
contains(Reg);
4133 return TRC == &AArch64::FPR128RegClass ||
4134 TRC == &AArch64::FPR128_loRegClass;
4140 switch (
MI.getOpcode()) {
4143 case AArch64::PACIASP:
4144 case AArch64::PACIBSP:
4147 case AArch64::PAUTH_PROLOGUE:
4150 case AArch64::HINT: {
4151 unsigned Imm =
MI.getOperand(0).getImm();
4153 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
4156 if (Imm == 25 || Imm == 27)
4169 auto Reg =
Op.getReg();
4170 if (Reg.isPhysical())
4171 return AArch64::FPR128RegClass.
contains(Reg) ||
4172 AArch64::FPR64RegClass.contains(Reg) ||
4173 AArch64::FPR32RegClass.contains(Reg) ||
4174 AArch64::FPR16RegClass.contains(Reg) ||
4175 AArch64::FPR8RegClass.contains(Reg);
4178 return TRC == &AArch64::FPR128RegClass ||
4179 TRC == &AArch64::FPR128_loRegClass ||
4180 TRC == &AArch64::FPR64RegClass ||
4181 TRC == &AArch64::FPR64_loRegClass ||
4182 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
4183 TRC == &AArch64::FPR8RegClass;
4205 if (FirstOpc == SecondOpc)
4211 case AArch64::STRSui:
4212 case AArch64::STURSi:
4213 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
4214 case AArch64::STRDui:
4215 case AArch64::STURDi:
4216 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
4217 case AArch64::STRQui:
4218 case AArch64::STURQi:
4219 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
4220 case AArch64::STRWui:
4221 case AArch64::STURWi:
4222 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
4223 case AArch64::STRXui:
4224 case AArch64::STURXi:
4225 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
4226 case AArch64::LDRSui:
4227 case AArch64::LDURSi:
4228 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
4229 case AArch64::LDRDui:
4230 case AArch64::LDURDi:
4231 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
4232 case AArch64::LDRQui:
4233 case AArch64::LDURQi:
4234 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
4235 case AArch64::LDRWui:
4236 case AArch64::LDURWi:
4237 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
4238 case AArch64::LDRSWui:
4239 case AArch64::LDURSWi:
4240 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
4241 case AArch64::LDRXui:
4242 case AArch64::LDURXi:
4243 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
4250 int64_t Offset1,
unsigned Opcode1,
int FI2,
4251 int64_t Offset2,
unsigned Opcode2) {
4257 assert(ObjectOffset1 <= ObjectOffset2 &&
"Object offsets are not ordered.");
4260 if (ObjectOffset1 % Scale1 != 0)
4262 ObjectOffset1 /= Scale1;
4264 if (ObjectOffset2 % Scale2 != 0)
4266 ObjectOffset2 /= Scale2;
4267 ObjectOffset1 += Offset1;
4268 ObjectOffset2 += Offset2;
4269 return ObjectOffset1 + 1 == ObjectOffset2;
4281 int64_t OpOffset2,
bool OffsetIsScalable2,
unsigned ClusterSize,
4282 unsigned NumBytes)
const {
4292 "Only base registers and frame indices are supported.");
4299 if (ClusterSize > 2)
4306 unsigned FirstOpc = FirstLdSt.
getOpcode();
4307 unsigned SecondOpc = SecondLdSt.
getOpcode();
4327 if (Offset1 > 63 || Offset1 < -64)
4332 if (BaseOp1.
isFI()) {
4334 "Caller should have ordered offsets.");
4339 BaseOp2.
getIndex(), Offset2, SecondOpc);
4342 assert(Offset1 <= Offset2 &&
"Caller should have ordered offsets.");
4344 return Offset1 + 1 == Offset2;
4348 unsigned Reg,
unsigned SubIdx,
4352 return MIB.
addReg(Reg, State);
4355 return MIB.
addReg(
TRI->getSubReg(Reg, SubIdx), State);
4356 return MIB.
addReg(Reg, State, SubIdx);
4363 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
4372 assert(Subtarget.hasNEON() &&
"Unexpected register copy without NEON");
4374 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4375 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4376 unsigned NumRegs = Indices.
size();
4378 int SubReg = 0,
End = NumRegs, Incr = 1;
4396 unsigned SrcReg,
bool KillSrc,
4397 unsigned Opcode,
unsigned ZeroReg,
4400 unsigned NumRegs = Indices.
size();
4403 uint16_t DestEncoding =
TRI->getEncodingValue(DestReg);
4404 uint16_t SrcEncoding =
TRI->getEncodingValue(SrcReg);
4405 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
4406 "GPR reg sequences should not be able to overlap");
4422 if (AArch64::GPR32spRegClass.
contains(DestReg) &&
4423 (AArch64::GPR32spRegClass.
contains(SrcReg) || SrcReg == AArch64::WZR)) {
4426 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
4428 if (Subtarget.hasZeroCycleRegMove()) {
4431 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4433 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4449 }
else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
4454 if (Subtarget.hasZeroCycleRegMove()) {
4457 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4459 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
4479 if (AArch64::PPRRegClass.
contains(DestReg) &&
4480 AArch64::PPRRegClass.
contains(SrcReg)) {
4491 bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
4492 bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
4493 if (DestIsPNR || SrcIsPNR) {
4494 assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4495 "Unexpected predicate-as-counter register.");
4497 return (R - AArch64::PN0) + AArch64::P0;
4499 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg;
4500 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg;
4502 if (PPRSrcReg != PPRDestReg) {
4514 if (AArch64::ZPRRegClass.
contains(DestReg) &&
4515 AArch64::ZPRRegClass.
contains(SrcReg)) {
4524 if ((AArch64::ZPR2RegClass.
contains(DestReg) ||
4525 AArch64::ZPR2StridedOrContiguousRegClass.
contains(DestReg)) &&
4526 (AArch64::ZPR2RegClass.
contains(SrcReg) ||
4527 AArch64::ZPR2StridedOrContiguousRegClass.
contains(SrcReg))) {
4529 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
4536 if (AArch64::ZPR3RegClass.
contains(DestReg) &&
4537 AArch64::ZPR3RegClass.
contains(SrcReg)) {
4539 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4547 if ((AArch64::ZPR4RegClass.
contains(DestReg) ||
4548 AArch64::ZPR4StridedOrContiguousRegClass.
contains(DestReg)) &&
4549 (AArch64::ZPR4RegClass.
contains(SrcReg) ||
4550 AArch64::ZPR4StridedOrContiguousRegClass.
contains(SrcReg))) {
4552 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
4553 AArch64::zsub2, AArch64::zsub3};
4559 if (AArch64::GPR64spRegClass.
contains(DestReg) &&
4560 (AArch64::GPR64spRegClass.
contains(SrcReg) || SrcReg == AArch64::XZR)) {
4561 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
4567 }
else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
4581 if (AArch64::DDDDRegClass.
contains(DestReg) &&
4582 AArch64::DDDDRegClass.
contains(SrcReg)) {
4583 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
4584 AArch64::dsub2, AArch64::dsub3};
4591 if (AArch64::DDDRegClass.
contains(DestReg) &&
4592 AArch64::DDDRegClass.
contains(SrcReg)) {
4593 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
4601 if (AArch64::DDRegClass.
contains(DestReg) &&
4602 AArch64::DDRegClass.
contains(SrcReg)) {
4603 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
4610 if (AArch64::QQQQRegClass.
contains(DestReg) &&
4611 AArch64::QQQQRegClass.
contains(SrcReg)) {
4612 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
4613 AArch64::qsub2, AArch64::qsub3};
4620 if (AArch64::QQQRegClass.
contains(DestReg) &&
4621 AArch64::QQQRegClass.
contains(SrcReg)) {
4622 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
4630 if (AArch64::QQRegClass.
contains(DestReg) &&
4631 AArch64::QQRegClass.
contains(SrcReg)) {
4632 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
4638 if (AArch64::XSeqPairsClassRegClass.
contains(DestReg) &&
4639 AArch64::XSeqPairsClassRegClass.
contains(SrcReg)) {
4640 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
4642 AArch64::XZR, Indices);
4646 if (AArch64::WSeqPairsClassRegClass.
contains(DestReg) &&
4647 AArch64::WSeqPairsClassRegClass.
contains(SrcReg)) {
4648 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
4650 AArch64::WZR, Indices);
4654 if (AArch64::FPR128RegClass.
contains(DestReg) &&
4655 AArch64::FPR128RegClass.
contains(SrcReg)) {
4659 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
4660 .
addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
4661 else if (Subtarget.hasNEON())
4680 if (AArch64::FPR64RegClass.
contains(DestReg) &&
4681 AArch64::FPR64RegClass.
contains(SrcReg)) {
4687 if (AArch64::FPR32RegClass.
contains(DestReg) &&
4688 AArch64::FPR32RegClass.
contains(SrcReg)) {
4694 if (AArch64::FPR16RegClass.
contains(DestReg) &&
4695 AArch64::FPR16RegClass.
contains(SrcReg)) {
4697 RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
4699 RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
4705 if (AArch64::FPR8RegClass.
contains(DestReg) &&
4706 AArch64::FPR8RegClass.
contains(SrcReg)) {
4708 RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
4710 RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
4717 if (AArch64::FPR64RegClass.
contains(DestReg) &&
4718 AArch64::GPR64RegClass.
contains(SrcReg)) {
4723 if (AArch64::GPR64RegClass.
contains(DestReg) &&
4724 AArch64::FPR64RegClass.
contains(SrcReg)) {
4730 if (AArch64::FPR32RegClass.
contains(DestReg) &&
4731 AArch64::GPR32RegClass.
contains(SrcReg)) {
4736 if (AArch64::GPR32RegClass.
contains(DestReg) &&
4737 AArch64::FPR32RegClass.
contains(SrcReg)) {
4743 if (DestReg == AArch64::NZCV) {
4744 assert(AArch64::GPR64RegClass.
contains(SrcReg) &&
"Invalid NZCV copy");
4746 .
addImm(AArch64SysReg::NZCV)
4752 if (SrcReg == AArch64::NZCV) {
4753 assert(AArch64::GPR64RegClass.
contains(DestReg) &&
"Invalid NZCV copy");
4755 .
addImm(AArch64SysReg::NZCV)
4762 errs() <<
TRI.getRegAsmName(DestReg) <<
" = COPY "
4763 <<
TRI.getRegAsmName(SrcReg) <<
"\n";
4773 unsigned SubIdx0,
unsigned SubIdx1,
int FI,
4778 SrcReg0 =
TRI.getSubReg(SrcReg, SubIdx0);
4780 SrcReg1 =
TRI.getSubReg(SrcReg, SubIdx1);
4793 Register SrcReg,
bool isKill,
int FI,
4808 switch (
TRI->getSpillSize(*RC)) {
4810 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
4811 Opc = AArch64::STRBui;
4814 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
4815 Opc = AArch64::STRHui;
4816 else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
4818 "Unexpected register store without SVE store instructions");
4819 Opc = AArch64::STR_PXI;
4821 }
else if (AArch64::PNRRegClass.hasSubClassEq(RC)) {
4822 assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4823 "Unexpected register store without SVE2p1 or SME2");
4831 SrcReg = (SrcReg - AArch64::PN0) + AArch64::P0;
4832 Opc = AArch64::STR_PXI;
4837 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
4838 Opc = AArch64::STRWui;
4842 assert(SrcReg != AArch64::WSP);
4843 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
4844 Opc = AArch64::STRSui;
4845 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
4846 Opc = AArch64::STR_PPXI;
4851 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
4852 Opc = AArch64::STRXui;
4856 assert(SrcReg != AArch64::SP);
4857 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
4858 Opc = AArch64::STRDui;
4859 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
4861 get(AArch64::STPWi), SrcReg, isKill,
4862 AArch64::sube32, AArch64::subo32, FI, MMO);
4867 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
4868 Opc = AArch64::STRQui;
4869 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
4870 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4871 Opc = AArch64::ST1Twov1d;
4873 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
4875 get(AArch64::STPXi), SrcReg, isKill,
4876 AArch64::sube64, AArch64::subo64, FI, MMO);
4878 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
4880 "Unexpected register store without SVE store instructions");
4881 Opc = AArch64::STR_ZXI;
4886 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
4887 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4888 Opc = AArch64::ST1Threev1d;
4893 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
4894 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4895 Opc = AArch64::ST1Fourv1d;
4897 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
4898 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4899 Opc = AArch64::ST1Twov2d;
4901 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
4902 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4904 "Unexpected register store without SVE store instructions");
4905 Opc = AArch64::STR_ZZXI;
4910 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
4911 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4912 Opc = AArch64::ST1Threev2d;
4914 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
4916 "Unexpected register store without SVE store instructions");
4917 Opc = AArch64::STR_ZZZXI;
4922 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
4923 assert(Subtarget.hasNEON() &&
"Unexpected register store without NEON");
4924 Opc = AArch64::ST1Fourv2d;
4926 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
4927 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4929 "Unexpected register store without SVE store instructions");
4930 Opc = AArch64::STR_ZZZZXI;
4935 assert(Opc &&
"Unknown register class");
4946 MI.addMemOperand(MMO);
4953 Register DestReg,
unsigned SubIdx0,
4954 unsigned SubIdx1,
int FI,
4958 bool IsUndef =
true;
4960 DestReg0 =
TRI.getSubReg(DestReg, SubIdx0);
4962 DestReg1 =
TRI.getSubReg(DestReg, SubIdx1);
4991 switch (
TRI->getSpillSize(*RC)) {
4993 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
4994 Opc = AArch64::LDRBui;
4997 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
4998 Opc = AArch64::LDRHui;
4999 else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
5001 "Unexpected register load without SVE load instructions");
5002 Opc = AArch64::LDR_PXI;
5004 }
else if (AArch64::PNRRegClass.hasSubClassEq(RC)) {
5005 assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
5006 "Unexpected register load without SVE2p1 or SME2");
5011 DestReg = (DestReg - AArch64::PN0) + AArch64::P0;
5012 Opc = AArch64::LDR_PXI;
5017 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
5018 Opc = AArch64::LDRWui;
5022 assert(DestReg != AArch64::WSP);
5023 }
else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
5024 Opc = AArch64::LDRSui;
5025 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
5026 Opc = AArch64::LDR_PPXI;
5031 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
5032 Opc = AArch64::LDRXui;
5036 assert(DestReg != AArch64::SP);
5037 }
else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
5038 Opc = AArch64::LDRDui;
5039 }
else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
5041 get(AArch64::LDPWi), DestReg, AArch64::sube32,
5042 AArch64::subo32, FI, MMO);
5047 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
5048 Opc = AArch64::LDRQui;
5049 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
5050 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5051 Opc = AArch64::LD1Twov1d;
5053 }
else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
5055 get(AArch64::LDPXi), DestReg, AArch64::sube64,
5056 AArch64::subo64, FI, MMO);
5058 }
else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5060 "Unexpected register load without SVE load instructions");
5061 Opc = AArch64::LDR_ZXI;
5066 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
5067 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5068 Opc = AArch64::LD1Threev1d;
5073 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
5074 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5075 Opc = AArch64::LD1Fourv1d;
5077 }
else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
5078 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5079 Opc = AArch64::LD1Twov2d;
5081 }
else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
5082 AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5084 "Unexpected register load without SVE load instructions");
5085 Opc = AArch64::LDR_ZZXI;
5090 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
5091 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5092 Opc = AArch64::LD1Threev2d;
5094 }
else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5096 "Unexpected register load without SVE load instructions");
5097 Opc = AArch64::LDR_ZZZXI;
5102 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
5103 assert(Subtarget.hasNEON() &&
"Unexpected register load without NEON");
5104 Opc = AArch64::LD1Fourv2d;
5106 }
else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
5107 AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5109 "Unexpected register load without SVE load instructions");
5110 Opc = AArch64::LDR_ZZZZXI;
5116 assert(Opc &&
"Unknown register class");
5126 MI.addMemOperand(MMO);
5137 UseMI.getIterator()),
5139 return I.modifiesRegister(AArch64::NZCV, TRI) ||
5140 I.readsRegister(AArch64::NZCV, TRI);
5149 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5156 ByteSized =
Offset.getFixed();
5157 VGSized =
Offset.getScalable() / 2;
5165 int64_t &NumDataVectors) {
5169 assert(
Offset.getScalable() % 2 == 0 &&
"Invalid frame offset");
5171 NumBytes =
Offset.getFixed();
5173 NumPredicateVectors =
Offset.getScalable() / 2;
5178 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
5179 NumPredicateVectors > 62) {
5180 NumDataVectors = NumPredicateVectors / 8;
5181 NumPredicateVectors -= NumDataVectors * 8;
5188 int NumVGScaledBytes,
unsigned VG,
5195 Expr.
push_back((uint8_t)dwarf::DW_OP_plus);
5196 Comment << (NumBytes < 0 ?
" - " :
" + ") << std::abs(NumBytes);
5199 if (NumVGScaledBytes) {
5200 Expr.
push_back((uint8_t)dwarf::DW_OP_consts);
5203 Expr.
push_back((uint8_t)dwarf::DW_OP_bregx);
5207 Expr.
push_back((uint8_t)dwarf::DW_OP_mul);
5208 Expr.
push_back((uint8_t)dwarf::DW_OP_plus);
5210 Comment << (NumVGScaledBytes < 0 ?
" - " :
" + ")
5211 << std::abs(NumVGScaledBytes) <<
" * VG";
5220 int64_t NumBytes, NumVGScaledBytes;
5223 std::string CommentBuffer;
5226 if (Reg == AArch64::SP)
5228 else if (Reg == AArch64::FP)
5235 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5236 Expr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
5239 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5243 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
5252 unsigned FrameReg,
unsigned Reg,
5254 bool LastAdjustmentWasScalable) {
5255 if (
Offset.getScalable())
5258 if (FrameReg == Reg && !LastAdjustmentWasScalable)
5261 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5268 int64_t NumBytes, NumVGScaledBytes;
5270 OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
5272 unsigned DwarfReg =
TRI.getDwarfRegNum(Reg,
true);
5275 if (!NumVGScaledBytes)
5278 std::string CommentBuffer;
5285 TRI.getDwarfRegNum(AArch64::VG,
true), Comment);
5289 CfaExpr.
push_back(dwarf::DW_CFA_expression);
5305 unsigned SrcReg, int64_t
Offset,
unsigned Opc,
5308 bool *HasWinCFI,
bool EmitCFAOffset,
5311 unsigned MaxEncoding, ShiftSize;
5313 case AArch64::ADDXri:
5314 case AArch64::ADDSXri:
5315 case AArch64::SUBXri:
5316 case AArch64::SUBSXri:
5317 MaxEncoding = 0xfff;
5320 case AArch64::ADDVL_XXI:
5321 case AArch64::ADDPL_XXI:
5322 case AArch64::ADDSVL_XXI:
5323 case AArch64::ADDSPL_XXI:
5338 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
5340 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
5354 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
5356 if (TmpReg == AArch64::XZR)
5358 &AArch64::GPR64RegClass);
5360 uint64_t ThisVal = std::min<uint64_t>(
Offset, MaxEncodableValue);
5361 unsigned LocalShiftSize = 0;
5362 if (ThisVal > MaxEncoding) {
5363 ThisVal = ThisVal >> ShiftSize;
5364 LocalShiftSize = ShiftSize;
5366 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
5367 "Encoding cannot handle value that big");
5369 Offset -= ThisVal << LocalShiftSize;
5374 .
addImm(Sign * (
int)ThisVal);
5384 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
5385 CFAOffset += Change;
5387 CFAOffset -= Change;
5388 if (EmitCFAOffset && DestReg == TmpReg) {
5401 assert(Sign == 1 &&
"SEH directives should always have a positive sign");
5402 int Imm = (int)(ThisVal << LocalShiftSize);
5403 if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
5404 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
5413 assert(
Offset == 0 &&
"Expected remaining offset to be zero to "
5414 "emit a single SEH directive");
5415 }
else if (DestReg == AArch64::SP) {
5418 assert(SrcReg == AArch64::SP &&
"Unexpected SrcReg for SEH_StackAlloc");
5431 unsigned DestReg,
unsigned SrcReg,
5434 bool NeedsWinCFI,
bool *HasWinCFI,
5436 unsigned FrameReg) {
5443 bool UseSVL =
F.hasFnAttribute(
"aarch64_pstate_sm_body");
5445 int64_t Bytes, NumPredicateVectors, NumDataVectors;
5447 Offset, Bytes, NumPredicateVectors, NumDataVectors);
5450 if (Bytes || (!
Offset && SrcReg != DestReg)) {
5451 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
5452 "SP increment/decrement not 8-byte aligned");
5453 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
5456 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
5459 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
5461 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
5468 assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
5469 "SetNZCV not supported with SVE vectors");
5470 assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
5471 "WinCFI not supported with SVE vectors");
5473 if (NumDataVectors) {
5475 UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
5476 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5477 CFAOffset, FrameReg);
5482 if (NumPredicateVectors) {
5483 assert(DestReg != AArch64::SP &&
"Unaligned access to SP");
5485 UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
5486 TII, Flag, NeedsWinCFI,
nullptr, EmitCFAOffset,
5487 CFAOffset, FrameReg);
5505 if (
MI.isFullCopy()) {
5508 if (SrcReg == AArch64::SP && DstReg.
isVirtual()) {
5512 if (DstReg == AArch64::SP && SrcReg.
isVirtual()) {
5517 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
5545 if (
MI.isCopy() && Ops.
size() == 1 &&
5547 (Ops[0] == 0 || Ops[0] == 1)) {
5548 bool IsSpill = Ops[0] == 0;
5549 bool IsFill = !IsSpill;
5561 :
TRI.getMinimalPhysRegClass(Reg);
5567 "Mismatched register size in non subreg COPY");
5574 return &*--InsertPt;
5586 if (IsSpill && DstMO.
isUndef() && SrcReg == AArch64::WZR &&
5589 "Unexpected subreg on physical register");
5591 FrameIndex, &AArch64::GPR64RegClass, &
TRI,
5593 return &*--InsertPt;
5611 case AArch64::sub_32:
5612 FillRC = &AArch64::GPR32RegClass;
5615 FillRC = &AArch64::FPR32RegClass;
5618 FillRC = &AArch64::FPR64RegClass;
5624 TRI.getRegSizeInBits(*FillRC) &&
5625 "Mismatched regclass size on folded subreg COPY");
5644 bool *OutUseUnscaledOp,
5645 unsigned *OutUnscaledOp,
5646 int64_t *EmittableOffset) {
5648 if (EmittableOffset)
5649 *EmittableOffset = 0;
5650 if (OutUseUnscaledOp)
5651 *OutUseUnscaledOp =
false;
5657 switch (
MI.getOpcode()) {
5660 case AArch64::LD1Rv1d:
5661 case AArch64::LD1Rv2s:
5662 case AArch64::LD1Rv2d:
5663 case AArch64::LD1Rv4h:
5664 case AArch64::LD1Rv4s:
5665 case AArch64::LD1Rv8b:
5666 case AArch64::LD1Rv8h:
5667 case AArch64::LD1Rv16b:
5668 case AArch64::LD1Twov2d:
5669 case AArch64::LD1Threev2d:
5670 case AArch64::LD1Fourv2d:
5671 case AArch64::LD1Twov1d:
5672 case AArch64::LD1Threev1d:
5673 case AArch64::LD1Fourv1d:
5674 case AArch64::ST1Twov2d:
5675 case AArch64::ST1Threev2d:
5676 case AArch64::ST1Fourv2d:
5677 case AArch64::ST1Twov1d:
5678 case AArch64::ST1Threev1d:
5679 case AArch64::ST1Fourv1d:
5680 case AArch64::ST1i8:
5681 case AArch64::ST1i16:
5682 case AArch64::ST1i32:
5683 case AArch64::ST1i64:
5685 case AArch64::IRGstack:
5686 case AArch64::STGloop:
5687 case AArch64::STZGloop:
5692 TypeSize ScaleValue(0U,
false), Width(0U,
false);
5693 int64_t MinOff, MaxOff;
5699 bool IsMulVL = ScaleValue.isScalable();
5700 unsigned Scale = ScaleValue.getKnownMinValue();
5710 std::optional<unsigned> UnscaledOp =
5712 bool useUnscaledOp = UnscaledOp && (
Offset % Scale ||
Offset < 0);
5713 if (useUnscaledOp &&
5718 Scale = ScaleValue.getKnownMinValue();
5719 assert(IsMulVL == ScaleValue.isScalable() &&
5720 "Unscaled opcode has different value for scalable");
5722 int64_t Remainder =
Offset % Scale;
5723 assert(!(Remainder && useUnscaledOp) &&
5724 "Cannot have remainder when using unscaled op");
5726 assert(MinOff < MaxOff &&
"Unexpected Min/Max offsets");
5727 int64_t NewOffset =
Offset / Scale;
5728 if (MinOff <= NewOffset && NewOffset <= MaxOff)
5731 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
5735 if (EmittableOffset)
5736 *EmittableOffset = NewOffset;
5737 if (OutUseUnscaledOp)
5738 *OutUseUnscaledOp = useUnscaledOp;
5739 if (OutUnscaledOp && UnscaledOp)
5740 *OutUnscaledOp = *UnscaledOp;
5753 unsigned Opcode =
MI.getOpcode();
5754 unsigned ImmIdx = FrameRegIdx + 1;
5756 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
5761 MI.eraseFromParent();
5767 unsigned UnscaledOp;
5770 &UnscaledOp, &NewOffset);
5774 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg,
false);
5776 MI.setDesc(
TII->get(UnscaledOp));
5778 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
5801 case AArch64::ADDSWrr:
5802 case AArch64::ADDSWri:
5803 case AArch64::ADDSXrr:
5804 case AArch64::ADDSXri:
5805 case AArch64::SUBSWrr:
5806 case AArch64::SUBSXrr:
5808 case AArch64::SUBSWri:
5809 case AArch64::SUBSXri:
5820 case AArch64::ADDWrr:
5821 case AArch64::ADDWri:
5822 case AArch64::SUBWrr:
5823 case AArch64::ADDSWrr:
5824 case AArch64::ADDSWri:
5825 case AArch64::SUBSWrr:
5827 case AArch64::SUBWri:
5828 case AArch64::SUBSWri:
5839 case AArch64::ADDXrr:
5840 case AArch64::ADDXri:
5841 case AArch64::SUBXrr:
5842 case AArch64::ADDSXrr:
5843 case AArch64::ADDSXri:
5844 case AArch64::SUBSXrr:
5846 case AArch64::SUBXri:
5847 case AArch64::SUBSXri:
5848 case AArch64::ADDv8i8:
5849 case AArch64::ADDv16i8:
5850 case AArch64::ADDv4i16:
5851 case AArch64::ADDv8i16:
5852 case AArch64::ADDv2i32:
5853 case AArch64::ADDv4i32:
5854 case AArch64::SUBv8i8:
5855 case AArch64::SUBv16i8:
5856 case AArch64::SUBv4i16:
5857 case AArch64::SUBv8i16:
5858 case AArch64::SUBv2i32:
5859 case AArch64::SUBv4i32:
5872 case AArch64::FADDHrr:
5873 case AArch64::FADDSrr:
5874 case AArch64::FADDDrr:
5875 case AArch64::FADDv4f16:
5876 case AArch64::FADDv8f16:
5877 case AArch64::FADDv2f32:
5878 case AArch64::FADDv2f64:
5879 case AArch64::FADDv4f32:
5880 case AArch64::FSUBHrr:
5881 case AArch64::FSUBSrr:
5882 case AArch64::FSUBDrr:
5883 case AArch64::FSUBv4f16:
5884 case AArch64::FSUBv8f16:
5885 case AArch64::FSUBv2f32:
5886 case AArch64::FSUBv2f64:
5887 case AArch64::FSUBv4f32:
5891 return Options.UnsafeFPMath ||
5908 unsigned CombineOpc,
unsigned ZeroReg = 0,
5909 bool CheckZeroReg =
false) {
5916 if (!
MI ||
MI->getParent() != &
MBB || (
unsigned)
MI->getOpcode() != CombineOpc)
5919 if (!
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
5923 assert(
MI->getNumOperands() >= 4 &&
MI->getOperand(0).isReg() &&
5924 MI->getOperand(1).isReg() &&
MI->getOperand(2).isReg() &&
5925 MI->getOperand(3).isReg() &&
"MAdd/MSub must have a least 4 regs");
5927 if (
MI->getOperand(3).getReg() != ZeroReg)
5932 MI->findRegisterDefOperandIdx(AArch64::NZCV,
true) == -1)
5941 unsigned MulOpc,
unsigned ZeroReg) {
5957 bool Invert)
const {
5963 case AArch64::FADDHrr:
5964 case AArch64::FADDSrr:
5965 case AArch64::FADDDrr:
5966 case AArch64::FMULHrr:
5967 case AArch64::FMULSrr:
5968 case AArch64::FMULDrr:
5969 case AArch64::FMULX16:
5970 case AArch64::FMULX32:
5971 case AArch64::FMULX64:
5973 case AArch64::FADDv4f16:
5974 case AArch64::FADDv8f16:
5975 case AArch64::FADDv2f32:
5976 case AArch64::FADDv4f32:
5977 case AArch64::FADDv2f64:
5978 case AArch64::FMULv4f16:
5979 case AArch64::FMULv8f16:
5980 case AArch64::FMULv2f32:
5981 case AArch64::FMULv4f32:
5982 case AArch64::FMULv2f64:
5983 case AArch64::FMULXv4f16:
5984 case AArch64::FMULXv8f16:
5985 case AArch64::FMULXv2f32:
5986 case AArch64::FMULXv4f32:
5987 case AArch64::FMULXv2f64:
5991 case AArch64::FADD_ZZZ_H:
5992 case AArch64::FADD_ZZZ_S:
5993 case AArch64::FADD_ZZZ_D:
5994 case AArch64::FMUL_ZZZ_H:
5995 case AArch64::FMUL_ZZZ_S:
5996 case AArch64::FMUL_ZZZ_D:
6008 case AArch64::ADDWrr:
6009 case AArch64::ADDXrr:
6010 case AArch64::ANDWrr:
6011 case AArch64::ANDXrr:
6012 case AArch64::ORRWrr:
6013 case AArch64::ORRXrr:
6014 case AArch64::EORWrr:
6015 case AArch64::EORXrr:
6016 case AArch64::EONWrr:
6017 case AArch64::EONXrr:
6021 case AArch64::ADDv8i8:
6022 case AArch64::ADDv16i8:
6023 case AArch64::ADDv4i16:
6024 case AArch64::ADDv8i16:
6025 case AArch64::ADDv2i32:
6026 case AArch64::ADDv4i32:
6027 case AArch64::ADDv1i64:
6028 case AArch64::ADDv2i64:
6029 case AArch64::MULv8i8:
6030 case AArch64::MULv16i8:
6031 case AArch64::MULv4i16:
6032 case AArch64::MULv8i16:
6033 case AArch64::MULv2i32:
6034 case AArch64::MULv4i32:
6035 case AArch64::ANDv8i8:
6036 case AArch64::ANDv16i8:
6037 case AArch64::ORRv8i8:
6038 case AArch64::ORRv16i8:
6039 case AArch64::EORv8i8:
6040 case AArch64::EORv16i8:
6042 case AArch64::ADD_ZZZ_B:
6043 case AArch64::ADD_ZZZ_H:
6044 case AArch64::ADD_ZZZ_S:
6045 case AArch64::ADD_ZZZ_D:
6046 case AArch64::MUL_ZZZ_B:
6047 case AArch64::MUL_ZZZ_H:
6048 case AArch64::MUL_ZZZ_S:
6049 case AArch64::MUL_ZZZ_D:
6050 case AArch64::AND_ZZZ:
6051 case AArch64::ORR_ZZZ:
6052 case AArch64::EOR_ZZZ:
6082 auto setFound = [&](
int Opcode,
int Operand,
unsigned ZeroReg,
6102 case AArch64::ADDWrr:
6104 "ADDWrr does not have register operands");
6105 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
6106 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
6108 case AArch64::ADDXrr:
6109 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
6110 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
6112 case AArch64::SUBWrr:
6113 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
6114 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
6116 case AArch64::SUBXrr:
6117 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
6118 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
6120 case AArch64::ADDWri:
6121 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
6123 case AArch64::ADDXri:
6124 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
6126 case AArch64::SUBWri:
6127 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
6129 case AArch64::SUBXri:
6130 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
6132 case AArch64::ADDv8i8:
6133 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
6134 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
6136 case AArch64::ADDv16i8:
6137 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
6138 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
6140 case AArch64::ADDv4i16:
6141 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
6142 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
6143 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
6144 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
6146 case AArch64::ADDv8i16:
6147 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
6148 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
6149 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
6150 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
6152 case AArch64::ADDv2i32:
6153 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
6154 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
6155 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
6156 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
6158 case AArch64::ADDv4i32:
6159 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
6160 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
6161 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
6162 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
6164 case AArch64::SUBv8i8:
6165 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
6166 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
6168 case AArch64::SUBv16i8:
6169 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
6170 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
6172 case AArch64::SUBv4i16:
6173 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
6174 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
6175 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
6176 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
6178 case AArch64::SUBv8i16:
6179 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
6180 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
6181 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
6182 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
6184 case AArch64::SUBv2i32:
6185 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
6186 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
6187 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
6188 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
6190 case AArch64::SUBv4i32:
6191 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
6192 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
6193 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
6194 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
6211 auto Match = [&](
int Opcode,
int Operand,
6224 assert(
false &&
"Unsupported FP instruction in combiner\n");
6226 case AArch64::FADDHrr:
6228 "FADDHrr does not have register operands");
6230 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
6231 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
6233 case AArch64::FADDSrr:
6235 "FADDSrr does not have register operands");
6237 Found |=
Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
6238 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
6240 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
6241 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
6243 case AArch64::FADDDrr:
6244 Found |=
Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
6245 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
6247 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
6248 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
6250 case AArch64::FADDv4f16:
6251 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
6252 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
6254 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
6255 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
6257 case AArch64::FADDv8f16:
6258 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
6259 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
6261 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
6262 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
6264 case AArch64::FADDv2f32:
6265 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
6266 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
6268 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
6269 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
6271 case AArch64::FADDv2f64:
6272 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
6273 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
6275 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
6276 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
6278 case AArch64::FADDv4f32:
6279 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
6280 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
6282 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
6283 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
6285 case AArch64::FSUBHrr:
6286 Found =
Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
6287 Found |=
Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
6288 Found |=
Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
6290 case AArch64::FSUBSrr:
6291 Found =
Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
6293 Found |=
Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
6294 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
6296 Found |=
Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
6298 case AArch64::FSUBDrr:
6299 Found =
Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
6301 Found |=
Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
6302 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
6304 Found |=
Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
6306 case AArch64::FSUBv4f16:
6307 Found |=
Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
6308 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
6310 Found |=
Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
6311 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
6313 case AArch64::FSUBv8f16:
6314 Found |=
Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
6315 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
6317 Found |=
Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
6318 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
6320 case AArch64::FSUBv2f32:
6321 Found |=
Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
6322 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
6324 Found |=
Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
6325 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
6327 case AArch64::FSUBv2f64:
6328 Found |=
Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
6329 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
6331 Found |=
Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
6332 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
6334 case AArch64::FSUBv4f32:
6335 Found |=
Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
6336 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
6338 Found |=
Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
6339 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
6350 auto Match = [&](
unsigned Opcode,
int Operand,
6358 if (
MI &&
MI->getOpcode() == TargetOpcode::COPY &&
6359 MI->getOperand(1).getReg().isVirtual())
6360 MI =
MRI.getUniqueVRegDef(
MI->getOperand(1).getReg());
6361 if (
MI &&
MI->getOpcode() == Opcode) {
6373 case AArch64::FMULv2f32:
6374 Found =
Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
6375 Found |=
Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
6377 case AArch64::FMULv2f64:
6378 Found =
Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
6379 Found |=
Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
6381 case AArch64::FMULv4f16:
6382 Found =
Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
6383 Found |=
Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
6385 case AArch64::FMULv4f32:
6386 Found =
Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
6387 Found |=
Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
6389 case AArch64::FMULv8f16:
6390 Found =
Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
6391 Found |=
Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
6407 if (
MI !=
nullptr && (
MI->getOpcode() == Opcode) &&
6408 MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()) &&
6422 case AArch64::FNEGDr:
6424 case AArch64::FNEGSr:
6558 case AArch64::SUBWrr:
6559 case AArch64::SUBSWrr:
6560 case AArch64::SUBXrr:
6561 case AArch64::SUBSXrr:
6591 bool DoRegPressureReduce)
const {
6608 DoRegPressureReduce);
6637 const Register *ReplacedAddend =
nullptr) {
6638 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
6640 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
6643 Register SrcReg0 = MUL->getOperand(1).getReg();
6644 bool Src0IsKill = MUL->getOperand(1).isKill();
6645 Register SrcReg1 = MUL->getOperand(2).getReg();
6646 bool Src1IsKill = MUL->getOperand(2).isKill();
6650 if (ReplacedAddend) {
6652 SrcReg2 = *ReplacedAddend;
6660 MRI.constrainRegClass(ResultReg, RC);
6662 MRI.constrainRegClass(SrcReg0, RC);
6664 MRI.constrainRegClass(SrcReg1, RC);
6666 MRI.constrainRegClass(SrcReg2, RC);
6669 if (kind == FMAInstKind::Default)
6674 else if (kind == FMAInstKind::Indexed)
6679 .
addImm(MUL->getOperand(3).getImm());
6680 else if (kind == FMAInstKind::Accumulator)
6686 assert(
false &&
"Invalid FMA instruction kind \n");
6700 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6701 Opc = AArch64::FNMADDSrrr;
6702 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
6703 Opc = AArch64::FNMADDDrrr;
6715 MRI.constrainRegClass(ResultReg, RC);
6717 MRI.constrainRegClass(SrcReg0, RC);
6719 MRI.constrainRegClass(SrcReg1, RC);
6721 MRI.constrainRegClass(SrcReg2, RC);
6737 unsigned IdxDupOp,
unsigned MulOpc,
6739 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
6740 "Invalid index of FMUL operand");
6748 if (Dup->
getOpcode() == TargetOpcode::COPY)
6752 MRI.clearKillFlags(DupSrcReg);
6753 MRI.constrainRegClass(DupSrcReg, RC);
6757 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
6781 FMAInstKind::Accumulator);
6798 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
6813 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
6815 FMAInstKind::Accumulator, &NewVR);
6827 FMAInstKind::Indexed);
6840 genNeg(MF,
MRI,
TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
6843 FMAInstKind::Indexed, &NewVR);
6868 unsigned IdxMulOpd,
unsigned MaddOpc,
unsigned VR,
6870 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
6874 Register SrcReg0 = MUL->getOperand(1).getReg();
6875 bool Src0IsKill = MUL->getOperand(1).isKill();
6876 Register SrcReg1 = MUL->getOperand(2).getReg();
6877 bool Src1IsKill = MUL->getOperand(2).isKill();
6880 MRI.constrainRegClass(ResultReg, RC);
6882 MRI.constrainRegClass(SrcReg0, RC);
6884 MRI.constrainRegClass(SrcReg1, RC);
6886 MRI.constrainRegClass(VR, RC);
6908 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
6909 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
6919 Register NewVR =
MRI.createVirtualRegister(
MRI.getRegClass(RegA));
6922 if (Opcode == AArch64::SUBSWrr)
6923 Opcode = AArch64::SUBWrr;
6924 else if (Opcode == AArch64::SUBSXrr)
6925 Opcode = AArch64::SUBXrr;
6927 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
6928 "Unexpected instruction opcode.");
6939 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
6965 DelInstrs, InstrIdxForVirtReg);
6971 InstrIdxForVirtReg);
6977 InstrIdxForVirtReg);
6986 Opc = AArch64::MADDWrrr;
6987 RC = &AArch64::GPR32RegClass;
6989 Opc = AArch64::MADDXrrr;
6990 RC = &AArch64::GPR64RegClass;
7001 Opc = AArch64::MADDWrrr;
7002 RC = &AArch64::GPR32RegClass;
7004 Opc = AArch64::MADDXrrr;
7005 RC = &AArch64::GPR64RegClass;
7017 unsigned BitSize, OrrOpc, ZeroReg;
7019 OrrOpc = AArch64::ORRWri;
7020 OrrRC = &AArch64::GPR32spRegClass;
7022 ZeroReg = AArch64::WZR;
7023 Opc = AArch64::MADDWrrr;
7024 RC = &AArch64::GPR32RegClass;
7026 OrrOpc = AArch64::ORRXri;
7027 OrrRC = &AArch64::GPR64spRegClass;
7029 ZeroReg = AArch64::XZR;
7030 Opc = AArch64::MADDXrrr;
7031 RC = &AArch64::GPR64RegClass;
7033 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7044 if (
Insn.size() != 1)
7046 auto MovI =
Insn.begin();
7049 if (MovI->Opcode == OrrOpc)
7055 assert((MovI->Opcode == AArch64::MOVNWi ||
7056 MovI->Opcode == AArch64::MOVZWi) &&
7059 assert((MovI->Opcode == AArch64::MOVNXi ||
7060 MovI->Opcode == AArch64::MOVZXi) &&
7067 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7068 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7079 unsigned SubOpc, ZeroReg;
7081 SubOpc = AArch64::SUBWrr;
7082 SubRC = &AArch64::GPR32spRegClass;
7083 ZeroReg = AArch64::WZR;
7084 Opc = AArch64::MADDWrrr;
7085 RC = &AArch64::GPR32RegClass;
7087 SubOpc = AArch64::SUBXrr;
7088 SubRC = &AArch64::GPR64spRegClass;
7089 ZeroReg = AArch64::XZR;
7090 Opc = AArch64::MADDXrrr;
7091 RC = &AArch64::GPR64RegClass;
7093 Register NewVR =
MRI.createVirtualRegister(SubRC);
7100 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7101 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7111 Opc = AArch64::MSUBWrrr;
7112 RC = &AArch64::GPR32RegClass;
7114 Opc = AArch64::MSUBXrrr;
7115 RC = &AArch64::GPR64RegClass;
7127 unsigned BitSize, OrrOpc, ZeroReg;
7129 OrrOpc = AArch64::ORRWri;
7130 OrrRC = &AArch64::GPR32spRegClass;
7132 ZeroReg = AArch64::WZR;
7133 Opc = AArch64::MADDWrrr;
7134 RC = &AArch64::GPR32RegClass;
7136 OrrOpc = AArch64::ORRXri;
7137 OrrRC = &AArch64::GPR64spRegClass;
7139 ZeroReg = AArch64::XZR;
7140 Opc = AArch64::MADDXrrr;
7141 RC = &AArch64::GPR64RegClass;
7143 Register NewVR =
MRI.createVirtualRegister(OrrRC);
7153 if (
Insn.size() != 1)
7155 auto MovI =
Insn.begin();
7158 if (MovI->Opcode == OrrOpc)
7164 assert((MovI->Opcode == AArch64::MOVNWi ||
7165 MovI->Opcode == AArch64::MOVZWi) &&
7168 assert((MovI->Opcode == AArch64::MOVNXi ||
7169 MovI->Opcode == AArch64::MOVZXi) &&
7176 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7177 MUL =
genMaddR(MF,
MRI,
TII, Root, InsInstrs, 1, Opc, NewVR, RC);
7182 Opc = AArch64::MLAv8i8;
7183 RC = &AArch64::FPR64RegClass;
7187 Opc = AArch64::MLAv8i8;
7188 RC = &AArch64::FPR64RegClass;
7192 Opc = AArch64::MLAv16i8;
7193 RC = &AArch64::FPR128RegClass;
7197 Opc = AArch64::MLAv16i8;
7198 RC = &AArch64::FPR128RegClass;
7202 Opc = AArch64::MLAv4i16;
7203 RC = &AArch64::FPR64RegClass;
7207 Opc = AArch64::MLAv4i16;
7208 RC = &AArch64::FPR64RegClass;
7212 Opc = AArch64::MLAv8i16;
7213 RC = &AArch64::FPR128RegClass;
7217 Opc = AArch64::MLAv8i16;
7218 RC = &AArch64::FPR128RegClass;
7222 Opc = AArch64::MLAv2i32;
7223 RC = &AArch64::FPR64RegClass;
7227 Opc = AArch64::MLAv2i32;
7228 RC = &AArch64::FPR64RegClass;
7232 Opc = AArch64::MLAv4i32;
7233 RC = &AArch64::FPR128RegClass;
7237 Opc = AArch64::MLAv4i32;
7238 RC = &AArch64::FPR128RegClass;
7243 Opc = AArch64::MLAv8i8;
7244 RC = &AArch64::FPR64RegClass;
7246 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
7250 Opc = AArch64::MLSv8i8;
7251 RC = &AArch64::FPR64RegClass;
7255 Opc = AArch64::MLAv16i8;
7256 RC = &AArch64::FPR128RegClass;
7258 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
7262 Opc = AArch64::MLSv16i8;
7263 RC = &AArch64::FPR128RegClass;
7267 Opc = AArch64::MLAv4i16;
7268 RC = &AArch64::FPR64RegClass;
7270 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7274 Opc = AArch64::MLSv4i16;
7275 RC = &AArch64::FPR64RegClass;
7279 Opc = AArch64::MLAv8i16;
7280 RC = &AArch64::FPR128RegClass;
7282 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7286 Opc = AArch64::MLSv8i16;
7287 RC = &AArch64::FPR128RegClass;
7291 Opc = AArch64::MLAv2i32;
7292 RC = &AArch64::FPR64RegClass;
7294 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7298 Opc = AArch64::MLSv2i32;
7299 RC = &AArch64::FPR64RegClass;
7303 Opc = AArch64::MLAv4i32;
7304 RC = &AArch64::FPR128RegClass;
7306 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7310 Opc = AArch64::MLSv4i32;
7311 RC = &AArch64::FPR128RegClass;
7316 Opc = AArch64::MLAv4i16_indexed;
7317 RC = &AArch64::FPR64RegClass;
7321 Opc = AArch64::MLAv4i16_indexed;
7322 RC = &AArch64::FPR64RegClass;
7326 Opc = AArch64::MLAv8i16_indexed;
7327 RC = &AArch64::FPR128RegClass;
7331 Opc = AArch64::MLAv8i16_indexed;
7332 RC = &AArch64::FPR128RegClass;
7336 Opc = AArch64::MLAv2i32_indexed;
7337 RC = &AArch64::FPR64RegClass;
7341 Opc = AArch64::MLAv2i32_indexed;
7342 RC = &AArch64::FPR64RegClass;
7346 Opc = AArch64::MLAv4i32_indexed;
7347 RC = &AArch64::FPR128RegClass;
7351 Opc = AArch64::MLAv4i32_indexed;
7352 RC = &AArch64::FPR128RegClass;
7357 Opc = AArch64::MLAv4i16_indexed;
7358 RC = &AArch64::FPR64RegClass;
7360 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
7364 Opc = AArch64::MLSv4i16_indexed;
7365 RC = &AArch64::FPR64RegClass;
7369 Opc = AArch64::MLAv8i16_indexed;
7370 RC = &AArch64::FPR128RegClass;
7372 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
7376 Opc = AArch64::MLSv8i16_indexed;
7377 RC = &AArch64::FPR128RegClass;
7381 Opc = AArch64::MLAv2i32_indexed;
7382 RC = &AArch64::FPR64RegClass;
7384 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
7388 Opc = AArch64::MLSv2i32_indexed;
7389 RC = &AArch64::FPR64RegClass;
7393 Opc = AArch64::MLAv4i32_indexed;
7394 RC = &AArch64::FPR128RegClass;
7396 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
7400 Opc = AArch64::MLSv4i32_indexed;
7401 RC = &AArch64::FPR128RegClass;
7407 Opc = AArch64::FMADDHrrr;
7408 RC = &AArch64::FPR16RegClass;
7412 Opc = AArch64::FMADDSrrr;
7413 RC = &AArch64::FPR32RegClass;
7417 Opc = AArch64::FMADDDrrr;
7418 RC = &AArch64::FPR64RegClass;
7423 Opc = AArch64::FMADDHrrr;
7424 RC = &AArch64::FPR16RegClass;
7428 Opc = AArch64::FMADDSrrr;
7429 RC = &AArch64::FPR32RegClass;
7433 Opc = AArch64::FMADDDrrr;
7434 RC = &AArch64::FPR64RegClass;
7439 Opc = AArch64::FMLAv1i32_indexed;
7440 RC = &AArch64::FPR32RegClass;
7442 FMAInstKind::Indexed);
7445 Opc = AArch64::FMLAv1i32_indexed;
7446 RC = &AArch64::FPR32RegClass;
7448 FMAInstKind::Indexed);
7452 Opc = AArch64::FMLAv1i64_indexed;
7453 RC = &AArch64::FPR64RegClass;
7455 FMAInstKind::Indexed);
7458 Opc = AArch64::FMLAv1i64_indexed;
7459 RC = &AArch64::FPR64RegClass;
7461 FMAInstKind::Indexed);
7465 RC = &AArch64::FPR64RegClass;
7466 Opc = AArch64::FMLAv4i16_indexed;
7468 FMAInstKind::Indexed);
7471 RC = &AArch64::FPR64RegClass;
7472 Opc = AArch64::FMLAv4f16;
7474 FMAInstKind::Accumulator);
7477 RC = &AArch64::FPR64RegClass;
7478 Opc = AArch64::FMLAv4i16_indexed;
7480 FMAInstKind::Indexed);
7483 RC = &AArch64::FPR64RegClass;
7484 Opc = AArch64::FMLAv4f16;
7486 FMAInstKind::Accumulator);
7491 RC = &AArch64::FPR64RegClass;
7493 Opc = AArch64::FMLAv2i32_indexed;
7495 FMAInstKind::Indexed);
7497 Opc = AArch64::FMLAv2f32;
7499 FMAInstKind::Accumulator);
7504 RC = &AArch64::FPR64RegClass;
7506 Opc = AArch64::FMLAv2i32_indexed;
7508 FMAInstKind::Indexed);
7510 Opc = AArch64::FMLAv2f32;
7512 FMAInstKind::Accumulator);
7517 RC = &AArch64::FPR128RegClass;
7518 Opc = AArch64::FMLAv8i16_indexed;
7520 FMAInstKind::Indexed);
7523 RC = &AArch64::FPR128RegClass;
7524 Opc = AArch64::FMLAv8f16;
7526 FMAInstKind::Accumulator);
7529 RC = &AArch64::FPR128RegClass;
7530 Opc = AArch64::FMLAv8i16_indexed;
7532 FMAInstKind::Indexed);
7535 RC = &AArch64::FPR128RegClass;
7536 Opc = AArch64::FMLAv8f16;
7538 FMAInstKind::Accumulator);
7543 RC = &AArch64::FPR128RegClass;
7545 Opc = AArch64::FMLAv2i64_indexed;
7547 FMAInstKind::Indexed);
7549 Opc = AArch64::FMLAv2f64;
7551 FMAInstKind::Accumulator);
7556 RC = &AArch64::FPR128RegClass;
7558 Opc = AArch64::FMLAv2i64_indexed;
7560 FMAInstKind::Indexed);
7562 Opc = AArch64::FMLAv2f64;
7564 FMAInstKind::Accumulator);
7570 RC = &AArch64::FPR128RegClass;
7572 Opc = AArch64::FMLAv4i32_indexed;
7574 FMAInstKind::Indexed);
7576 Opc = AArch64::FMLAv4f32;
7578 FMAInstKind::Accumulator);
7584 RC = &AArch64::FPR128RegClass;
7586 Opc = AArch64::FMLAv4i32_indexed;
7588 FMAInstKind::Indexed);
7590 Opc = AArch64::FMLAv4f32;
7592 FMAInstKind::Accumulator);
7597 Opc = AArch64::FNMSUBHrrr;
7598 RC = &AArch64::FPR16RegClass;
7602 Opc = AArch64::FNMSUBSrrr;
7603 RC = &AArch64::FPR32RegClass;
7607 Opc = AArch64::FNMSUBDrrr;
7608 RC = &AArch64::FPR64RegClass;
7613 Opc = AArch64::FNMADDHrrr;
7614 RC = &AArch64::FPR16RegClass;
7618 Opc = AArch64::FNMADDSrrr;
7619 RC = &AArch64::FPR32RegClass;
7623 Opc = AArch64::FNMADDDrrr;
7624 RC = &AArch64::FPR64RegClass;
7629 Opc = AArch64::FMSUBHrrr;
7630 RC = &AArch64::FPR16RegClass;
7634 Opc = AArch64::FMSUBSrrr;
7635 RC = &AArch64::FPR32RegClass;
7639 Opc = AArch64::FMSUBDrrr;
7640 RC = &AArch64::FPR64RegClass;
7645 Opc = AArch64::FMLSv1i32_indexed;
7646 RC = &AArch64::FPR32RegClass;
7648 FMAInstKind::Indexed);
7652 Opc = AArch64::FMLSv1i64_indexed;
7653 RC = &AArch64::FPR64RegClass;
7655 FMAInstKind::Indexed);
7660 RC = &AArch64::FPR64RegClass;
7666 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7668 Opc = AArch64::FMLAv4f16;
7670 FMAInstKind::Accumulator, &NewVR);
7672 Opc = AArch64::FMLAv4i16_indexed;
7674 FMAInstKind::Indexed, &NewVR);
7679 RC = &AArch64::FPR64RegClass;
7680 Opc = AArch64::FMLSv4f16;
7682 FMAInstKind::Accumulator);
7685 RC = &AArch64::FPR64RegClass;
7686 Opc = AArch64::FMLSv4i16_indexed;
7688 FMAInstKind::Indexed);
7693 RC = &AArch64::FPR64RegClass;
7695 Opc = AArch64::FMLSv2i32_indexed;
7697 FMAInstKind::Indexed);
7699 Opc = AArch64::FMLSv2f32;
7701 FMAInstKind::Accumulator);
7707 RC = &AArch64::FPR128RegClass;
7713 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7715 Opc = AArch64::FMLAv8f16;
7717 FMAInstKind::Accumulator, &NewVR);
7719 Opc = AArch64::FMLAv8i16_indexed;
7721 FMAInstKind::Indexed, &NewVR);
7726 RC = &AArch64::FPR128RegClass;
7727 Opc = AArch64::FMLSv8f16;
7729 FMAInstKind::Accumulator);
7732 RC = &AArch64::FPR128RegClass;
7733 Opc = AArch64::FMLSv8i16_indexed;
7735 FMAInstKind::Indexed);
7740 RC = &AArch64::FPR128RegClass;
7742 Opc = AArch64::FMLSv2i64_indexed;
7744 FMAInstKind::Indexed);
7746 Opc = AArch64::FMLSv2f64;
7748 FMAInstKind::Accumulator);
7754 RC = &AArch64::FPR128RegClass;
7756 Opc = AArch64::FMLSv4i32_indexed;
7758 FMAInstKind::Indexed);
7760 Opc = AArch64::FMLSv4f32;
7762 FMAInstKind::Accumulator);
7767 RC = &AArch64::FPR64RegClass;
7773 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7775 Opc = AArch64::FMLAv2i32_indexed;
7777 FMAInstKind::Indexed, &NewVR);
7779 Opc = AArch64::FMLAv2f32;
7781 FMAInstKind::Accumulator, &NewVR);
7787 RC = &AArch64::FPR128RegClass;
7793 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7795 Opc = AArch64::FMLAv4i32_indexed;
7797 FMAInstKind::Indexed, &NewVR);
7799 Opc = AArch64::FMLAv4f32;
7801 FMAInstKind::Accumulator, &NewVR);
7807 RC = &AArch64::FPR128RegClass;
7813 InstrIdxForVirtReg.
insert(std::make_pair(NewVR, 0));
7815 Opc = AArch64::FMLAv2i64_indexed;
7817 FMAInstKind::Indexed, &NewVR);
7819 Opc = AArch64::FMLAv2f64;
7821 FMAInstKind::Accumulator, &NewVR);
7830 &AArch64::FPR128RegClass,
MRI);
7838 &AArch64::FPR128RegClass,
MRI);
7846 &AArch64::FPR128_loRegClass,
MRI);
7854 &AArch64::FPR128RegClass,
MRI);
7862 &AArch64::FPR128_loRegClass,
MRI);
7881 for (
auto *
MI : InsInstrs)
7882 MI->setFlags(Flags);
7923 bool IsNegativeBranch =
false;
7924 bool IsTestAndBranch =
false;
7925 unsigned TargetBBInMI = 0;
7926 switch (
MI.getOpcode()) {
7935 case AArch64::CBNZW:
7936 case AArch64::CBNZX:
7938 IsNegativeBranch =
true;
7943 IsTestAndBranch =
true;
7945 case AArch64::TBNZW:
7946 case AArch64::TBNZX:
7948 IsNegativeBranch =
true;
7949 IsTestAndBranch =
true;
7955 if (IsTestAndBranch &&
MI.getOperand(1).getImm())
7959 assert(
MI.getParent() &&
"Incomplete machine instruciton\n");
7972 if (!
MRI->hasOneNonDBGUse(CopyVReg))
7974 if (!
MRI->hasOneDef(CopyVReg))
7983 case AArch64::ANDWri:
7984 case AArch64::ANDXri: {
7985 if (IsTestAndBranch)
7989 if (!
MRI->hasOneNonDBGUse(VReg))
8003 assert(!
MRI->def_empty(NewReg) &&
"Register must be defined.");
8009 unsigned Opc = (Imm < 32)
8010 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
8011 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
8024 if (!Is32Bit && Imm < 32)
8026 MI.eraseFromParent();
8030 case AArch64::CSINCWr:
8031 case AArch64::CSINCXr: {
8050 if (IsNegativeBranch)
8053 MI.eraseFromParent();
8059std::pair<unsigned, unsigned>
8062 return std::make_pair(TF & Mask, TF & ~Mask);
8067 using namespace AArch64II;
8069 static const std::pair<unsigned, const char *> TargetFlags[] = {
8070 {MO_PAGE,
"aarch64-page"}, {MO_PAGEOFF,
"aarch64-pageoff"},
8071 {MO_G3,
"aarch64-g3"}, {MO_G2,
"aarch64-g2"},
8072 {MO_G1,
"aarch64-g1"}, {MO_G0,
"aarch64-g0"},
8073 {MO_HI12,
"aarch64-hi12"}};
8079 using namespace AArch64II;
8081 static const std::pair<unsigned, const char *> TargetFlags[] = {
8082 {MO_COFFSTUB,
"aarch64-coffstub"},
8083 {MO_GOT,
"aarch64-got"},
8084 {MO_NC,
"aarch64-nc"},
8085 {MO_S,
"aarch64-s"},
8086 {MO_TLS,
"aarch64-tls"},
8087 {MO_DLLIMPORT,
"aarch64-dllimport"},
8088 {MO_PREL,
"aarch64-prel"},
8089 {MO_TAGGED,
"aarch64-tagged"},
8090 {MO_ARM64EC_CALLMANGLE,
"aarch64-arm64ec-callmangle"},
8097 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
8205 for (
unsigned Reg : AArch64::GPR64RegClass) {
8207 Reg != AArch64::LR &&
8208 Reg != AArch64::X16 &&
8209 Reg != AArch64::X17 &&
8210 C.isAvailableAcrossAndOutOfSeq(
Reg,
TRI) &&
8211 C.isAvailableInsideSeq(
Reg,
TRI))
8242 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
8245std::optional<outliner::OutlinedFunction>
8247 std::vector<outliner::Candidate> &RepeatedSequenceLocs)
const {
8250 unsigned SequenceSize = 0;
8251 for (
auto &
MI : FirstCand)
8254 unsigned NumBytesToCreateFrame = 0;
8264 if (std::adjacent_find(
8265 RepeatedSequenceLocs.begin(), RepeatedSequenceLocs.end(),
8269 if (outliningCandidatesSigningScopeConsensus(a, b) &&
8270 outliningCandidatesSigningKeyConsensus(a, b) &&
8271 outliningCandidatesV8_3OpsConsensus(a, b)) {
8275 }) != RepeatedSequenceLocs.end()) {
8276 return std::nullopt;
8293 unsigned NumBytesToCheckLRInTCEpilogue = 0;
8294 if (FirstCand.getMF()
8298 NumBytesToCreateFrame += 8;
8301 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod();
8302 NumBytesToCheckLRInTCEpilogue =
8306 if (isTailCallReturnInst(RepeatedSequenceLocs[0].back()))
8307 SequenceSize += NumBytesToCheckLRInTCEpilogue;
8315 for (
auto &
MI :
C) {
8316 if (
MI.modifiesRegister(AArch64::SP, &
TRI)) {
8317 switch (
MI.getOpcode()) {
8318 case AArch64::ADDXri:
8319 case AArch64::ADDWri:
8320 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8322 "Expected operand to be immediate");
8324 "Expected operand to be a register");
8328 if (
MI.getOperand(1).getReg() == AArch64::SP)
8329 SPValue +=
MI.getOperand(2).getImm();
8333 case AArch64::SUBXri:
8334 case AArch64::SUBWri:
8335 assert(
MI.getNumOperands() == 4 &&
"Wrong number of operands");
8337 "Expected operand to be immediate");
8339 "Expected operand to be a register");
8343 if (
MI.getOperand(1).getReg() == AArch64::SP)
8344 SPValue -=
MI.getOperand(2).getImm();
8361 if (RepeatedSequenceLocs.size() < 2)
8362 return std::nullopt;
8366 unsigned FlagsSetInAll = 0xF;
8370 FlagsSetInAll &=
C.Flags;
8372 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
8375 auto SetCandidateCallInfo =
8376 [&RepeatedSequenceLocs](
unsigned CallID,
unsigned NumBytesForCall) {
8378 C.setCallInfo(CallID, NumBytesForCall);
8382 NumBytesToCreateFrame += 4;
8385 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
8390 unsigned CFICount = 0;
8391 for (
auto &
I : RepeatedSequenceLocs[0]) {
8392 if (
I.isCFIInstruction())
8402 std::vector<MCCFIInstruction> CFIInstructions =
8403 C.getMF()->getFrameInstructions();
8405 if (CFICount > 0 && CFICount != CFIInstructions.size())
8406 return std::nullopt;
8414 if (!
MI.modifiesRegister(AArch64::SP, &
TRI) &&
8415 !
MI.readsRegister(AArch64::SP, &
TRI))
8421 if (
MI.modifiesRegister(AArch64::SP, &
TRI))
8426 if (
MI.mayLoadOrStore()) {
8429 bool OffsetIsScalable;
8433 if (!getMemOperandWithOffset(
MI,
Base,
Offset, OffsetIsScalable, &
TRI) ||
8434 !
Base->isReg() ||
Base->getReg() != AArch64::SP)
8438 if (OffsetIsScalable)
8446 TypeSize Scale(0U,
false), DummyWidth(0U,
false);
8447 getMemOpInfo(
MI.getOpcode(), Scale, DummyWidth, MinOffset, MaxOffset);
8450 if (
Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
8451 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
8466 bool AllStackInstrsSafe =
llvm::all_of(FirstCand, IsSafeToFixup);
8470 if (RepeatedSequenceLocs[0].back().isTerminator()) {
8472 NumBytesToCreateFrame = 0;
8473 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
8477 else if (LastInstrOpcode == AArch64::BL ||
8478 ((LastInstrOpcode == AArch64::BLR ||
8479 LastInstrOpcode == AArch64::BLRNoIP) &&
8483 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
8491 unsigned NumBytesNoStackCalls = 0;
8492 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
8497 (
C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
8498 ?
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI)
8507 C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn);
8510 if (LRAvailable && !IsNoReturn) {
8511 NumBytesNoStackCalls += 4;
8513 CandidatesWithoutStackFixups.push_back(
C);
8518 else if (findRegisterToSaveLRTo(
C)) {
8519 NumBytesNoStackCalls += 12;
8521 CandidatesWithoutStackFixups.push_back(
C);
8526 else if (
C.isAvailableInsideSeq(AArch64::SP,
TRI)) {
8527 NumBytesNoStackCalls += 12;
8529 CandidatesWithoutStackFixups.push_back(
C);
8535 NumBytesNoStackCalls += SequenceSize;
8542 if (!AllStackInstrsSafe ||
8543 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
8544 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
8594 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
8598 (!
C.isAvailableAcrossAndOutOfSeq(AArch64::LR,
TRI) ||
8599 !findRegisterToSaveLRTo(
C));
8605 if (RepeatedSequenceLocs.size() < 2) {
8606 RepeatedSequenceLocs.clear();
8607 return std::nullopt;
8613 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
8616 bool ModStackToSaveLR =
false;
8617 if (std::any_of(FirstCand.begin(), std::prev(FirstCand.end()),
8619 ModStackToSaveLR =
true;
8628 ModStackToSaveLR =
true;
8630 if (ModStackToSaveLR) {
8632 if (!AllStackInstrsSafe) {
8633 RepeatedSequenceLocs.clear();
8634 return std::nullopt;
8638 NumBytesToCreateFrame += 8;
8645 return std::nullopt;
8648 NumBytesToCreateFrame, FrameID);
8652 Function &
F, std::vector<outliner::Candidate> &Candidates)
const {
8656 const auto &CFn = Candidates.front().getMF()->getFunction();
8660 if (CFn.hasFnAttribute(
"sign-return-address"))
8661 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address"));
8662 if (CFn.hasFnAttribute(
"sign-return-address-key"))
8663 F.addFnAttr(CFn.getFnAttribute(
"sign-return-address-key"));
8665 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(
F, Candidates);
8673 if (!OutlineFromLinkOnceODRs &&
F.hasLinkOnceODRLinkage())
8687 if (!AFI || AFI->
hasRedZone().value_or(
true))
8700 unsigned &Flags)
const {
8702 "Must track liveness!");
8704 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
8719 auto AreAllUnsafeRegsDead = [&LRU]() {
8736 bool LRAvailableEverywhere =
true;
8741 if (
MI.isCall() && !
MI.isTerminator())
8742 Flags |= MachineOutlinerMBBFlags::HasCalls;
8747 auto CreateNewRangeStartingAt =
8748 [&RangeBegin, &RangeEnd,
8750 RangeBegin = NewBegin;
8751 RangeEnd = std::next(RangeBegin);
8754 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
8759 Ranges.push_back(std::make_pair(RangeBegin, RangeEnd));
8767 for (; FirstPossibleEndPt !=
MBB.
instr_rend(); ++FirstPossibleEndPt) {
8771 UpdateWholeMBBFlags(*FirstPossibleEndPt);
8772 if (AreAllUnsafeRegsDead())
8779 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
8785 UpdateWholeMBBFlags(
MI);
8786 if (!AreAllUnsafeRegsDead()) {
8787 SaveRangeIfNonEmpty();
8788 CreateNewRangeStartingAt(
MI.getIterator());
8791 LRAvailableEverywhere &= LRU.
available(AArch64::LR);
8792 RangeBegin =
MI.getIterator();
8797 if (AreAllUnsafeRegsDead())
8798 SaveRangeIfNonEmpty();
8803 std::reverse(Ranges.begin(), Ranges.end());
8806 if (!LRAvailableEverywhere)
8807 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
8813 unsigned Flags)
const {
8821 switch (
MI.getOpcode()) {
8823 case AArch64::PACIASP:
8824 case AArch64::PACIBSP:
8825 case AArch64::PACIASPPC:
8826 case AArch64::PACIBSPPC:
8827 case AArch64::AUTIASP:
8828 case AArch64::AUTIBSP:
8829 case AArch64::AUTIASPPCi:
8830 case AArch64::AUTIASPPCr:
8831 case AArch64::AUTIBSPPCi:
8832 case AArch64::AUTIBSPPCr:
8833 case AArch64::RETAA:
8834 case AArch64::RETAB:
8835 case AArch64::RETAASPPCi:
8836 case AArch64::RETAASPPCr:
8837 case AArch64::RETABSPPCi:
8838 case AArch64::RETABSPPCr:
8839 case AArch64::EMITBKEY:
8840 case AArch64::PAUTH_PROLOGUE:
8841 case AArch64::PAUTH_EPILOGUE:
8855 if (
MI.isCFIInstruction())
8859 if (
MI.isTerminator())
8868 assert(!MOP.isCFIIndex());
8871 if (MOP.isReg() && !MOP.isImplicit() &&
8872 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
8879 if (
MI.getOpcode() == AArch64::ADRP)
8900 if (MOP.isGlobal()) {
8901 Callee = dyn_cast<Function>(MOP.getGlobal());
8908 if (Callee && Callee->getName() ==
"\01_mcount")
8916 if (
MI.getOpcode() == AArch64::BLR ||
8917 MI.getOpcode() == AArch64::BLRNoIP ||
MI.getOpcode() == AArch64::BL)
8921 return UnknownCallOutlineType;
8929 return UnknownCallOutlineType;
8937 return UnknownCallOutlineType;
8962 bool OffsetIsScalable;
8965 if (!
MI.mayLoadOrStore() ||
8968 (
Base->isReg() &&
Base->getReg() != AArch64::SP))
8973 int64_t Dummy1, Dummy2;
8976 assert(StackOffsetOperand.
isImm() &&
"Stack offset wasn't immediate!");
8978 assert(Scale != 0 &&
"Unexpected opcode!");
8979 assert(!OffsetIsScalable &&
"Expected offset to be a byte offset");
8984 int64_t NewImm = (
Offset + 16) / (int64_t)Scale.getFixedValue();
8985 StackOffsetOperand.
setImm(NewImm);
8991 bool ShouldSignReturnAddr) {
8992 if (!ShouldSignReturnAddr)
8998 TII->get(AArch64::PAUTH_EPILOGUE))
9014 unsigned TailOpcode;
9015 if (Call->getOpcode() == AArch64::BL) {
9016 TailOpcode = AArch64::TCRETURNdi;
9018 assert(Call->getOpcode() == AArch64::BLR ||
9019 Call->getOpcode() == AArch64::BLRNoIP);
9020 TailOpcode = AArch64::TCRETURNriALL;
9023 .
add(Call->getOperand(0))
9026 Call->eraseFromParent();
9031 bool IsLeafFunction =
true;
9035 return MI.isCall() && !
MI.isReturn();
9045 "Can only fix up stack references once");
9046 fixupPostOutline(
MBB);
9048 IsLeafFunction =
false;
9059 Et = std::prev(
MBB.
end());
9072 unsigned DwarfReg =
MRI->getDwarfRegNum(AArch64::LR,
true);
9075 int64_t StackPosEntry =
9128 fixupPostOutline(
MBB);
9139 .addGlobalAddress(M.getNamedValue(MF.
getName()))
9149 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9164 Register Reg = findRegisterToSaveLRTo(
C);
9165 assert(Reg &&
"No callee-saved register available?");
9199 .addGlobalAddress(M.getNamedValue(MF.
getName())));
9215 bool AllowSideEffects)
const {
9220 if (
TRI.isGeneralPurposeRegister(MF, Reg)) {
9222 }
else if (STI.hasSVE()) {
9232std::optional<DestSourcePair>
9237 if (
MI.getOpcode() == AArch64::ORRWrs &&
9238 MI.getOperand(1).getReg() == AArch64::WZR &&
9239 MI.getOperand(3).getImm() == 0x0 &&
9241 (!
MI.getOperand(0).getReg().isVirtual() ||
9242 MI.getOperand(0).getSubReg() == 0) &&
9243 (!
MI.getOperand(0).getReg().isPhysical() ||
9244 MI.findRegisterDefOperandIdx(
MI.getOperand(0).getReg() - AArch64::W0 +
9245 AArch64::X0) == -1))
9248 if (
MI.getOpcode() == AArch64::ORRXrs &&
9249 MI.getOperand(1).getReg() == AArch64::XZR &&
9250 MI.getOperand(3).getImm() == 0x0)
9253 return std::nullopt;
9256std::optional<DestSourcePair>
9258 if (
MI.getOpcode() == AArch64::ORRWrs &&
9259 MI.getOperand(1).getReg() == AArch64::WZR &&
9260 MI.getOperand(3).getImm() == 0x0)
9262 return std::nullopt;
9265std::optional<RegImmPair>
9274 return std::nullopt;
9276 switch (
MI.getOpcode()) {
9278 return std::nullopt;
9279 case AArch64::SUBWri:
9280 case AArch64::SUBXri:
9281 case AArch64::SUBSWri:
9282 case AArch64::SUBSXri:
9285 case AArch64::ADDSWri:
9286 case AArch64::ADDSXri:
9287 case AArch64::ADDWri:
9288 case AArch64::ADDXri: {
9290 if (!
MI.getOperand(0).isReg() || !
MI.getOperand(1).isReg() ||
9291 !
MI.getOperand(2).isImm())
9292 return std::nullopt;
9293 int Shift =
MI.getOperand(3).getImm();
9294 assert((Shift == 0 || Shift == 12) &&
"Shift can be either 0 or 12");
9295 Offset = Sign * (
MI.getOperand(2).getImm() << Shift);
9304static std::optional<ParamLoadedValue>
9308 auto DestSrc =
TII->isCopyLikeInstr(
MI);
9310 return std::nullopt;
9312 Register DestReg = DestSrc->Destination->getReg();
9313 Register SrcReg = DestSrc->Source->getReg();
9318 if (DestReg == DescribedReg)
9322 if (
MI.getOpcode() == AArch64::ORRWrs &&
9323 TRI->isSuperRegister(DestReg, DescribedReg))
9327 if (
MI.getOpcode() == AArch64::ORRXrs &&
9328 TRI->isSubRegister(DestReg, DescribedReg)) {
9329 Register SrcSubReg =
TRI->getSubReg(SrcReg, AArch64::sub_32);
9333 assert(!
TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
9334 "Unhandled ORR[XW]rs copy case");
9336 return std::nullopt;
9355 return MI.getOpcode() == AArch64::INLINEASM_BR;
9373 switch (
MI.getOpcode()) {
9374 case TargetOpcode::G_BRJT:
9375 case AArch64::JumpTableDest32:
9376 case AArch64::JumpTableDest16:
9377 case AArch64::JumpTableDest8:
9388std::optional<ParamLoadedValue>
9393 switch (
MI.getOpcode()) {
9394 case AArch64::MOVZWi:
9395 case AArch64::MOVZXi: {
9398 if (!
TRI->isSuperRegisterEq(
MI.getOperand(0).getReg(), Reg))
9399 return std::nullopt;
9401 if (!
MI.getOperand(1).isImm())
9402 return std::nullopt;
9403 int64_t Immediate =
MI.getOperand(1).getImm();
9404 int Shift =
MI.getOperand(2).getImm();
9408 case AArch64::ORRWrs:
9409 case AArch64::ORRXrs:
9419 ExtMI.
getOpcode() == TargetOpcode::G_ZEXT ||
9420 ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT);
9423 if (ExtMI.
getOpcode() == TargetOpcode::G_ANYEXT)
9427 if (!
MRI.hasOneNonDBGUse(DefReg))
9432 auto *UserMI = &*
MRI.use_instr_nodbg_begin(DefReg);
9433 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
9454 unsigned Scale)
const {
9465 unsigned Shift =
Log2_64(NumBytes);
9466 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
9474 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
9479 return AArch64::BLRNoIP;
9481 return AArch64::BLR;
9486 Register TargetReg,
bool FrameSetup)
const {
9487 assert(TargetReg != AArch64::SP &&
"New top of stack cannot aleady be in SP");
9499 MF.
insert(MBBInsertPoint, LoopTestMBB);
9502 MF.
insert(MBBInsertPoint, LoopBodyMBB);
9504 MF.
insert(MBBInsertPoint, ExitMBB);
9514 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(AArch64::SUBSXrx64),
9528 BuildMI(*LoopBodyMBB, LoopBodyMBB->
end(),
DL,
TII->get(AArch64::STRXui))
9541 BuildMI(*ExitMBB, ExitMBB->
end(),
DL,
TII->get(AArch64::ADDXri), AArch64::SP)
9564 bool anyChange =
false;
9569 }
while (anyChange);
9573 return ExitMBB->
begin();
9584 : PredBranch(PredBranch),
Cond(
Cond.begin(),
Cond.end()) {}
9586 bool shouldIgnoreForPipelining(
const MachineInstr *
MI)
const override {
9589 return MI == PredBranch;
9592 std::optional<bool> createTripCountGreaterCondition(
9604 void adjustTripCount(
int TripCountAdjust)
override {}
9606 void disposed()
override {}
9614 case AArch64::CBNZW:
9615 case AArch64::CBNZX:
9618 case AArch64::TBNZW:
9619 case AArch64::TBNZX:
9625std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
9633 if (
TBB == LoopBB && FBB == LoopBB)
9640 assert((
TBB == LoopBB || FBB == LoopBB) &&
9641 "The Loop must be a single-basic-block loop");
9652 if (CondBranch->
getOpcode() == AArch64::Bcc) {
9654 if (
MI.modifiesRegister(AArch64::NZCV, &
TRI)) {
9664 if (!Reg.isVirtual())
9666 PredBranch =
MRI.getVRegDef(Reg);
9669 if (PredBranch->
isPHI())
9678 return std::make_unique<AArch64PipelinerLoopInfo>(PredBranch,
Cond);
9681#define GET_INSTRINFO_HELPERS
9682#define GET_INSTRMAP_INFO
9683#include "AArch64GenInstrInfo.inc"
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static cl::opt< unsigned > BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg)
Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr)
Find a condition code used by the instruction.
static MachineInstr * genFusedMultiplyAcc(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
static bool getMiscPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns)
Find other MI combine patterns.
static bool isCombineInstrCandidate64(unsigned Opc)
static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg)
static bool areCFlagsAccessedBetweenInstrs(MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
True when condition flags are accessed (either by writing or reading) on the instruction trace starti...
static bool isADDSRegImm(unsigned Opcode)
static MachineInstr * genFusedMultiplyIdxNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static bool isCompareAndBranch(unsigned Opcode)
static unsigned sForm(MachineInstr &Instr)
Get opcode of S version of Instr.
static bool isCombineInstrSettingFlag(unsigned Opc)
static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc)
static int findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr)
static unsigned getBranchDisplacementBits(unsigned Opc)
static std::optional< ParamLoadedValue > describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then,...
static bool getFNEGPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns)
static MachineInstr * genFusedMultiplyAccNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional...
static void appendVGScaledOffsetExpr(SmallVectorImpl< char > &Expr, int NumBytes, int NumVGScaledBytes, unsigned VG, llvm::raw_string_ostream &Comment)
static MachineInstr * genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example ...
static bool scaleOffset(unsigned Opc, int64_t &Offset)
static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale)
static MachineInstr * genFusedMultiplyIdx(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
static MachineInstr * genIndexedMultiply(MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static bool isSUBSRegImm(unsigned Opcode)
static bool UpdateOperandRegClass(MachineInstr &Instr)
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
unsigned unscaledOffsetOpcode(unsigned Opcode)
static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
Check if CmpInstr can be substituted by MI.
static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC)
static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
genNeg - Helper to generate an intermediate negation of the second operand of Root
static bool isCombineInstrCandidateFP(const MachineInstr &Inst)
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI)
Return the opcode that does not set flags when possible - otherwise return the original opcode.
static const MachineInstrBuilder & AddSubReg(const MachineInstrBuilder &MIB, unsigned Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool isCombineInstrCandidate32(unsigned Opc)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
static unsigned offsetExtendOpcode(unsigned Opcode)
static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static bool outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a, const outliner::Candidate &b)
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
static bool getFMULPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns)
static cl::opt< unsigned > TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
static bool getFMAPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns)
Floating-Point Support.
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
static MachineInstr * genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC, FMAInstKind kind=FMAInstKind::Default, const Register *ReplacedAddend=nullptr)
genFusedMultiply - Generate fused multiply instructions.
static bool isCombineInstrCandidate(unsigned Opc)
static unsigned regOffsetOpcode(unsigned Opcode)
MachineOutlinerClass
Constants defining how certain sequences should be outlined.
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerRegSave
Emit a call and tail-call.
@ MachineOutlinerNoLRSave
Only emit a branch.
@ MachineOutlinerThunk
Emit a call and return.
static cl::opt< unsigned > BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB)
Check if AArch64::NZCV should be alive in successors of MBB.
static void emitFrameOffsetAdj(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize)
static cl::opt< unsigned > CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewVReg=nullptr)
static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
static MachineInstr * genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs)
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static bool getMaddPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns)
Find instructions that can be turned into madd.
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool shouldSignReturnAddress(const MachineFunction &MF) const
const SetOfInstructions & getLOHRelated() const
bool needsDwarfUnwindInfo(const MachineFunction &MF) const
void setOutliningStyle(std::string Style)
std::optional< bool > hasRedZone() const
bool shouldSignWithBKey() const
static bool isHForm(const MachineInstr &MI)
Returns whether the instruction is in H form (16 bit operands)
void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DstReg, ArrayRef< MachineOperand > Cond, Register TrueReg, Register FalseReg) const override
static bool hasBTISemantics(const MachineInstr &MI)
Returns whether the instruction can be compatible with non-zero BTYPE.
static bool isQForm(const MachineInstr &MI)
Returns whether the instruction is in Q form (128 bit operands)
static void decomposeStackOffsetForFrameOffsets(const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors, int64_t &NumDataVectors)
Returns the offset in parts to which this frame offset can be decomposed for the purpose of describin...
static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width, int64_t &MinOffset, int64_t &MaxOffset)
Returns true if opcode Opc is a memory operation.
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isFPRCopy(const MachineInstr &MI)
Does this instruction rename an FPR without modifying bits?
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
bool isThroughputPattern(MachineCombinerPattern Pattern) const override
Return true when a code sequence can improve throughput.
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
If the specific machine instruction is an instruction that moves/copies value from one register to an...
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
GetInstSize - Return the number of bytes of code the specified instruction may be.
uint64_t getElementSizeForOpcode(unsigned Opc) const
Returns the vector element size (B, H, S or D) of an SVE opcode.
outliner::InstrType getOutliningTypeImpl(MachineBasicBlock::iterator &MIT, unsigned Flags) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
static bool isGPRCopy(const MachineInstr &MI)
Does this instruction rename a GPR without modifying bits?
static unsigned convertToFlagSettingOpc(unsigned Opc)
Return the opcode that set flags when possible.
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
bool canInsertSelect(const MachineBasicBlock &, ArrayRef< MachineOperand > Cond, Register, Register, Register, int &, int &, int &) const override
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, unsigned &SubIdx) const override
bool isWhileOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE WHILE## instruction.
static std::optional< unsigned > getUnscaledLdSt(unsigned Opc)
Returns the unscaled load/store for the scaled load/store opcode, if there is a corresponding unscale...
static bool hasUnscaledLdStOffset(unsigned Opc)
Return true if it has an unscaled load/store offset.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
std::optional< ExtAddrMode > getAddrModeFromMemoryOp(const MachineInstr &MemI, const TargetRegisterInfo *TRI) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
bool analyzeBranchPredicate(MachineBasicBlock &MBB, MachineBranchPredicate &MBP, bool AllowModify) const override
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
SmallVector< std::pair< MachineBasicBlock::iterator, MachineBasicBlock::iterator > > getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override
static bool isPairableLdStInst(const MachineInstr &MI)
Return true if pairing the given load or store may be paired with another.
const AArch64RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
static bool isPreSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed store.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
AArch64InstrInfo(const AArch64Subtarget &STI)
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
bool useMachineCombiner() const override
AArch64 supports MachineCombiner.
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool isExtendLikelyToBeFolded(MachineInstr &ExtMI, MachineRegisterInfo &MRI) const override
static bool isFalkorShiftExtFast(const MachineInstr &MI)
Returns true if the instruction has a shift by immediate that can be executed in one cycle less.
std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, TypeSize &Width, const TargetRegisterInfo *TRI) const
If OffsetIsScalable is set to 'true', the offset is scaled by vscale.
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
static bool isStridedAccess(const MachineInstr &MI)
Return true if the given load or store is a strided memory access.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
Detect opportunities for ldp/stp formation.
void genAlternativeCodeSequence(MachineInstr &Root, MachineCombinerPattern Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const override
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
bool expandPostRAPseudo(MachineInstr &MI) const override
unsigned int getTailDuplicateSize(CodeGenOptLevel OptLevel) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
static bool isFpOrNEON(const MachineInstr &MI)
Returns whether the instruction is FP or NEON.
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL, unsigned DestReg, unsigned SrcReg, bool KillSrc, unsigned Opcode, unsigned ZeroReg, llvm::ArrayRef< unsigned > Indices) const
void buildClearRegister(Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator Iter, DebugLoc &DL, bool AllowSideEffects=true) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
MachineOperand & getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const
Return the immediate offset of the base register in a load/store LdSt.
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
static bool isLdStPairSuppressed(const MachineInstr &MI)
Return true if pairing the given load or store is hinted to be unprofitable.
bool isFunctionSafeToSplit(const MachineFunction &MF) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
Return true when Inst is associative and commutative so that it can be reassociated.
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI, Register TargetReg, bool FrameSetup) const
bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const override
optimizeCompareInstr - Convert the instruction supplying the argument to the comparison into one that...
std::optional< outliner::OutlinedFunction > getOutliningCandidateInfo(std::vector< outliner::Candidate > &RepeatedSequenceLocs) const override
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns, bool DoRegPressureReduce) const override
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
static unsigned getLoadStoreImmIdx(unsigned Opc)
Returns the index for the immediate for a given instruction.
static bool isGPRZero(const MachineInstr &MI)
Does this instruction set its full destination register to zero?
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const override
analyzeCompare - For a comparison instruction, return the source registers in SrcReg and SrcReg2,...
bool isMBBSafeToSplitToCold(const MachineBasicBlock &MBB) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool isLegalAddressingMode(unsigned NumBytes, int64_t Offset, unsigned Scale) const
std::optional< DestSourcePair > isCopyLikeInstrImpl(const MachineInstr &MI) const override
static void suppressLdStPair(MachineInstr &MI)
Hint that pairing the given load or store is unprofitable.
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
static bool isPreLd(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load.
ArrayRef< std::pair< unsigned, const char * > > getSerializableBitmaskMachineOperandTargetFlags() const override
void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, unsigned Opcode, llvm::ArrayRef< unsigned > Indices) const
bool optimizeCondBranch(MachineInstr &MI) const override
Replace csincr-branch sequence by simple conditional branch.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
bool isCandidateToMergeOrPair(const MachineInstr &MI) const
Return true if this is a load/store that can be potentially paired/merged.
MCInst getNop() const override
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
bool isPTestLikeOpcode(unsigned Opc) const
Returns true if the opcode is for an SVE instruction that sets the condition codes as if it's results...
void mergeOutliningCandidateAttributes(Function &F, std::vector< outliner::Candidate > &Candidates) const override
static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized)
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
bool isNeonAvailable() const
Returns true if the target has NEON and the function at runtime is known to have NEON enabled (e....
const AArch64RegisterInfo * getRegisterInfo() const override
bool isTargetILP32() const
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Module * getParent()
Get the module that this global value is contained inside of...
A set of register units used to track register liveness.
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
void stepBackward(const MachineInstr &MI)
Updates liveness when stepping backwards over the instruction MI.
void addLiveOuts(const MachineBasicBlock &MBB)
Adds registers living out of block MBB.
This class is intended to be used as a base class for asm properties and features specific to the tar...
bool usesWindowsCFI() const
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
static constexpr unsigned NoRegister
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
reverse_instr_iterator instr_rbegin()
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MBBSectionID getSectionID() const
Returns the section ID of this basic block.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rend()
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
Instructions::iterator instr_iterator
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
instr_iterator instr_end()
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
unsigned getNumObjects() const
Return the number of objects.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
int findRegisterDefOperandIdx(Register Reg, bool isDead=false, bool Overlap=false, const TargetRegisterInfo *TRI=nullptr) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
uint32_t mergeFlagsWith(const MachineInstr &Other) const
Return the MIFlags which represent both MachineInstrs.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr fully defines the specified register.
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
const std::vector< MachineJumpTableEntry > & getJumpTables() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineFunction * getMachineFunction(const Function &F) const
Returns the MachineFunction associated to IR function F if there is one, otherwise nullptr.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsDead(bool Val=true)
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
bool isRegUsed(Register Reg, bool includeReserved=true) const
Return if a specific register is currently used.
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Represents a location in source code.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual std::optional< ParamLoadedValue > describeLoadedValue(const MachineInstr &MI, Register Reg) const
Produce the expression describing the MI loading a value into the physical register Reg.
virtual void genAlternativeCodeSequence(MachineInstr &Root, MachineCombinerPattern Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
virtual bool isFunctionSafeToSplit(const MachineFunction &MF) const
Return true if the function is a viable candidate for machine function splitting.
Primary interface to the complete machine description for the target machine.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
unsigned getCheckerSizeInBytes(AuthCheckMethod Method)
Returns the number of bytes added by checkAuthenticatedRegister.
const SysReg * lookupSysRegByName(StringRef)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getArithShiftValue(unsigned Imm)
getArithShiftValue - get the arithmetic shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static AArch64_AM::ShiftExtendType getArithExtendType(unsigned Imm)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static const uint64_t InstrFlagIsWhile
static constexpr unsigned SVEMaxBitsPerVector
static const uint64_t InstrFlagIsPTestLike
static constexpr unsigned SVEBitsPerBlock
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static bool isCondBranchOpcode(int Opc)
std::pair< MachineOperand, DIExpression * > ParamLoadedValue
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
static bool isPTrueOpcode(unsigned Opc)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
static bool isIndirectBranchOpcode(int Opc)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetIsLegal
Offset is legal.
@ AArch64FrameOffsetCanUpdate
Offset can apply, at least partly.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
MachineCombinerPattern
These are instruction patterns matched by the machine combiner pass.
@ MULSUBv4i16_indexed_OP2
@ MULSUBv8i16_indexed_OP2
@ MULADDv4i16_indexed_OP1
@ MULSUBv2i32_indexed_OP1
@ MULADDv2i32_indexed_OP2
@ MULSUBv4i16_indexed_OP1
@ MULADDv4i32_indexed_OP2
@ MULSUBv2i32_indexed_OP2
@ MULADDv4i32_indexed_OP1
@ MULSUBv8i16_indexed_OP1
@ MULSUBv4i32_indexed_OP2
@ MULSUBv4i32_indexed_OP1
@ MULADDv8i16_indexed_OP1
@ MULADDv8i16_indexed_OP2
@ MULADDv4i16_indexed_OP2
@ MULADDv2i32_indexed_OP1
std::optional< UsedNZCV > examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > *CCUseInstrs=nullptr)
CodeGenOptLevel
Code generation optimization level.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
unsigned getUndefRegState(bool B)
unsigned getDefRegState(bool B)
unsigned getKillRegState(bool B)
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
DWARFExpression::Operation Op
static bool isUncondBranchOpcode(int Opc)
unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a SLEB128 value to an output stream.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx, unsigned FrameReg, StackOffset &Offset, const AArch64InstrInfo *TII)
rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the FP.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
static const MachineMemOperand::Flags MOSuppressPair
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI, const MachineInstr &UseMI, const TargetRegisterInfo *TRI)
Return true if there is an instruction /after/ DefMI and before UseMI which either reads or clobbers ...
static const MachineMemOperand::Flags MOStridedAccess
@ Default
The result values are uniform if and only if all operands are uniform.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
static bool recomputeLiveIns(MachineBasicBlock &MBB)
Convenience function for recomputing live-in's for a MBB.
Description of the encoding of one expression Op.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
static const MBBSectionID ColdSectionID
MachineJumpTableEntry - One jump table in the jump table info.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Used to describe a register and immediate addition.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.
unsigned FrameConstructionID
Target-defined identifier for constructing a frame for this function.