46#define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"
72 unsigned ContiguousOpc,
unsigned StridedOpc);
79 unsigned LdarOp,
unsigned StlrOp,
unsigned CmpOp,
80 unsigned ExtendImm,
unsigned ZeroReg,
104char AArch64ExpandPseudo::ID = 0;
116 assert(MO.isReg() && MO.getReg());
135 if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) {
138 MI.eraseFromParent();
148 bool LastItem = std::next(
I) ==
E;
153 case AArch64::ORRWri:
154 case AArch64::ORRXri:
157 .
add(
MI.getOperand(0))
158 .
addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR)
162 bool DstIsDead =
MI.getOperand(0).isDead();
172 case AArch64::ANDXri:
173 case AArch64::EORXri:
176 .
add(
MI.getOperand(0))
177 .
addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR)
181 bool DstIsDead =
MI.getOperand(0).isDead();
191 case AArch64::MOVNWi:
192 case AArch64::MOVNXi:
193 case AArch64::MOVZWi:
194 case AArch64::MOVZXi: {
195 bool DstIsDead =
MI.getOperand(0).isDead();
203 case AArch64::MOVKWi:
204 case AArch64::MOVKXi: {
206 bool DstIsDead =
MI.getOperand(0).isDead();
219 MI.eraseFromParent();
223bool AArch64ExpandPseudo::expandCMP_SWAP(
225 unsigned StlrOp,
unsigned CmpOp,
unsigned ExtendImm,
unsigned ZeroReg,
230 Register StatusReg =
MI.getOperand(1).getReg();
231 bool StatusDead =
MI.getOperand(1).isDead();
234 assert(!
MI.getOperand(2).isUndef() &&
"cannot handle undef");
236 Register DesiredReg =
MI.getOperand(3).getReg();
245 MF->
insert(++LoadCmpBB->getIterator(), StoreBB);
246 MF->
insert(++StoreBB->getIterator(), DoneBB);
254 BuildMI(LoadCmpBB, MIMD,
TII->get(AArch64::MOVZWi), StatusReg)
258 BuildMI(LoadCmpBB, MIMD,
TII->get(CmpOp), ZeroReg)
262 BuildMI(LoadCmpBB, MIMD,
TII->get(AArch64::Bcc))
266 LoadCmpBB->addSuccessor(DoneBB);
267 LoadCmpBB->addSuccessor(StoreBB);
272 BuildMI(StoreBB, MIMD,
TII->get(StlrOp), StatusReg)
275 BuildMI(StoreBB, MIMD,
TII->get(AArch64::CBNZW))
278 StoreBB->addSuccessor(LoadCmpBB);
279 StoreBB->addSuccessor(DoneBB);
281 DoneBB->splice(DoneBB->end(), &
MBB,
MI,
MBB.
end());
282 DoneBB->transferSuccessors(&
MBB);
287 MI.eraseFromParent();
295 StoreBB->clearLiveIns();
297 LoadCmpBB->clearLiveIns();
303bool AArch64ExpandPseudo::expandCMP_SWAP_128(
310 Register StatusReg =
MI.getOperand(2).getReg();
311 bool StatusDead =
MI.getOperand(2).isDead();
314 assert(!
MI.getOperand(3).isUndef() &&
"cannot handle undef");
316 Register DesiredLoReg =
MI.getOperand(4).getReg();
317 Register DesiredHiReg =
MI.getOperand(5).getReg();
318 Register NewLoReg =
MI.getOperand(6).getReg();
319 Register NewHiReg =
MI.getOperand(7).getReg();
321 unsigned LdxpOp, StxpOp;
323 switch (
MI.getOpcode()) {
324 case AArch64::CMP_SWAP_128_MONOTONIC:
325 LdxpOp = AArch64::LDXPX;
326 StxpOp = AArch64::STXPX;
328 case AArch64::CMP_SWAP_128_RELEASE:
329 LdxpOp = AArch64::LDXPX;
330 StxpOp = AArch64::STLXPX;
332 case AArch64::CMP_SWAP_128_ACQUIRE:
333 LdxpOp = AArch64::LDAXPX;
334 StxpOp = AArch64::STXPX;
336 case AArch64::CMP_SWAP_128:
337 LdxpOp = AArch64::LDAXPX;
338 StxpOp = AArch64::STLXPX;
351 MF->
insert(++LoadCmpBB->getIterator(), StoreBB);
352 MF->
insert(++StoreBB->getIterator(), FailBB);
353 MF->
insert(++FailBB->getIterator(), DoneBB);
364 BuildMI(LoadCmpBB, MIMD,
TII->get(AArch64::SUBSXrs), AArch64::XZR)
368 BuildMI(LoadCmpBB, MIMD,
TII->get(AArch64::CSINCWr), StatusReg)
372 BuildMI(LoadCmpBB, MIMD,
TII->get(AArch64::SUBSXrs), AArch64::XZR)
376 BuildMI(LoadCmpBB, MIMD,
TII->get(AArch64::CSINCWr), StatusReg)
380 BuildMI(LoadCmpBB, MIMD,
TII->get(AArch64::CBNZW))
383 LoadCmpBB->addSuccessor(FailBB);
384 LoadCmpBB->addSuccessor(StoreBB);
389 BuildMI(StoreBB, MIMD,
TII->get(StxpOp), StatusReg)
393 BuildMI(StoreBB, MIMD,
TII->get(AArch64::CBNZW))
397 StoreBB->addSuccessor(LoadCmpBB);
398 StoreBB->addSuccessor(DoneBB);
403 BuildMI(FailBB, MIMD,
TII->get(StxpOp), StatusReg)
407 BuildMI(FailBB, MIMD,
TII->get(AArch64::CBNZW))
410 FailBB->addSuccessor(LoadCmpBB);
411 FailBB->addSuccessor(DoneBB);
413 DoneBB->splice(DoneBB->end(), &
MBB,
MI,
MBB.
end());
414 DoneBB->transferSuccessors(&
MBB);
419 MI.eraseFromParent();
429 FailBB->clearLiveIns();
431 StoreBB->clearLiveIns();
433 LoadCmpBB->clearLiveIns();
477bool AArch64ExpandPseudo::expand_DestructiveOp(
486 bool DstIsDead =
MI.getOperand(0).isDead();
488 unsigned PredIdx, DOPIdx, SrcIdx, Src2Idx;
493 if (DstReg ==
MI.getOperand(3).getReg()) {
495 std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(1, 3, 2);
502 std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(1, 2, 3);
505 std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(2, 3, 3);
508 std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 2, 3, 4);
509 if (DstReg ==
MI.getOperand(3).getReg()) {
511 std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 3, 4, 2);
513 }
else if (DstReg ==
MI.getOperand(4).getReg()) {
515 std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 4, 3, 2);
526 bool DOPRegIsUnique =
false;
529 DOPRegIsUnique = DstReg !=
MI.getOperand(SrcIdx).getReg();
534 DstReg !=
MI.getOperand(DOPIdx).getReg() ||
535 MI.getOperand(DOPIdx).getReg() !=
MI.getOperand(SrcIdx).getReg();
539 DOPRegIsUnique =
true;
543 DstReg !=
MI.getOperand(DOPIdx).getReg() ||
544 (
MI.getOperand(DOPIdx).getReg() !=
MI.getOperand(SrcIdx).getReg() &&
545 MI.getOperand(DOPIdx).getReg() !=
MI.getOperand(Src2Idx).getReg());
562 unsigned MovPrfx, LSLZero, MovPrfxZero;
563 switch (ElementSize) {
566 MovPrfx = AArch64::MOVPRFX_ZZ;
567 LSLZero = AArch64::LSL_ZPmI_B;
568 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_B;
571 MovPrfx = AArch64::MOVPRFX_ZZ;
572 LSLZero = AArch64::LSL_ZPmI_H;
573 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_H;
576 MovPrfx = AArch64::MOVPRFX_ZZ;
577 LSLZero = AArch64::LSL_ZPmI_S;
578 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_S;
581 MovPrfx = AArch64::MOVPRFX_ZZ;
582 LSLZero = AArch64::LSL_ZPmI_D;
583 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_D;
599 "The destructive operand should be unique");
601 "This instruction is unpredicated");
606 .
addReg(
MI.getOperand(PredIdx).getReg())
607 .
addReg(
MI.getOperand(DOPIdx).getReg());
621 .
add(
MI.getOperand(PredIdx))
625 }
else if (DstReg !=
MI.getOperand(DOPIdx).getReg()) {
626 assert(DOPRegIsUnique &&
"The destructive operand should be unique");
629 .
addReg(
MI.getOperand(DOPIdx).getReg());
642 .
add(
MI.getOperand(PredIdx))
643 .
add(
MI.getOperand(SrcIdx));
649 DOP.
add(
MI.getOperand(PredIdx))
651 .
add(
MI.getOperand(SrcIdx));
654 DOP.
add(
MI.getOperand(PredIdx))
656 .
add(
MI.getOperand(SrcIdx))
657 .
add(
MI.getOperand(Src2Idx));
663 transferImpOps(
MI, PRFX, DOP);
665 transferImpOps(
MI, DOP, DOP);
667 MI.eraseFromParent();
671bool AArch64ExpandPseudo::expandSetTagLoop(
677 Register AddressReg =
MI.getOperand(1).getReg();
681 bool ZeroData =
MI.getOpcode() == AArch64::STZGloop_wback;
682 const unsigned OpCode1 =
683 ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex;
684 const unsigned OpCode2 =
685 ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex;
687 unsigned Size =
MI.getOperand(2).getImm();
689 if (
Size % (16 * 2) != 0) {
705 MF->
insert(++LoopBB->getIterator(), DoneBB);
724 LoopBB->addSuccessor(LoopBB);
725 LoopBB->addSuccessor(DoneBB);
727 DoneBB->splice(DoneBB->end(), &
MBB,
MI,
MBB.
end());
728 DoneBB->transferSuccessors(&
MBB);
733 MI.eraseFromParent();
740 LoopBB->clearLiveIns();
742 DoneBB->clearLiveIns();
750 unsigned Opc,
unsigned N) {
755 int ImmOffset =
MI.getOperand(2).getImm() +
Offset;
756 bool Kill = (
Offset + 1 ==
N) ?
MI.getOperand(1).isKill() :
false;
757 assert(ImmOffset >= -256 && ImmOffset < 256 &&
758 "Immediate spill offset out of range");
761 TRI->getSubReg(
MI.getOperand(0).getReg(), AArch64::zsub0 +
Offset),
766 MI.eraseFromParent();
770bool AArch64ExpandPseudo::expandCALL_RVMARKER(
783 "invalid operand for regular call");
784 assert(RVTarget.
isGlobal() &&
"invalid operand for attached call");
785 unsigned Opc = CallTarget.
isGlobal() ? AArch64::BL : AArch64::BLR;
789 unsigned RegMaskStartIdx = 2;
792 while (!
MI.getOperand(RegMaskStartIdx).isRegMask()) {
793 auto MOP =
MI.getOperand(RegMaskStartIdx);
794 assert(MOP.isReg() &&
"can only add register operands");
796 MOP.getReg(),
false,
true,
false,
797 false, MOP.isUndef()));
814 if (
MI.shouldUpdateCallSiteInfo())
817 MI.eraseFromParent();
819 std::next(RVCall->getIterator()));
834 "invalid operand for regular call");
835 unsigned Opc = CallTarget.
isGlobal() ? AArch64::BL : AArch64::BLR;
838 Call->addOperand(CallTarget);
848 if (
MI.shouldUpdateCallSiteInfo())
851 MI.eraseFromParent();
856bool AArch64ExpandPseudo::expandStoreSwiftAsyncContext(
864 if (STI.getTargetTriple().getArchName() !=
"arm64e") {
881 unsigned Opc =
Offset >= 0 ? AArch64::ADDXri : AArch64::SUBXri;
918 MI.getParent()->successors().begin() !=
919 MI.getParent()->successors().end()) &&
920 "Unexpected unreachable in block that restores ZA");
925 .
add(
MI.getOperand(0));
947 for (
unsigned I = 2;
I <
MI.getNumOperands(); ++
I)
948 MIB.
add(
MI.getOperand(
I));
951 MI.eraseFromParent();
965 MI.getParent()->successors().begin() ==
966 MI.getParent()->successors().end()) {
967 MI.eraseFromParent();
1009 auto PStateSM =
MI.getOperand(2).getReg();
1011 unsigned SMReg32 =
TRI->getSubReg(PStateSM, AArch64::sub_32);
1012 bool IsStreamingCallee =
MI.getOperand(3).getImm();
1013 unsigned Opc = IsStreamingCallee ? AArch64::TBZW : AArch64::TBNZW;
1035 TII->get(AArch64::MSRpstatesvcrImm1));
1039 MIB.
add(
MI.getOperand(0));
1040 MIB.
add(
MI.getOperand(1));
1041 for (
unsigned i = 4; i <
MI.getNumOperands(); ++i)
1042 MIB.
add(
MI.getOperand(i));
1046 MI.eraseFromParent();
1050bool AArch64ExpandPseudo::expandMultiVecPseudo(
1053 unsigned ContiguousOp,
unsigned StridedOpc) {
1068 .
add(
MI.getOperand(0))
1069 .
add(
MI.getOperand(1))
1070 .
add(
MI.getOperand(2))
1071 .
add(
MI.getOperand(3));
1072 transferImpOps(
MI, MIB, MIB);
1073 MI.eraseFromParent();
1087 if (OrigInstr != -1) {
1088 auto &Orig =
TII->get(OrigInstr);
1091 return expand_DestructiveOp(
MI,
MBB,
MBBI);
1099 case AArch64::BSPv8i8:
1100 case AArch64::BSPv16i8: {
1102 if (DstReg ==
MI.getOperand(3).getReg()) {
1105 TII->get(
Opcode == AArch64::BSPv8i8 ? AArch64::BITv8i8
1106 : AArch64::BITv16i8))
1107 .
add(
MI.getOperand(0))
1108 .
add(
MI.getOperand(3))
1109 .
add(
MI.getOperand(2))
1110 .
add(
MI.getOperand(1));
1111 }
else if (DstReg ==
MI.getOperand(2).getReg()) {
1114 TII->get(
Opcode == AArch64::BSPv8i8 ? AArch64::BIFv8i8
1115 : AArch64::BIFv16i8))
1116 .
add(
MI.getOperand(0))
1117 .
add(
MI.getOperand(2))
1118 .
add(
MI.getOperand(3))
1119 .
add(
MI.getOperand(1));
1122 if (DstReg ==
MI.getOperand(1).getReg()) {
1124 TII->get(
Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1125 : AArch64::BSLv16i8))
1126 .
add(
MI.getOperand(0))
1127 .
add(
MI.getOperand(1))
1128 .
add(
MI.getOperand(2))
1129 .
add(
MI.getOperand(3));
1132 TII->get(
Opcode == AArch64::BSPv8i8 ? AArch64::ORRv8i8
1133 : AArch64::ORRv16i8))
1137 .
add(
MI.getOperand(1))
1138 .
add(
MI.getOperand(1));
1140 TII->get(
Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1141 : AArch64::BSLv16i8))
1142 .
add(
MI.getOperand(0))
1146 .
add(
MI.getOperand(2))
1147 .
add(
MI.getOperand(3));
1150 MI.eraseFromParent();
1154 case AArch64::ADDWrr:
1155 case AArch64::SUBWrr:
1156 case AArch64::ADDXrr:
1157 case AArch64::SUBXrr:
1158 case AArch64::ADDSWrr:
1159 case AArch64::SUBSWrr:
1160 case AArch64::ADDSXrr:
1161 case AArch64::SUBSXrr:
1162 case AArch64::ANDWrr:
1163 case AArch64::ANDXrr:
1164 case AArch64::BICWrr:
1165 case AArch64::BICXrr:
1166 case AArch64::ANDSWrr:
1167 case AArch64::ANDSXrr:
1168 case AArch64::BICSWrr:
1169 case AArch64::BICSXrr:
1170 case AArch64::EONWrr:
1171 case AArch64::EONXrr:
1172 case AArch64::EORWrr:
1173 case AArch64::EORXrr:
1174 case AArch64::ORNWrr:
1175 case AArch64::ORNXrr:
1176 case AArch64::ORRWrr:
1177 case AArch64::ORRXrr: {
1179 switch (
MI.getOpcode()) {
1182 case AArch64::ADDWrr:
Opcode = AArch64::ADDWrs;
break;
1183 case AArch64::SUBWrr:
Opcode = AArch64::SUBWrs;
break;
1184 case AArch64::ADDXrr:
Opcode = AArch64::ADDXrs;
break;
1185 case AArch64::SUBXrr:
Opcode = AArch64::SUBXrs;
break;
1186 case AArch64::ADDSWrr:
Opcode = AArch64::ADDSWrs;
break;
1187 case AArch64::SUBSWrr:
Opcode = AArch64::SUBSWrs;
break;
1188 case AArch64::ADDSXrr:
Opcode = AArch64::ADDSXrs;
break;
1189 case AArch64::SUBSXrr:
Opcode = AArch64::SUBSXrs;
break;
1190 case AArch64::ANDWrr:
Opcode = AArch64::ANDWrs;
break;
1191 case AArch64::ANDXrr:
Opcode = AArch64::ANDXrs;
break;
1192 case AArch64::BICWrr:
Opcode = AArch64::BICWrs;
break;
1193 case AArch64::BICXrr:
Opcode = AArch64::BICXrs;
break;
1194 case AArch64::ANDSWrr:
Opcode = AArch64::ANDSWrs;
break;
1195 case AArch64::ANDSXrr:
Opcode = AArch64::ANDSXrs;
break;
1196 case AArch64::BICSWrr:
Opcode = AArch64::BICSWrs;
break;
1197 case AArch64::BICSXrr:
Opcode = AArch64::BICSXrs;
break;
1198 case AArch64::EONWrr:
Opcode = AArch64::EONWrs;
break;
1199 case AArch64::EONXrr:
Opcode = AArch64::EONXrs;
break;
1200 case AArch64::EORWrr:
Opcode = AArch64::EORWrs;
break;
1201 case AArch64::EORXrr:
Opcode = AArch64::EORXrs;
break;
1202 case AArch64::ORNWrr:
Opcode = AArch64::ORNWrs;
break;
1203 case AArch64::ORNXrr:
Opcode = AArch64::ORNXrs;
break;
1204 case AArch64::ORRWrr:
Opcode = AArch64::ORRWrs;
break;
1205 case AArch64::ORRXrr:
Opcode = AArch64::ORRXrs;
break;
1213 MIB1->setPCSections(MF,
MI.getPCSections());
1215 .add(
MI.getOperand(1))
1216 .add(
MI.getOperand(2))
1218 transferImpOps(
MI, MIB1, MIB1);
1219 if (
auto DebugNumber =
MI.peekDebugInstrNum())
1221 MI.eraseFromParent();
1225 case AArch64::LOADgot: {
1234 TII->get(AArch64::LDRXl), DstReg);
1242 "Only expect globals, externalsymbols, or constant pools");
1255 unsigned Reg32 =
TRI->getSubReg(DstReg, AArch64::sub_32);
1256 unsigned DstFlags =
MI.getOperand(0).getTargetFlags();
1264 .
add(
MI.getOperand(0))
1279 "Only expect globals, externalsymbols, or constant pools");
1287 transferImpOps(
MI, MIB1, MIB2);
1289 MI.eraseFromParent();
1292 case AArch64::MOVaddrBA: {
1299 assert(
MI.getOperand(1).getOffset() == 0 &&
"unexpected offset");
1309 TII->get(AArch64::LDRXui), DstReg)
1313 transferImpOps(
MI, MIB1, MIB2);
1314 MI.eraseFromParent();
1319 case AArch64::MOVaddr:
1320 case AArch64::MOVaddrJT:
1321 case AArch64::MOVaddrCP:
1322 case AArch64::MOVaddrTLS:
1323 case AArch64::MOVaddrEXT: {
1326 assert(DstReg != AArch64::XZR);
1329 .
add(
MI.getOperand(1));
1339 auto Tag =
MI.getOperand(1);
1341 Tag.setOffset(0x100000000);
1350 .
add(
MI.getOperand(0))
1352 .
add(
MI.getOperand(2))
1355 transferImpOps(
MI, MIB1, MIB2);
1356 MI.eraseFromParent();
1359 case AArch64::ADDlowTLS:
1362 .
add(
MI.getOperand(0))
1363 .
add(
MI.getOperand(1))
1364 .
add(
MI.getOperand(2))
1366 MI.eraseFromParent();
1369 case AArch64::MOVbaseTLS: {
1371 auto SysReg = AArch64SysReg::TPIDR_EL0;
1374 SysReg = AArch64SysReg::TPIDR_EL3;
1376 SysReg = AArch64SysReg::TPIDR_EL2;
1378 SysReg = AArch64SysReg::TPIDR_EL1;
1380 SysReg = AArch64SysReg::TPIDRRO_EL0;
1383 MI.eraseFromParent();
1387 case AArch64::MOVi32imm:
1389 case AArch64::MOVi64imm:
1391 case AArch64::RET_ReallyLR: {
1400 transferImpOps(
MI, MIB, MIB);
1401 MI.eraseFromParent();
1404 case AArch64::CMP_SWAP_8:
1405 return expandCMP_SWAP(
MBB,
MBBI, AArch64::LDAXRB, AArch64::STLXRB,
1408 AArch64::WZR, NextMBBI);
1409 case AArch64::CMP_SWAP_16:
1410 return expandCMP_SWAP(
MBB,
MBBI, AArch64::LDAXRH, AArch64::STLXRH,
1413 AArch64::WZR, NextMBBI);
1414 case AArch64::CMP_SWAP_32:
1415 return expandCMP_SWAP(
MBB,
MBBI, AArch64::LDAXRW, AArch64::STLXRW,
1418 AArch64::WZR, NextMBBI);
1419 case AArch64::CMP_SWAP_64:
1420 return expandCMP_SWAP(
MBB,
MBBI,
1421 AArch64::LDAXRX, AArch64::STLXRX, AArch64::SUBSXrs,
1423 AArch64::XZR, NextMBBI);
1424 case AArch64::CMP_SWAP_128:
1425 case AArch64::CMP_SWAP_128_RELEASE:
1426 case AArch64::CMP_SWAP_128_ACQUIRE:
1427 case AArch64::CMP_SWAP_128_MONOTONIC:
1428 return expandCMP_SWAP_128(
MBB,
MBBI, NextMBBI);
1430 case AArch64::AESMCrrTied:
1431 case AArch64::AESIMCrrTied: {
1434 TII->get(
Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :
1436 .
add(
MI.getOperand(0))
1437 .
add(
MI.getOperand(1));
1438 transferImpOps(
MI, MIB, MIB);
1439 MI.eraseFromParent();
1442 case AArch64::IRGstack: {
1453 StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference(
1454 MF, BaseOffset,
false ,
false , FrameReg,
1458 if (FrameRegOffset) {
1460 SrcReg =
MI.getOperand(0).getReg();
1462 FrameRegOffset,
TII);
1465 .
add(
MI.getOperand(0))
1467 .
add(
MI.getOperand(2));
1468 MI.eraseFromParent();
1471 case AArch64::TAGPstack: {
1472 int64_t
Offset =
MI.getOperand(2).getImm();
1474 TII->get(
Offset >= 0 ? AArch64::ADDG : AArch64::SUBG))
1475 .
add(
MI.getOperand(0))
1476 .
add(
MI.getOperand(1))
1478 .
add(
MI.getOperand(4));
1479 MI.eraseFromParent();
1482 case AArch64::STGloop_wback:
1483 case AArch64::STZGloop_wback:
1484 return expandSetTagLoop(
MBB,
MBBI, NextMBBI);
1485 case AArch64::STGloop:
1486 case AArch64::STZGloop:
1488 "Non-writeback variants of STGloop / STZGloop should not "
1489 "survive past PrologEpilogInserter.");
1490 case AArch64::STR_ZZZZXI:
1491 return expandSVESpillFill(
MBB,
MBBI, AArch64::STR_ZXI, 4);
1492 case AArch64::STR_ZZZXI:
1493 return expandSVESpillFill(
MBB,
MBBI, AArch64::STR_ZXI, 3);
1494 case AArch64::STR_ZZXI:
1495 return expandSVESpillFill(
MBB,
MBBI, AArch64::STR_ZXI, 2);
1496 case AArch64::LDR_ZZZZXI:
1497 return expandSVESpillFill(
MBB,
MBBI, AArch64::LDR_ZXI, 4);
1498 case AArch64::LDR_ZZZXI:
1499 return expandSVESpillFill(
MBB,
MBBI, AArch64::LDR_ZXI, 3);
1500 case AArch64::LDR_ZZXI:
1501 return expandSVESpillFill(
MBB,
MBBI, AArch64::LDR_ZXI, 2);
1502 case AArch64::BLR_RVMARKER:
1503 return expandCALL_RVMARKER(
MBB,
MBBI);
1504 case AArch64::BLR_BTI:
1505 return expandCALL_BTI(
MBB,
MBBI);
1506 case AArch64::StoreSwiftAsyncContext:
1507 return expandStoreSwiftAsyncContext(
MBB,
MBBI);
1508 case AArch64::RestoreZAPseudo: {
1509 auto *NewMBB = expandRestoreZA(
MBB,
MBBI);
1514 case AArch64::MSRpstatePseudo: {
1515 auto *NewMBB = expandCondSMToggle(
MBB,
MBBI);
1520 case AArch64::LD1B_2Z_IMM_PSEUDO:
1521 return expandMultiVecPseudo(
1522 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1523 AArch64::LD1B_2Z_IMM, AArch64::LD1B_2Z_STRIDED_IMM);
1524 case AArch64::LD1H_2Z_IMM_PSEUDO:
1525 return expandMultiVecPseudo(
1526 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1527 AArch64::LD1H_2Z_IMM, AArch64::LD1H_2Z_STRIDED_IMM);
1528 case AArch64::LD1W_2Z_IMM_PSEUDO:
1529 return expandMultiVecPseudo(
1530 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1531 AArch64::LD1W_2Z_IMM, AArch64::LD1W_2Z_STRIDED_IMM);
1532 case AArch64::LD1D_2Z_IMM_PSEUDO:
1533 return expandMultiVecPseudo(
1534 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1535 AArch64::LD1D_2Z_IMM, AArch64::LD1D_2Z_STRIDED_IMM);
1536 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
1537 return expandMultiVecPseudo(
1538 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1539 AArch64::LDNT1B_2Z_IMM, AArch64::LDNT1B_2Z_STRIDED_IMM);
1540 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
1541 return expandMultiVecPseudo(
1542 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1543 AArch64::LDNT1H_2Z_IMM, AArch64::LDNT1H_2Z_STRIDED_IMM);
1544 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
1545 return expandMultiVecPseudo(
1546 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1547 AArch64::LDNT1W_2Z_IMM, AArch64::LDNT1W_2Z_STRIDED_IMM);
1548 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
1549 return expandMultiVecPseudo(
1550 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1551 AArch64::LDNT1D_2Z_IMM, AArch64::LDNT1D_2Z_STRIDED_IMM);
1552 case AArch64::LD1B_2Z_PSEUDO:
1553 return expandMultiVecPseudo(
MBB,
MBBI, AArch64::ZPR2RegClass,
1554 AArch64::ZPR2StridedRegClass, AArch64::LD1B_2Z,
1555 AArch64::LD1B_2Z_STRIDED);
1556 case AArch64::LD1H_2Z_PSEUDO:
1557 return expandMultiVecPseudo(
MBB,
MBBI, AArch64::ZPR2RegClass,
1558 AArch64::ZPR2StridedRegClass, AArch64::LD1H_2Z,
1559 AArch64::LD1H_2Z_STRIDED);
1560 case AArch64::LD1W_2Z_PSEUDO:
1561 return expandMultiVecPseudo(
MBB,
MBBI, AArch64::ZPR2RegClass,
1562 AArch64::ZPR2StridedRegClass, AArch64::LD1W_2Z,
1563 AArch64::LD1W_2Z_STRIDED);
1564 case AArch64::LD1D_2Z_PSEUDO:
1565 return expandMultiVecPseudo(
MBB,
MBBI, AArch64::ZPR2RegClass,
1566 AArch64::ZPR2StridedRegClass, AArch64::LD1D_2Z,
1567 AArch64::LD1D_2Z_STRIDED);
1568 case AArch64::LDNT1B_2Z_PSEUDO:
1569 return expandMultiVecPseudo(
1570 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1571 AArch64::LDNT1B_2Z, AArch64::LDNT1B_2Z_STRIDED);
1572 case AArch64::LDNT1H_2Z_PSEUDO:
1573 return expandMultiVecPseudo(
1574 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1575 AArch64::LDNT1H_2Z, AArch64::LDNT1H_2Z_STRIDED);
1576 case AArch64::LDNT1W_2Z_PSEUDO:
1577 return expandMultiVecPseudo(
1578 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1579 AArch64::LDNT1W_2Z, AArch64::LDNT1W_2Z_STRIDED);
1580 case AArch64::LDNT1D_2Z_PSEUDO:
1581 return expandMultiVecPseudo(
1582 MBB,
MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1583 AArch64::LDNT1D_2Z, AArch64::LDNT1D_2Z_STRIDED);
1584 case AArch64::LD1B_4Z_IMM_PSEUDO:
1585 return expandMultiVecPseudo(
1586 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1587 AArch64::LD1B_4Z_IMM, AArch64::LD1B_4Z_STRIDED_IMM);
1588 case AArch64::LD1H_4Z_IMM_PSEUDO:
1589 return expandMultiVecPseudo(
1590 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1591 AArch64::LD1H_4Z_IMM, AArch64::LD1H_4Z_STRIDED_IMM);
1592 case AArch64::LD1W_4Z_IMM_PSEUDO:
1593 return expandMultiVecPseudo(
1594 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1595 AArch64::LD1W_4Z_IMM, AArch64::LD1W_4Z_STRIDED_IMM);
1596 case AArch64::LD1D_4Z_IMM_PSEUDO:
1597 return expandMultiVecPseudo(
1598 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1599 AArch64::LD1D_4Z_IMM, AArch64::LD1D_4Z_STRIDED_IMM);
1600 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
1601 return expandMultiVecPseudo(
1602 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1603 AArch64::LDNT1B_4Z_IMM, AArch64::LDNT1B_4Z_STRIDED_IMM);
1604 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
1605 return expandMultiVecPseudo(
1606 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1607 AArch64::LDNT1H_4Z_IMM, AArch64::LDNT1H_4Z_STRIDED_IMM);
1608 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
1609 return expandMultiVecPseudo(
1610 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1611 AArch64::LDNT1W_4Z_IMM, AArch64::LDNT1W_4Z_STRIDED_IMM);
1612 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
1613 return expandMultiVecPseudo(
1614 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1615 AArch64::LDNT1D_4Z_IMM, AArch64::LDNT1D_4Z_STRIDED_IMM);
1616 case AArch64::LD1B_4Z_PSEUDO:
1617 return expandMultiVecPseudo(
MBB,
MBBI, AArch64::ZPR4RegClass,
1618 AArch64::ZPR4StridedRegClass, AArch64::LD1B_4Z,
1619 AArch64::LD1B_4Z_STRIDED);
1620 case AArch64::LD1H_4Z_PSEUDO:
1621 return expandMultiVecPseudo(
MBB,
MBBI, AArch64::ZPR4RegClass,
1622 AArch64::ZPR4StridedRegClass, AArch64::LD1H_4Z,
1623 AArch64::LD1H_4Z_STRIDED);
1624 case AArch64::LD1W_4Z_PSEUDO:
1625 return expandMultiVecPseudo(
MBB,
MBBI, AArch64::ZPR4RegClass,
1626 AArch64::ZPR4StridedRegClass, AArch64::LD1W_4Z,
1627 AArch64::LD1W_4Z_STRIDED);
1628 case AArch64::LD1D_4Z_PSEUDO:
1629 return expandMultiVecPseudo(
MBB,
MBBI, AArch64::ZPR4RegClass,
1630 AArch64::ZPR4StridedRegClass, AArch64::LD1D_4Z,
1631 AArch64::LD1D_4Z_STRIDED);
1632 case AArch64::LDNT1B_4Z_PSEUDO:
1633 return expandMultiVecPseudo(
1634 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1635 AArch64::LDNT1B_4Z, AArch64::LDNT1B_4Z_STRIDED);
1636 case AArch64::LDNT1H_4Z_PSEUDO:
1637 return expandMultiVecPseudo(
1638 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1639 AArch64::LDNT1H_4Z, AArch64::LDNT1H_4Z_STRIDED);
1640 case AArch64::LDNT1W_4Z_PSEUDO:
1641 return expandMultiVecPseudo(
1642 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1643 AArch64::LDNT1W_4Z, AArch64::LDNT1W_4Z_STRIDED);
1644 case AArch64::LDNT1D_4Z_PSEUDO:
1645 return expandMultiVecPseudo(
1646 MBB,
MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1647 AArch64::LDNT1D_4Z, AArch64::LDNT1D_4Z_STRIDED);
1671 for (
auto &
MBB : MF)
1678 return new AArch64ExpandPseudo();
#define AARCH64_EXPAND_PSEUDO_NAME
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static constexpr uint32_t Opcode
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
unsigned getTaggedBasePointerOffset() const
bool isTargetILP32() const
bool isTargetMachO() const
The address of a basic block.
FunctionPass class - This class is used to implement most global optimizations.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Describe properties that are true of each instruction in the target description file.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImplicit=false)
CreateMachineInstr - Allocate a new MachineInstr.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
void moveCallSiteInfo(const MachineInstr *Old, const MachineInstr *New)
Move the call site info from Old to \New call site info.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
void setDebugInstrNum(unsigned Num)
Set instruction number of this MachineInstr.
void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isCPI() const
isCPI - Tests if this is a MO_ConstantPoolIndex operand.
bool isSymbol() const
isSymbol - Tests if this is a MO_ExternalSymbol operand.
unsigned getTargetFlags() const
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
const char * getSymbolName() const
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
int64_t getOffset() const
Return the offset from the symbol in this operand.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
StringRef - Represent a constant reference to a string, i.e.
CodeModel::Model getCodeModel() const
Returns the code model.
iterator_range< SmallVectorImpl< MCPhysReg >::const_iterator > getRegisters() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
int getSVERevInstr(uint16_t Opcode)
@ DestructiveInstTypeMask
@ DestructiveUnaryPassthru
@ DestructiveTernaryCommWithRev
@ DestructiveBinaryCommWithRev
int getSVEPseudoMap(uint16_t Opcode)
int getSVENonRevInstr(uint16_t Opcode)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
APFloat abs(APFloat X)
Returns the absolute value of the argument.
unsigned getDeadRegState(bool B)
void initializeAArch64ExpandPseudoPass(PassRegistry &)
FunctionPass * createAArch64ExpandPseudoPass()
Returns an instance of the pseudo instruction expansion pass.
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
unsigned getKillRegState(bool B)
unsigned getRenamableRegState(bool B)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().
This struct is a compact representation of a valid (non-zero power of two) alignment.
Description of the encoding of one expression Op.