236#define DEBUG_TYPE "frame-info"
239 cl::desc(
"enable use of redzone on AArch64"),
243 "stack-tagging-merge-settag",
253 cl::desc(
"Emit homogeneous prologue and epilogue for the size "
254 "optimization (default = off)"));
256STATISTIC(NumRedZoneFunctions,
"Number of functions using red zone");
272 int64_t ArgumentPopSize = 0;
273 if (IsTailCallReturn) {
279 ArgumentPopSize = StackAdjust.
getImm();
288 return ArgumentPopSize;
299bool AArch64FrameLowering::homogeneousPrologEpilog(
324 if (AFI->hasSwiftAsyncContext())
331 unsigned NumGPRs = 0;
332 for (
unsigned I = 0; CSRegs[
I]; ++
I) {
334 if (Reg == AArch64::LR) {
335 assert(CSRegs[
I + 1] == AArch64::FP);
336 if (NumGPRs % 2 != 0)
340 if (AArch64::GPR64RegClass.
contains(Reg))
348bool AArch64FrameLowering::producePairRegisters(
MachineFunction &MF)
const {
367 if (
MI.isDebugInstr() ||
MI.isPseudo() ||
368 MI.getOpcode() == AArch64::ADDXri ||
369 MI.getOpcode() == AArch64::ADDSXri)
396 if (!IsWin64 || IsFunclet) {
401 Attribute::SwiftAsync))
406 const unsigned UnwindHelpObject = (MF.
hasEHFunclets() ? 8 : 0);
408 alignTo(VarArgsArea + UnwindHelpObject, 16);
425 const unsigned RedZoneSize =
434 return !(MFI.
hasCalls() ||
hasFP(MF) || NumBytes > RedZoneSize ||
495 unsigned Opc =
I->getOpcode();
496 bool IsDestroy = Opc ==
TII->getCallFrameDestroyOpcode();
497 uint64_t CalleePopAmount = IsDestroy ?
I->getOperand(1).getImm() : 0;
500 int64_t Amount =
I->getOperand(0).getImm();
508 if (CalleePopAmount == 0) {
519 assert(Amount > -0xffffff && Amount < 0xffffff &&
"call frame too large");
530 "non-reserved call frame without var sized objects?");
539 }
else if (CalleePopAmount != 0) {
542 assert(CalleePopAmount < 0xffffff &&
"call frame too large");
549void AArch64FrameLowering::emitCalleeSavedGPRLocations(
563 for (
const auto &Info : CSI) {
567 assert(!
Info.isSpilledToReg() &&
"Spilling to registers not implemented");
568 unsigned DwarfReg =
TRI.getDwarfRegNum(
Info.getReg(),
true);
580void AArch64FrameLowering::emitCalleeSavedSVELocations(
596 for (
const auto &Info : CSI) {
602 assert(!
Info.isSpilledToReg() &&
"Spilling to registers not implemented");
637 const MCInstrDesc &CFIDesc =
TII.get(TargetOpcode::CFI_INSTRUCTION);
643 nullptr,
TRI.getDwarfRegNum(AArch64::SP,
true), 0));
647 if (MFI.shouldSignReturnAddress(MF)) {
653 if (MFI.needsShadowCallStackPrologueEpilogue(MF))
655 TRI.getDwarfRegNum(AArch64::X18,
true));
658 const std::vector<CalleeSavedInfo> &CSI =
660 for (
const auto &
Info : CSI) {
661 unsigned Reg =
Info.getReg();
662 if (!
TRI.regNeedsCFI(Reg, Reg))
665 TRI.getDwarfRegNum(Reg,
true));
684 for (
const auto &
Info : CSI) {
689 unsigned Reg =
Info.getReg();
695 nullptr,
TRI.getDwarfRegNum(
Info.getReg(),
true)));
702void AArch64FrameLowering::emitCalleeSavedGPRRestores(
707void AArch64FrameLowering::emitCalleeSavedSVERestores(
715 static const int64_t MAX_BYTES_PER_SCALABLE_BYTE = 16;
716 return Size.getScalable() * MAX_BYTES_PER_SCALABLE_BYTE +
Size.getFixed();
719void AArch64FrameLowering::allocateStackSpace(
721 int64_t RealignmentPadding,
StackOffset AllocSize,
bool NeedsWinCFI,
722 bool *HasWinCFI,
bool EmitCFI,
StackOffset InitialOffset,
723 bool FollowupAllocs)
const {
736 const uint64_t AndMask = ~(MaxAlign - 1);
739 Register TargetReg = RealignmentPadding
745 EmitCFI, InitialOffset);
747 if (RealignmentPadding) {
768 if (AllocSize.
getScalable() == 0 && RealignmentPadding == 0) {
770 assert(ScratchReg != AArch64::NoRegister);
780 if (FollowupAllocs) {
797 if (
upperBound(AllocSize) + RealignmentPadding <= ProbeSize) {
798 Register ScratchReg = RealignmentPadding
801 assert(ScratchReg != AArch64::NoRegister);
805 EmitCFI, InitialOffset);
806 if (RealignmentPadding) {
814 if (FollowupAllocs ||
upperBound(AllocSize) + RealignmentPadding >
830 assert(TargetReg != AArch64::NoRegister);
834 EmitCFI, InitialOffset);
835 if (RealignmentPadding) {
855 if (RealignmentPadding)
868 case AArch64::W##n: \
869 case AArch64::X##n: \
894 case AArch64::B##n: \
895 case AArch64::H##n: \
896 case AArch64::S##n: \
897 case AArch64::D##n: \
898 case AArch64::Q##n: \
899 return HasSVE ? AArch64::Z##n : AArch64::Q##n
936void AArch64FrameLowering::emitZeroCallUsedRegs(
BitVector RegsToZero,
952 bool HasSVE = STI.hasSVE();
954 if (
TRI.isGeneralPurposeRegister(MF, Reg)) {
957 GPRsToZero.set(XReg);
958 }
else if (AArch64::FPR128RegClass.
contains(Reg) ||
959 AArch64::FPR64RegClass.
contains(Reg) ||
960 AArch64::FPR32RegClass.
contains(Reg) ||
961 AArch64::FPR16RegClass.
contains(Reg) ||
962 AArch64::FPR8RegClass.
contains(Reg)) {
965 FPRsToZero.set(XReg);
981 {AArch64::P0, AArch64::P1, AArch64::P2, AArch64::P3, AArch64::P4,
982 AArch64::P5, AArch64::P6, AArch64::P7, AArch64::P8, AArch64::P9,
983 AArch64::P10, AArch64::P11, AArch64::P12, AArch64::P13, AArch64::P14,
985 if (RegsToZero[PReg])
997 for (
unsigned i = 0; CSRegs[i]; ++i)
998 LiveRegs.
addReg(CSRegs[i]);
1029 for (
unsigned Reg : AArch64::GPR64RegClass) {
1033 return AArch64::NoRegister;
1079 StackSizeInBytes >=
uint64_t(MFI.getStackProbeSize());
1085 F.needsUnwindTableEntry();
1088bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
1094 if (homogeneousPrologEpilog(MF))
1117 if (MFI.hasVarSizedObjects())
1120 if (
RegInfo->hasStackRealignment(MF))
1137bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
1139 if (!shouldCombineCSRLocalStackBump(*
MBB.
getParent(), StackBumpBytes))
1149 while (LastI != Begin) {
1151 if (LastI->isTransient())
1156 switch (LastI->getOpcode()) {
1157 case AArch64::STGloop:
1158 case AArch64::STZGloop:
1160 case AArch64::STZGi:
1161 case AArch64::ST2Gi:
1162 case AArch64::STZ2Gi:
1175 unsigned Opc =
MBBI->getOpcode();
1179 unsigned ImmIdx =
MBBI->getNumOperands() - 1;
1180 int Imm =
MBBI->getOperand(ImmIdx).getImm();
1188 case AArch64::LDPDpost:
1191 case AArch64::STPDpre: {
1192 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1193 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(2).getReg());
1194 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFRegP_X))
1201 case AArch64::LDPXpost:
1204 case AArch64::STPXpre: {
1207 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1208 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFPLR_X))
1212 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveRegP_X))
1219 case AArch64::LDRDpost:
1222 case AArch64::STRDpre: {
1223 unsigned Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1224 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFReg_X))
1230 case AArch64::LDRXpost:
1233 case AArch64::STRXpre: {
1234 unsigned Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1241 case AArch64::STPDi:
1242 case AArch64::LDPDi: {
1243 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1244 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1252 case AArch64::STPXi:
1253 case AArch64::LDPXi: {
1256 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1268 case AArch64::STRXui:
1269 case AArch64::LDRXui: {
1270 int Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1277 case AArch64::STRDui:
1278 case AArch64::LDRDui: {
1279 unsigned Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1286 case AArch64::STPQi:
1287 case AArch64::LDPQi: {
1288 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1289 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1290 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegQP))
1297 case AArch64::LDPQpost:
1300 case AArch64::STPQpre: {
1301 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1302 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(2).getReg());
1303 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegQPX))
1317 unsigned LocalStackSize) {
1319 unsigned ImmIdx =
MBBI->getNumOperands() - 1;
1320 switch (
MBBI->getOpcode()) {
1323 case AArch64::SEH_SaveFPLR:
1324 case AArch64::SEH_SaveRegP:
1325 case AArch64::SEH_SaveReg:
1326 case AArch64::SEH_SaveFRegP:
1327 case AArch64::SEH_SaveFReg:
1328 case AArch64::SEH_SaveAnyRegQP:
1329 case AArch64::SEH_SaveAnyRegQPX:
1330 ImmOpnd = &
MBBI->getOperand(ImmIdx);
1343 bool NeedsWinCFI,
bool *HasWinCFI,
bool EmitCFI,
1345 int CFAOffset = 0) {
1347 switch (
MBBI->getOpcode()) {
1350 case AArch64::STPXi:
1351 NewOpc = AArch64::STPXpre;
1353 case AArch64::STPDi:
1354 NewOpc = AArch64::STPDpre;
1356 case AArch64::STPQi:
1357 NewOpc = AArch64::STPQpre;
1359 case AArch64::STRXui:
1360 NewOpc = AArch64::STRXpre;
1362 case AArch64::STRDui:
1363 NewOpc = AArch64::STRDpre;
1365 case AArch64::STRQui:
1366 NewOpc = AArch64::STRQpre;
1368 case AArch64::LDPXi:
1369 NewOpc = AArch64::LDPXpost;
1371 case AArch64::LDPDi:
1372 NewOpc = AArch64::LDPDpost;
1374 case AArch64::LDPQi:
1375 NewOpc = AArch64::LDPQpost;
1377 case AArch64::LDRXui:
1378 NewOpc = AArch64::LDRXpost;
1380 case AArch64::LDRDui:
1381 NewOpc = AArch64::LDRDpost;
1383 case AArch64::LDRQui:
1384 NewOpc = AArch64::LDRQpost;
1389 auto SEH = std::next(
MBBI);
1391 SEH->eraseFromParent();
1395 int64_t MinOffset, MaxOffset;
1397 NewOpc, Scale, Width, MinOffset, MaxOffset);
1404 if (
MBBI->getOperand(
MBBI->getNumOperands() - 1).getImm() != 0 ||
1405 CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) {
1408 false,
false,
nullptr, EmitCFI,
1411 return std::prev(
MBBI);
1418 unsigned OpndIdx = 0;
1419 for (
unsigned OpndEnd =
MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
1421 MIB.
add(
MBBI->getOperand(OpndIdx));
1423 assert(
MBBI->getOperand(OpndIdx).getImm() == 0 &&
1424 "Unexpected immediate offset in first/last callee-save save/restore "
1426 assert(
MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
1427 "Unexpected base register in callee-save save/restore instruction!");
1428 assert(CSStackSizeInc % Scale == 0);
1429 MIB.
addImm(CSStackSizeInc / (
int)Scale);
1460 unsigned Opc =
MI.getOpcode();
1463 case AArch64::STPXi:
1464 case AArch64::STRXui:
1465 case AArch64::STPDi:
1466 case AArch64::STRDui:
1467 case AArch64::LDPXi:
1468 case AArch64::LDRXui:
1469 case AArch64::LDPDi:
1470 case AArch64::LDRDui:
1473 case AArch64::STPQi:
1474 case AArch64::STRQui:
1475 case AArch64::LDPQi:
1476 case AArch64::LDRQui:
1483 unsigned OffsetIdx =
MI.getNumExplicitOperands() - 1;
1484 assert(
MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
1485 "Unexpected base register in callee-save save/restore instruction!");
1489 assert(LocalStackSize % Scale == 0);
1490 OffsetOpnd.
setImm(OffsetOpnd.
getImm() + LocalStackSize / Scale);
1495 assert(
MBBI !=
MI.getParent()->end() &&
"Expecting a valid instruction");
1497 "Expecting a SEH instruction");
1508 switch (
I->getOpcode()) {
1511 case AArch64::STR_ZXI:
1512 case AArch64::STR_PXI:
1513 case AArch64::LDR_ZXI:
1514 case AArch64::LDR_PXI:
1525 bool NeedsUnwindInfo) {
1541 if (NeedsUnwindInfo) {
1544 static const char CFIInst[] = {
1545 dwarf::DW_CFA_val_expression,
1548 static_cast<char>(
unsigned(dwarf::DW_OP_breg18)),
1549 static_cast<char>(-8) & 0x7f,
1552 nullptr,
StringRef(CFIInst,
sizeof(CFIInst))));
1590 const int OffsetToFirstCalleeSaveFromFP =
1594 unsigned Reg =
TRI->getDwarfRegNum(
FramePtr,
true);
1596 nullptr, Reg, FixedObject - OffsetToFirstCalleeSaveFromFP));
1629 bool HasFP =
hasFP(MF);
1631 bool HasWinCFI =
false;
1640 while (NonFrameStart !=
End &&
1645 if (NonFrameStart !=
MBB.
end()) {
1655 if (NonFrameStart ==
MBB.
end())
1660 for (auto &Op : MI.operands())
1661 if (Op.isReg() && Op.isDef())
1662 assert(!LiveRegs.contains(Op.getReg()) &&
1663 "live register clobbered by inserted prologue instructions");
1680 if (MFnI.needsShadowCallStackPrologueEpilogue(MF))
1682 MFnI.needsDwarfUnwindInfo(MF));
1684 if (MFnI.shouldSignReturnAddress(MF)) {
1691 if (EmitCFI && MFnI.isMTETagged()) {
1769 assert(!HasFP &&
"unexpected function without stack frame but with FP");
1771 "unexpected function without stack frame but with SVE objects");
1780 ++NumRedZoneFunctions;
1813 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1814 bool HomPrologEpilog = homogeneousPrologEpilog(MF);
1815 if (CombineSPBump) {
1816 assert(!SVEStackSize &&
"Cannot combine SP bump with SVE");
1822 }
else if (HomPrologEpilog) {
1824 NumBytes -= PrologueSaveSize;
1825 }
else if (PrologueSaveSize != 0) {
1827 MBB,
MBBI,
DL,
TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI,
1829 NumBytes -= PrologueSaveSize;
1831 assert(NumBytes >= 0 &&
"Negative stack allocation size!?");
1840 NeedsWinCFI, &HasWinCFI);
1845 if (!IsFunclet && HasFP) {
1857 bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync);
1858 if (HaveInitialContext)
1860 Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR;
1876 if (HomPrologEpilog) {
1889 if (NeedsWinCFI && HasWinCFI) {
1894 NeedsWinCFI =
false;
1905 emitCalleeSavedGPRLocations(
MBB,
MBBI);
1908 const bool NeedsRealignment =
1909 NumBytes && !IsFunclet && RegInfo->hasStackRealignment(MF);
1910 const int64_t RealignmentPadding =
1916 uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4;
1924 if (NumBytes >= (1 << 28))
1926 "unwinding purposes");
1928 uint32_t LowNumWords = NumWords & 0xFFFF;
1935 if ((NumWords & 0xFFFF0000) != 0) {
1938 .
addImm((NumWords & 0xFFFF0000) >> 16)
2009 if (RealignmentPadding > 0) {
2010 if (RealignmentPadding >= 4096) {
2013 .
addImm(RealignmentPadding)
2023 .
addImm(RealignmentPadding)
2040 StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize;
2046 LLVM_DEBUG(
dbgs() <<
"SVECalleeSavedStackSize = " << CalleeSavedSize
2049 CalleeSavesBegin =
MBBI;
2053 CalleeSavesEnd =
MBBI;
2056 SVELocalsSize = SVEStackSize - SVECalleeSavesSize;
2063 allocateStackSpace(
MBB, CalleeSavesBegin, 0, SVECalleeSavesSize,
false,
2064 nullptr, EmitAsyncCFI && !HasFP, CFAOffset,
2066 CFAOffset += SVECalleeSavesSize;
2069 emitCalleeSavedSVELocations(
MBB, CalleeSavesEnd);
2074 "Cannot use redzone with stack realignment");
2079 allocateStackSpace(
MBB, CalleeSavesEnd, RealignmentPadding,
2081 NeedsWinCFI, &HasWinCFI, EmitAsyncCFI && !HasFP,
2093 if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
2105 if (NeedsWinCFI && HasWinCFI) {
2113 if (IsFunclet &&
F.hasPersonalityFn()) {
2123 if (EmitCFI && !EmitAsyncCFI) {
2130 *RegInfo, AArch64::SP, AArch64::SP, TotalSize,
2136 emitCalleeSavedGPRLocations(
MBB,
MBBI);
2137 emitCalleeSavedSVELocations(
MBB,
MBBI);
2142 switch (
MI.getOpcode()) {
2145 case AArch64::CATCHRET:
2146 case AArch64::CLEANUPRET:
2161 bool HasWinCFI =
false;
2162 bool IsFunclet =
false;
2165 DL =
MBBI->getDebugLoc();
2173 BuildMI(MBB, MBB.getFirstTerminator(), DL,
2174 TII->get(AArch64::PAUTH_EPILOGUE))
2175 .setMIFlag(MachineInstr::FrameDestroy);
2185 TII->get(AArch64::SEH_EpilogEnd))
2212 int64_t AfterCSRPopSize = ArgumentStackToRestore;
2220 if (homogeneousPrologEpilog(MF, &
MBB)) {
2224 auto HomogeneousEpilog = std::prev(LastPopI);
2225 if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog)
2226 LastPopI = HomogeneousEpilog;
2236 assert(AfterCSRPopSize == 0);
2239 bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(
MBB, NumBytes);
2242 bool CombineAfterCSRBump =
false;
2243 if (!CombineSPBump && PrologueSaveSize != 0) {
2245 while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
2247 Pop = std::prev(Pop);
2250 const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
2254 if (OffsetOp.
getImm() == 0 && AfterCSRPopSize >= 0) {
2256 MBB, Pop,
DL,
TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, EmitCFI,
2263 AfterCSRPopSize += PrologueSaveSize;
2264 CombineAfterCSRBump =
true;
2273 while (LastPopI != Begin) {
2279 }
else if (CombineSPBump)
2281 NeedsWinCFI, &HasWinCFI);
2293 EpilogStartI = LastPopI;
2329 if (CombineSPBump) {
2330 assert(!SVEStackSize &&
"Cannot combine SP bump with SVE");
2333 if (EmitCFI &&
hasFP(MF)) {
2335 unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP,
true);
2350 NumBytes -= PrologueSaveSize;
2351 assert(NumBytes >= 0 &&
"Negative stack allocation size!?");
2355 StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
2358 RestoreBegin = std::prev(RestoreEnd);
2359 while (RestoreBegin !=
MBB.
begin() &&
2368 DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
2369 DeallocateAfter = CalleeSavedSizeAsOffset;
2391 MBB, RestoreBegin,
DL, AArch64::SP, AArch64::SP,
2393 false,
false,
nullptr, EmitCFI && !
hasFP(MF),
2400 false,
nullptr, EmitCFI && !
hasFP(MF),
2406 false,
nullptr, EmitCFI && !
hasFP(MF),
2411 emitCalleeSavedSVERestores(
MBB, RestoreEnd);
2418 if (RedZone && AfterCSRPopSize == 0)
2425 bool NoCalleeSaveRestore = PrologueSaveSize == 0;
2426 int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
2427 if (NoCalleeSaveRestore)
2428 StackRestoreBytes += AfterCSRPopSize;
2431 MBB, LastPopI,
DL, AArch64::SP, AArch64::SP,
2438 if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
2451 MBB, LastPopI,
DL, AArch64::SP, AArch64::FP,
2454 }
else if (NumBytes)
2460 if (EmitCFI &&
hasFP(MF)) {
2462 unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP,
true);
2473 if (AfterCSRPopSize) {
2474 assert(AfterCSRPopSize > 0 &&
"attempting to reallocate arg stack that an "
2475 "interrupt may have clobbered");
2480 false, NeedsWinCFI, &HasWinCFI, EmitCFI,
2511 int64_t ObjectOffset) {
2516 unsigned FixedObject =
2525 int64_t ObjectOffset) {
2536 return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
2543 bool ForSimm)
const {
2546 bool isFixed = MFI.isFixedObjectIndex(FI);
2553 const MachineFunction &MF, int64_t ObjectOffset,
bool isFixed,
bool isSVE,
2554 Register &FrameReg,
bool PreferFP,
bool ForSimm)
const {
2577 PreferFP &= !SVEStackSize;
2585 }
else if (isCSR && RegInfo->hasStackRealignment(MF)) {
2589 assert(
hasFP(MF) &&
"Re-aligned stack must have frame pointer");
2591 }
else if (
hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
2596 bool FPOffsetFits = !ForSimm || FPOffset >= -256;
2597 PreferFP |=
Offset > -FPOffset && !SVEStackSize;
2599 if (MFI.hasVarSizedObjects()) {
2603 bool CanUseBP = RegInfo->hasBasePointer(MF);
2604 if (FPOffsetFits && CanUseBP)
2611 }
else if (FPOffset >= 0) {
2616 }
else if (MF.
hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
2623 "Funclets should only be present on Win64");
2627 if (FPOffsetFits && PreferFP)
2634 ((isFixed || isCSR) || !RegInfo->hasStackRealignment(MF) || !UseFP) &&
2635 "In the presence of dynamic stack pointer realignment, "
2636 "non-argument/CSR objects cannot be accessed through the frame pointer");
2648 RegInfo->hasStackRealignment(MF))) {
2649 FrameReg = RegInfo->getFrameRegister(MF);
2653 FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
2659 if (UseFP && !(isFixed || isCSR))
2660 ScalableOffset = -SVEStackSize;
2661 if (!UseFP && (isFixed || isCSR))
2662 ScalableOffset = SVEStackSize;
2665 FrameReg = RegInfo->getFrameRegister(MF);
2670 if (RegInfo->hasBasePointer(MF))
2671 FrameReg = RegInfo->getBaseRegister();
2673 assert(!MFI.hasVarSizedObjects() &&
2674 "Can't use SP when we have var sized objects.");
2675 FrameReg = AArch64::SP;
2701 Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
2706 bool NeedsWinCFI,
bool IsFirst,
2715 if (Reg2 == AArch64::FP)
2719 if (
TRI->getEncodingValue(Reg2) ==
TRI->getEncodingValue(Reg1) + 1)
2726 if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
2727 (Reg1 - AArch64::X19) % 2 == 0 && Reg2 == AArch64::LR && !IsFirst)
2737 bool UsesWinAAPCS,
bool NeedsWinCFI,
2738 bool NeedsFrameRecord,
bool IsFirst,
2746 if (NeedsFrameRecord)
2747 return Reg2 == AArch64::LR;
2755 unsigned Reg1 = AArch64::NoRegister;
2756 unsigned Reg2 = AArch64::NoRegister;
2759 enum RegType { GPR, FPR64, FPR128, PPR, ZPR }
Type;
2761 RegPairInfo() =
default;
2763 bool isPaired()
const {
return Reg2 != AArch64::NoRegister; }
2765 unsigned getScale()
const {
2779 bool isScalable()
const {
return Type == PPR ||
Type == ZPR; }
2787 bool NeedsFrameRecord) {
2797 unsigned Count = CSI.
size();
2804 "Odd number of callee-saved regs to spill!");
2806 int StackFillDir = -1;
2808 unsigned FirstReg = 0;
2816 FirstReg = Count - 1;
2822 for (
unsigned i = FirstReg; i < Count; i += RegInc) {
2824 RPI.Reg1 = CSI[i].getReg();
2826 if (AArch64::GPR64RegClass.
contains(RPI.Reg1))
2827 RPI.Type = RegPairInfo::GPR;
2828 else if (AArch64::FPR64RegClass.
contains(RPI.Reg1))
2829 RPI.Type = RegPairInfo::FPR64;
2830 else if (AArch64::FPR128RegClass.
contains(RPI.Reg1))
2831 RPI.Type = RegPairInfo::FPR128;
2832 else if (AArch64::ZPRRegClass.
contains(RPI.Reg1))
2833 RPI.Type = RegPairInfo::ZPR;
2834 else if (AArch64::PPRRegClass.
contains(RPI.Reg1))
2835 RPI.Type = RegPairInfo::PPR;
2840 if (
unsigned(i + RegInc) < Count) {
2841 Register NextReg = CSI[i + RegInc].getReg();
2842 bool IsFirst = i == FirstReg;
2844 case RegPairInfo::GPR:
2845 if (AArch64::GPR64RegClass.
contains(NextReg) &&
2847 NeedsWinCFI, NeedsFrameRecord, IsFirst,
2851 case RegPairInfo::FPR64:
2852 if (AArch64::FPR64RegClass.
contains(NextReg) &&
2857 case RegPairInfo::FPR128:
2858 if (AArch64::FPR128RegClass.
contains(NextReg))
2861 case RegPairInfo::PPR:
2862 case RegPairInfo::ZPR:
2873 assert((!RPI.isPaired() ||
2874 (CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
2875 "Out of order callee saved regs!");
2877 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
2878 RPI.Reg1 == AArch64::LR) &&
2879 "FrameRecord must be allocated together with LR");
2882 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
2883 RPI.Reg2 == AArch64::LR) &&
2884 "FrameRecord must be allocated together with LR");
2892 ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
2893 RPI.Reg1 + 1 == RPI.Reg2))) &&
2894 "Callee-save registers not saved as adjacent register pair!");
2896 RPI.FrameIdx = CSI[i].getFrameIdx();
2899 RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
2901 int Scale = RPI.getScale();
2903 int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2904 assert(OffsetPre % Scale == 0);
2906 if (RPI.isScalable())
2907 ScalableByteOffset += StackFillDir * Scale;
2909 ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
2914 ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
2915 (IsWindows && RPI.Reg2 == AArch64::LR)))
2916 ByteOffset += StackFillDir * 8;
2918 assert(!(RPI.isScalable() && RPI.isPaired()) &&
2919 "Paired spill/fill instructions don't exist for SVE vectors");
2923 if (NeedGapToAlignStack && !NeedsWinCFI &&
2924 !RPI.isScalable() && RPI.Type != RegPairInfo::FPR128 &&
2925 !RPI.isPaired() && ByteOffset % 16 != 0) {
2926 ByteOffset += 8 * StackFillDir;
2927 assert(MFI.getObjectAlign(RPI.FrameIdx) <=
Align(16));
2931 MFI.setObjectAlignment(RPI.FrameIdx,
Align(16));
2932 NeedGapToAlignStack =
false;
2935 int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2936 assert(OffsetPost % Scale == 0);
2939 int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
2944 ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
2945 (IsWindows && RPI.Reg2 == AArch64::LR)))
2947 RPI.Offset =
Offset / Scale;
2949 assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
2950 (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
2951 "Offset out of bounds for LDP/STP immediate");
2955 if (NeedsFrameRecord && ((!IsWindows && RPI.Reg1 == AArch64::LR &&
2956 RPI.Reg2 == AArch64::FP) ||
2957 (IsWindows && RPI.Reg1 == AArch64::FP &&
2958 RPI.Reg2 == AArch64::LR)))
2972 MFI.setObjectAlignment(CSI[0].getFrameIdx(),
Align(16));
2975 std::reverse(RegPairs.
begin(), RegPairs.
end());
2991 if (homogeneousPrologEpilog(MF)) {
2995 for (
auto &RPI : RegPairs) {
3000 if (!
MRI.isReserved(RPI.Reg1))
3002 if (RPI.isPaired() && !
MRI.isReserved(RPI.Reg2))
3008 unsigned Reg1 = RPI.Reg1;
3009 unsigned Reg2 = RPI.Reg2;
3025 case RegPairInfo::GPR:
3026 StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
3028 Alignment =
Align(8);
3030 case RegPairInfo::FPR64:
3031 StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
3033 Alignment =
Align(8);
3035 case RegPairInfo::FPR128:
3036 StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
3038 Alignment =
Align(16);
3040 case RegPairInfo::ZPR:
3041 StrOpc = AArch64::STR_ZXI;
3043 Alignment =
Align(16);
3045 case RegPairInfo::PPR:
3046 StrOpc = AArch64::STR_PXI;
3048 Alignment =
Align(2);
3053 dbgs() <<
") -> fi#(" << RPI.FrameIdx;
3054 if (RPI.isPaired())
dbgs() <<
", " << RPI.FrameIdx + 1;
3057 assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
3058 "Windows unwdinding requires a consecutive (FP,LR) pair");
3062 unsigned FrameIdxReg1 = RPI.FrameIdx;
3063 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
3064 if (NeedsWinCFI && RPI.isPaired()) {
3069 if (!
MRI.isReserved(Reg1))
3071 if (RPI.isPaired()) {
3072 if (!
MRI.isReserved(Reg2))
3092 if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR)
3109 DL =
MBBI->getDebugLoc();
3113 if (homogeneousPrologEpilog(MF, &
MBB)) {
3116 for (
auto &RPI : RegPairs) {
3124 auto IsPPR = [](
const RegPairInfo &c) {
return c.Type == RegPairInfo::PPR; };
3125 auto PPRBegin = std::find_if(RegPairs.
begin(), RegPairs.
end(), IsPPR);
3126 auto PPREnd = std::find_if_not(PPRBegin, RegPairs.
end(), IsPPR);
3127 std::reverse(PPRBegin, PPREnd);
3128 auto IsZPR = [](
const RegPairInfo &c) {
return c.Type == RegPairInfo::ZPR; };
3129 auto ZPRBegin = std::find_if(RegPairs.
begin(), RegPairs.
end(), IsZPR);
3130 auto ZPREnd = std::find_if_not(ZPRBegin, RegPairs.
end(), IsZPR);
3131 std::reverse(ZPRBegin, ZPREnd);
3133 for (
const RegPairInfo &RPI : RegPairs) {
3134 unsigned Reg1 = RPI.Reg1;
3135 unsigned Reg2 = RPI.Reg2;
3149 case RegPairInfo::GPR:
3150 LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
3152 Alignment =
Align(8);
3154 case RegPairInfo::FPR64:
3155 LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
3157 Alignment =
Align(8);
3159 case RegPairInfo::FPR128:
3160 LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
3162 Alignment =
Align(16);
3164 case RegPairInfo::ZPR:
3165 LdrOpc = AArch64::LDR_ZXI;
3167 Alignment =
Align(16);
3169 case RegPairInfo::PPR:
3170 LdrOpc = AArch64::LDR_PXI;
3172 Alignment =
Align(2);
3177 dbgs() <<
") -> fi#(" << RPI.FrameIdx;
3178 if (RPI.isPaired())
dbgs() <<
", " << RPI.FrameIdx + 1;
3184 unsigned FrameIdxReg1 = RPI.FrameIdx;
3185 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
3186 if (NeedsWinCFI && RPI.isPaired()) {
3191 if (RPI.isPaired()) {
3225 unsigned UnspilledCSGPR = AArch64::NoRegister;
3226 unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
3235 unsigned ExtraCSSpill = 0;
3236 bool HasUnpairedGPR64 =
false;
3238 for (
unsigned i = 0; CSRegs[i]; ++i) {
3239 const unsigned Reg = CSRegs[i];
3242 if (Reg == BasePointerReg)
3245 bool RegUsed = SavedRegs.
test(Reg);
3246 unsigned PairedReg = AArch64::NoRegister;
3247 const bool RegIsGPR64 = AArch64::GPR64RegClass.contains(Reg);
3248 if (RegIsGPR64 || AArch64::FPR64RegClass.
contains(Reg) ||
3249 AArch64::FPR128RegClass.contains(Reg)) {
3252 if (HasUnpairedGPR64)
3253 PairedReg = CSRegs[i % 2 == 0 ? i - 1 : i + 1];
3255 PairedReg = CSRegs[i ^ 1];
3262 if (RegIsGPR64 && !AArch64::GPR64RegClass.
contains(PairedReg)) {
3263 PairedReg = AArch64::NoRegister;
3264 HasUnpairedGPR64 =
true;
3266 assert(PairedReg == AArch64::NoRegister ||
3267 AArch64::GPR64RegClass.
contains(Reg, PairedReg) ||
3268 AArch64::FPR64RegClass.
contains(Reg, PairedReg) ||
3269 AArch64::FPR128RegClass.
contains(Reg, PairedReg));
3272 if (AArch64::GPR64RegClass.
contains(Reg) &&
3274 UnspilledCSGPR = Reg;
3275 UnspilledCSGPRPaired = PairedReg;
3283 if (producePairRegisters(MF) && PairedReg != AArch64::NoRegister &&
3284 !SavedRegs.
test(PairedReg)) {
3285 SavedRegs.
set(PairedReg);
3286 if (AArch64::GPR64RegClass.
contains(PairedReg) &&
3288 ExtraCSSpill = PairedReg;
3299 SavedRegs.
set(AArch64::X18);
3303 unsigned CSStackSize = 0;
3304 unsigned SVECSStackSize = 0;
3307 for (
unsigned Reg : SavedRegs.
set_bits()) {
3309 if (AArch64::PPRRegClass.
contains(Reg) ||
3310 AArch64::ZPRRegClass.
contains(Reg))
3317 unsigned NumSavedRegs = SavedRegs.
count();
3323 SavedRegs.
set(AArch64::FP);
3324 SavedRegs.
set(AArch64::LR);
3334 int64_t SVEStackSize =
3335 alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
3336 bool CanEliminateFrame = (SavedRegs.
count() == 0) && !SVEStackSize;
3345 int64_t CalleeStackUsed = 0;
3348 if (FixedOff > CalleeStackUsed) CalleeStackUsed = FixedOff;
3352 bool BigStack = SVEStackSize || (EstimatedStackSize + CSStackSize +
3353 CalleeStackUsed) > EstimatedStackSizeLimit;
3355 AFI->setHasStackFrame(
true);
3364 if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
3366 <<
" to get a scratch register.\n");
3367 SavedRegs.
set(UnspilledCSGPR);
3368 ExtraCSSpill = UnspilledCSGPR;
3373 if (producePairRegisters(MF)) {
3374 if (UnspilledCSGPRPaired == AArch64::NoRegister) {
3377 SavedRegs.
reset(UnspilledCSGPR);
3378 ExtraCSSpill = AArch64::NoRegister;
3381 SavedRegs.
set(UnspilledCSGPRPaired);
3390 unsigned Size =
TRI->getSpillSize(RC);
3391 Align Alignment =
TRI->getSpillAlign(RC);
3394 LLVM_DEBUG(
dbgs() <<
"No available CS registers, allocated fi#" << FI
3395 <<
" as the emergency spill slot.\n");
3400 CSStackSize += 8 * (SavedRegs.
count() - NumSavedRegs);
3404 if (
hasFP(MF) && AFI->hasSwiftAsyncContext())
3409 << EstimatedStackSize + AlignedCSStackSize
3413 AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
3414 "Should not invalidate callee saved info");
3418 AFI->setCalleeSavedStackSize(AlignedCSStackSize);
3419 AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
3420 AFI->setSVECalleeSavedStackSize(
alignTo(SVECSStackSize, 16));
3425 std::vector<CalleeSavedInfo> &CSI,
unsigned &MinCSFrameIndex,
3426 unsigned &MaxCSFrameIndex)
const {
3434 std::reverse(CSI.begin(), CSI.end());
3448 if ((
unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3449 if ((
unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3452 for (
auto &CS : CSI) {
3459 CS.setFrameIdx(FrameIdx);
3461 if ((
unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3462 if ((
unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3466 Reg == AArch64::FP) {
3469 if ((
unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3470 if ((
unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3490 int &Min,
int &Max) {
3491 Min = std::numeric_limits<int>::max();
3492 Max = std::numeric_limits<int>::min();
3498 for (
auto &CS : CSI) {
3499 if (AArch64::ZPRRegClass.
contains(CS.getReg()) ||
3500 AArch64::PPRRegClass.contains(CS.getReg())) {
3501 assert((Max == std::numeric_limits<int>::min() ||
3502 Max + 1 == CS.getFrameIdx()) &&
3503 "SVE CalleeSaves are not consecutive");
3505 Min = std::min(Min, CS.getFrameIdx());
3506 Max = std::max(Max, CS.getFrameIdx());
3509 return Min != std::numeric_limits<int>::max();
3518 int &MinCSFrameIndex,
3519 int &MaxCSFrameIndex,
3520 bool AssignOffsets) {
3525 "SVE vectors should never be passed on the stack by value, only by "
3529 auto Assign = [&MFI](
int FI, int64_t
Offset) {
3539 for (
int I = MinCSFrameIndex;
I <= MaxCSFrameIndex; ++
I) {
3555 int StackProtectorFI = -1;
3559 ObjectsToAllocate.
push_back(StackProtectorFI);
3565 if (
I == StackProtectorFI)
3567 if (MaxCSFrameIndex >=
I &&
I >= MinCSFrameIndex)
3576 for (
unsigned FI : ObjectsToAllocate) {
3581 if (Alignment >
Align(16))
3583 "Alignment of scalable vectors > 16 bytes is not yet supported");
3593int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
3595 int MinCSFrameIndex, MaxCSFrameIndex;
3599int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
3610 "Upwards growing stack unsupported");
3612 int MinCSFrameIndex, MaxCSFrameIndex;
3613 int64_t SVEStackSize =
3614 assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
3634 int64_t FixedObject =
3647 assert(DstReg &&
"There must be a free register after frame setup");
3656struct TagStoreInstr {
3679 std::optional<int64_t> FrameRegUpdate;
3681 unsigned FrameRegUpdateFlags;
3692 :
MBB(
MBB), ZeroData(ZeroData) {
3698 void addInstruction(TagStoreInstr
I) {
3700 TagStores.
back().Offset + TagStores.
back().Size ==
I.Offset) &&
3701 "Non-adjacent tag store instructions.");
3716 const int64_t kMinOffset = -256 * 16;
3717 const int64_t kMaxOffset = 255 * 16;
3720 int64_t BaseRegOffsetBytes = FrameRegOffset.
getFixed();
3721 if (BaseRegOffsetBytes < kMinOffset ||
3722 BaseRegOffsetBytes + (
Size -
Size % 32) > kMaxOffset ||
3726 BaseRegOffsetBytes % 16 != 0) {
3727 Register ScratchReg =
MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3730 BaseReg = ScratchReg;
3731 BaseRegOffsetBytes = 0;
3736 int64_t InstrSize = (
Size > 16) ? 32 : 16;
3739 ? (ZeroData ? AArch64::STZGi : AArch64::STGi)
3740 : (ZeroData ? AArch64::STZ2Gi : AArch64::ST2Gi);
3741 assert(BaseRegOffsetBytes % 16 == 0);
3745 .
addImm(BaseRegOffsetBytes / 16)
3749 if (BaseRegOffsetBytes == 0)
3751 BaseRegOffsetBytes += InstrSize;
3765 :
MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3766 Register SizeReg =
MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3770 int64_t LoopSize =
Size;
3773 if (FrameRegUpdate && *FrameRegUpdate)
3774 LoopSize -= LoopSize % 32;
3776 TII->get(ZeroData ? AArch64::STZGloop_wback
3777 : AArch64::STGloop_wback))
3784 LoopI->
setFlags(FrameRegUpdateFlags);
3786 int64_t ExtraBaseRegUpdate =
3787 FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.
getFixed() -
Size) : 0;
3788 if (LoopSize <
Size) {
3793 TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
3797 .
addImm(1 + ExtraBaseRegUpdate / 16)
3800 }
else if (ExtraBaseRegUpdate) {
3804 TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
3807 .
addImm(std::abs(ExtraBaseRegUpdate))
3817 int64_t
Size, int64_t *TotalOffset) {
3819 if ((
MI.getOpcode() == AArch64::ADDXri ||
3820 MI.getOpcode() == AArch64::SUBXri) &&
3821 MI.getOperand(0).getReg() == Reg &&
MI.getOperand(1).getReg() == Reg) {
3823 int64_t
Offset =
MI.getOperand(2).getImm() << Shift;
3824 if (
MI.getOpcode() == AArch64::SUBXri)
3826 int64_t AbsPostOffset = std::abs(
Offset -
Size);
3827 const int64_t kMaxOffset =
3829 if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
3840 for (
auto &TS : TSE) {
3844 if (
MI->memoperands_empty()) {
3848 MemRefs.
append(
MI->memoperands_begin(),
MI->memoperands_end());
3854 bool TryMergeSPUpdate) {
3855 if (TagStores.
empty())
3857 TagStoreInstr &FirstTagStore = TagStores[0];
3858 TagStoreInstr &LastTagStore = TagStores[TagStores.
size() - 1];
3859 Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
3860 DL = TagStores[0].MI->getDebugLoc();
3864 *MF, FirstTagStore.Offset,
false ,
false , Reg,
3867 FrameRegUpdate = std::nullopt;
3869 mergeMemRefs(TagStores, CombinedMemRefs);
3872 for (
const auto &Instr
3873 : TagStores) {
dbgs() <<
" " << *
Instr.MI; });
3879 if (TagStores.size() < 2)
3881 emitUnrolled(InsertI);
3884 int64_t TotalOffset = 0;
3885 if (TryMergeSPUpdate) {
3891 if (InsertI !=
MBB->
end() &&
3892 canMergeRegUpdate(InsertI, FrameReg, FrameRegOffset.
getFixed() +
Size,
3894 UpdateInstr = &*InsertI++;
3900 if (!UpdateInstr && TagStores.size() < 2)
3904 FrameRegUpdate = TotalOffset;
3905 FrameRegUpdateFlags = UpdateInstr->
getFlags();
3912 for (
auto &TS : TagStores)
3913 TS.MI->eraseFromParent();
3917 int64_t &
Size,
bool &ZeroData) {
3921 unsigned Opcode =
MI.getOpcode();
3922 ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGi ||
3923 Opcode == AArch64::STZ2Gi);
3925 if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
3926 if (!
MI.getOperand(0).isDead() || !
MI.getOperand(1).isDead())
3928 if (!
MI.getOperand(2).isImm() || !
MI.getOperand(3).isFI())
3931 Size =
MI.getOperand(2).getImm();
3935 if (Opcode == AArch64::STGi || Opcode == AArch64::STZGi)
3937 else if (Opcode == AArch64::ST2Gi || Opcode == AArch64::STZ2Gi)
3942 if (
MI.getOperand(0).getReg() != AArch64::SP || !
MI.getOperand(1).isFI())
3946 16 *
MI.getOperand(2).getImm();
3966 if (!isMergeableStackTaggingInstruction(
MI,
Offset,
Size, FirstZeroData))
3972 constexpr int kScanLimit = 10;
3975 NextI != E && Count < kScanLimit; ++NextI) {
3984 if (isMergeableStackTaggingInstruction(
MI,
Offset,
Size, ZeroData)) {
3985 if (ZeroData != FirstZeroData)
3993 if (!
MI.isTransient())
4002 if (
MI.mayLoadOrStore() ||
MI.hasUnmodeledSideEffects())
4018 LiveRegs.addLiveOuts(*
MBB);
4023 LiveRegs.stepBackward(*
I);
4026 if (LiveRegs.contains(AArch64::NZCV))
4030 [](
const TagStoreInstr &
Left,
const TagStoreInstr &
Right) {
4035 int64_t CurOffset = Instrs[0].Offset;
4036 for (
auto &Instr : Instrs) {
4037 if (CurOffset >
Instr.Offset)
4044 TagStoreEdit TSE(
MBB, FirstZeroData);
4045 std::optional<int64_t> EndOffset;
4046 for (
auto &Instr : Instrs) {
4047 if (EndOffset && *EndOffset !=
Instr.Offset) {
4049 TSE.emitCode(InsertI, TFI,
false);
4053 TSE.addInstruction(Instr);
4072 II = tryMergeAdjacentSTG(II,
this, RS);
4080 bool IgnoreSPUpdates)
const {
4082 if (IgnoreSPUpdates) {
4085 FrameReg = AArch64::SP;
4095 FrameReg = AArch64::SP;
4120 bool IsValid =
false;
4122 int ObjectIndex = 0;
4124 int GroupIndex = -1;
4126 bool ObjectFirst =
false;
4129 bool GroupFirst =
false;
4134 int NextGroupIndex = 0;
4135 std::vector<FrameObject> &Objects;
4138 GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
4140 void EndCurrentGroup() {
4141 if (CurrentMembers.
size() > 1) {
4146 for (
int Index : CurrentMembers) {
4147 Objects[
Index].GroupIndex = NextGroupIndex;
4153 CurrentMembers.clear();
4157bool FrameObjectCompare(
const FrameObject &
A,
const FrameObject &
B) {
4175 return std::make_tuple(!
A.IsValid,
A.ObjectFirst,
A.GroupFirst,
A.GroupIndex,
4177 std::make_tuple(!
B.IsValid,
B.ObjectFirst,
B.GroupFirst,
B.GroupIndex,
4189 for (
auto &Obj : ObjectsToAllocate) {
4190 FrameObjects[Obj].IsValid =
true;
4191 FrameObjects[Obj].ObjectIndex = Obj;
4195 GroupBuilder GB(FrameObjects);
4196 for (
auto &
MBB : MF) {
4197 for (
auto &
MI :
MBB) {
4198 if (
MI.isDebugInstr())
4201 switch (
MI.getOpcode()) {
4202 case AArch64::STGloop:
4203 case AArch64::STZGloop:
4207 case AArch64::STZGi:
4208 case AArch64::ST2Gi:
4209 case AArch64::STZ2Gi:
4222 FrameObjects[FI].IsValid)
4230 GB.AddMember(TaggedFI);
4232 GB.EndCurrentGroup();
4235 GB.EndCurrentGroup();
4245 FrameObjects[*TBPI].ObjectFirst =
true;
4246 FrameObjects[*TBPI].GroupFirst =
true;
4247 int FirstGroupIndex = FrameObjects[*TBPI].GroupIndex;
4248 if (FirstGroupIndex >= 0)
4249 for (FrameObject &Object : FrameObjects)
4250 if (Object.GroupIndex == FirstGroupIndex)
4251 Object.GroupFirst =
true;
4257 for (
auto &Obj : FrameObjects) {
4261 ObjectsToAllocate[i++] = Obj.ObjectIndex;
4268 dbgs() <<
" " << Obj.ObjectIndex <<
": group " << Obj.GroupIndex;
4269 if (Obj.ObjectFirst)
4270 dbgs() <<
", first";
4272 dbgs() <<
", group-first";
4282AArch64FrameLowering::inlineStackProbeLoopExactMultiple(
4293 MF.
insert(MBBInsertPoint, LoopMBB);
4295 MF.
insert(MBBInsertPoint, ExitMBB);
4328 bool anyChange =
false;
4331 }
while (anyChange);
4333 return ExitMBB->
begin();
4336void AArch64FrameLowering::inlineStackProbeFixed(
4349 int64_t NumBlocks = FrameSize / ProbeSize;
4350 int64_t ResidualSize = FrameSize % ProbeSize;
4352 LLVM_DEBUG(
dbgs() <<
"Stack probing: total " << FrameSize <<
" bytes, "
4353 << NumBlocks <<
" blocks of " << ProbeSize
4354 <<
" bytes, plus " << ResidualSize <<
" bytes\n");
4359 for (
int i = 0; i < NumBlocks; ++i) {
4365 EmitAsyncCFI && !HasFP, CFAOffset);
4374 }
else if (NumBlocks != 0) {
4380 EmitAsyncCFI && !HasFP, CFAOffset);
4382 MBBI = inlineStackProbeLoopExactMultiple(
MBBI, ProbeSize, ScratchReg);
4384 if (EmitAsyncCFI && !HasFP) {
4388 unsigned Reg =
RegInfo.getDwarfRegNum(AArch64::SP,
true);
4397 if (ResidualSize != 0) {
4403 EmitAsyncCFI && !HasFP, CFAOffset);
4422 if (
MI.getOpcode() == AArch64::PROBED_STACKALLOC ||
4423 MI.getOpcode() == AArch64::PROBED_STACKALLOC_VAR)
4427 if (
MI->getOpcode() == AArch64::PROBED_STACKALLOC) {
4428 Register ScratchReg =
MI->getOperand(0).getReg();
4429 int64_t FrameSize =
MI->getOperand(1).getImm();
4431 MI->getOperand(3).getImm());
4432 inlineStackProbeFixed(
MI->getIterator(), ScratchReg, FrameSize,
4435 assert(
MI->getOpcode() == AArch64::PROBED_STACKALLOC_VAR &&
4436 "Stack probe pseudo-instruction expected");
4439 Register TargetReg =
MI->getOperand(0).getReg();
4440 (void)
TII->probedStackAlloc(
MI->getIterator(), TargetReg,
true);
4442 MI->eraseFromParent();
unsigned const MachineRegisterInfo * MRI
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
static int64_t getArgumentStackToRestore(MachineFunction &MF, MachineBasicBlock &MBB)
Returns how much of the incoming argument stack area (in bytes) we should clean up in an epilogue.
static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL)
static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB)
static void emitCalleeSavedRestores(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool SVE)
static void computeCalleeSaveRegisterPairs(MachineFunction &MF, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI, SmallVectorImpl< RegPairInfo > &RegPairs, bool NeedsFrameRecord)
static const unsigned DefaultSafeSPDisplacement
This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using ...
static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned FixedObject)
static bool needsWinCFI(const MachineFunction &MF)
static void insertCFISameValue(const MCInstrDesc &Desc, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, unsigned DwarfReg)
static cl::opt< bool > StackTaggingMergeSetTag("stack-tagging-merge-settag", cl::desc("merge settag instruction in function epilog"), cl::init(true), cl::Hidden)
static bool produceCompactUnwindFrame(MachineFunction &MF)
static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex, bool AssignOffsets)
static cl::opt< bool > OrderFrameObjects("aarch64-order-frame-objects", cl::desc("sort stack allocations"), cl::init(true), cl::Hidden)
static bool windowsRequiresStackProbe(MachineFunction &MF, uint64_t StackSizeInBytes)
static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, uint64_t LocalStackSize, bool NeedsWinCFI, bool *HasWinCFI)
static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2, bool NeedsWinCFI, bool IsFirst, const TargetRegisterInfo *TRI)
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, MachineInstr::MIFlag FrameFlag=MachineInstr::FrameSetup, int CFAOffset=0)
static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, unsigned LocalStackSize)
static StackOffset getSVEStackSize(const MachineFunction &MF)
Returns the size of the entire SVE stackframe (calleesaves + spills).
static cl::opt< bool > EnableRedZone("aarch64-redzone", cl::desc("enable use of redzone on AArch64"), cl::init(false), cl::Hidden)
static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI, const TargetInstrInfo &TII, MachineInstr::MIFlag Flag)
static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB)
static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI, LivePhysRegs &LiveRegs)
Collect live registers from the end of MI's parent up to (including) MI in LiveRegs.
cl::opt< bool > EnableHomogeneousPrologEpilog("homogeneous-prolog-epilog", cl::Hidden, cl::desc("Emit homogeneous prologue and epilogue for the size " "optimization (default = off)"))
static bool IsSVECalleeSave(MachineBasicBlock::iterator I)
static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2, bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord, bool IsFirst, const TargetRegisterInfo *TRI)
Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg)
static StackOffset getFPOffset(const MachineFunction &MF, int64_t ObjectOffset)
static bool isTargetWindows(const MachineFunction &MF)
static StackOffset getStackOffset(const MachineFunction &MF, int64_t ObjectOffset)
static int64_t upperBound(StackOffset Size)
static unsigned estimateRSStackSizeLimit(MachineFunction &MF)
Look at each instruction that references stack frames and return the stack size limit beyond which so...
static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI, int &Min, int &Max)
returns true if there are any SVE callee saves.
static MCRegister getRegisterOrZero(MCRegister Reg, bool HasSVE)
static bool isFuncletReturnInstr(const MachineInstr &MI)
static void emitShadowCallStackPrologue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool NeedsWinCFI, bool NeedsUnwindInfo)
static unsigned getFixedObjectSize(const MachineFunction &MF, const AArch64FunctionInfo *AFI, bool IsWin64, bool IsFunclet)
Returns the size of the fixed object area (allocated next to sp on entry) On Win64 this may include a...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const int kSetTagLoopThreshold
This file contains the simple types necessary to represent the attributes associated with functions a...
#define CASE(ATTRNAME, AANAME,...)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
static void clear(coro::Shape &Shape)
static const HTTPClientCleanup Cleanup
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned const TargetRegisterInfo * TRI
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static const unsigned FramePtr
void processFunctionBeforeFrameIndicesReplaced(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameIndicesReplaced - This method is called immediately before MO_FrameIndex op...
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
bool enableStackSlotScavenging(const MachineFunction &MF) const override
Returns true if the stack slot holes in the fixed and callee-save stack area should be used when allo...
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
StackOffset getNonLocalFrameIndexReference(const MachineFunction &MF, int FI) const override
getNonLocalFrameIndexReference - This method returns the offset used to reference a frame index locat...
TargetStackID::Value getStackIDForScalableVectors() const override
Returns the StackID that scalable vectors should be associated with.
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register.
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
bool enableCFIFixup(MachineFunction &MF) const override
Returns true if we may need to fix the unwind information for the function.
void resetCFIToInitialState(MachineBasicBlock &MBB) const override
Emit CFI instructions that recreate the state of the unwind information upon fucntion entry.
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
bool canUseRedZone(const MachineFunction &MF) const
Can this function use the red zone for local allocations.
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
int getSEHFrameIndexOffset(const MachineFunction &MF, int FI) const
unsigned getWinEHFuncletFrameSize(const MachineFunction &MF) const
Funclets only need to account for space for the callee saved registers, as the locals are accounted f...
void orderFrameObjects(const MachineFunction &MF, SmallVectorImpl< int > &ObjectsToAllocate) const override
Order the symbols in the local stack frame.
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - Provide a base+offset reference to an FI slot for debug info.
StackOffset resolveFrameOffsetReference(const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE, Register &FrameReg, bool PreferFP, bool ForSimm) const
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI, unsigned &MinCSFrameIndex, unsigned &MaxCSFrameIndex) const override
assignCalleeSavedSpillSlots - Allows target to override spill slot assignment logic.
StackOffset getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI, Register &FrameReg, bool IgnoreSPUpdates) const override
For Win64 AArch64 EH, the offset to the Unwind object is from the SP before the update.
StackOffset resolveFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP, bool ForSimm) const
unsigned getWinEHParentFrameOffset(const MachineFunction &MF) const override
The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve the parent's frame pointer...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF) const
void setSwiftAsyncContextFrameIdx(int FI)
unsigned getTailCallReservedStack() const
unsigned getCalleeSavedStackSize(const MachineFrameInfo &MFI) const
void setCalleeSaveBaseToFrameRecordOffset(int Offset)
bool hasStackProbing() const
unsigned getArgumentStackToRestore() const
void setLocalStackSize(uint64_t Size)
int getCalleeSaveBaseToFrameRecordOffset() const
bool hasStreamingModeChanges() const
bool shouldSignReturnAddress(const MachineFunction &MF) const
int64_t getStackProbeSize() const
uint64_t getStackSizeSVE() const
void setHasRedZone(bool s)
bool hasStackFrame() const
std::optional< int > getTaggedBasePointerIndex() const
uint64_t getLocalStackSize() const
void setStackRealigned(bool s)
bool needsDwarfUnwindInfo(const MachineFunction &MF) const
unsigned getVarArgsGPRSize() const
void setStackSizeSVE(uint64_t S)
bool isStackRealigned() const
bool hasSwiftAsyncContext() const
void setTaggedBasePointerOffset(unsigned Offset)
unsigned getSVECalleeSavedStackSize() const
bool needsAsyncDwarfUnwindInfo(const MachineFunction &MF) const
void setMinMaxSVECSFrameIndex(int Min, int Max)
bool hasCalleeSaveStackFreeSpace() const
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
bool hasBasePointer(const MachineFunction &MF) const
bool cannotEliminateFrame(const MachineFunction &MF) const
unsigned getBaseRegister() const
bool isTargetWindows() const
const AArch64RegisterInfo * getRegisterInfo() const override
const AArch64InstrInfo * getInstrInfo() const override
bool isTargetILP32() const
const AArch64TargetLowering * getTargetLowering() const override
bool isTargetMachO() const
const Triple & getTargetTriple() const
bool isCallingConvWin64(CallingConv::ID CC) const
const char * getChkStkName() const
bool swiftAsyncContextIsDynamicallySet() const
Return whether FrameLowering should always set the "extended frame present" bit in FP,...
bool hasInlineStackProbe(const MachineFunction &MF) const override
True if stack clash protection is enabled for this functions.
unsigned getRedZoneSize(const Function &F) const
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
bool hasAttrSomewhere(Attribute::AttrKind Kind, unsigned *Index=nullptr) const
Return true if the specified attribute is set for at least one parameter or for the return value.
bool test(unsigned Idx) const
size_type count() const
count - Returns the number of bits which are set.
iterator_range< const_set_bits_iterator > set_bits() const
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
Emit instructions to copy a pair of physical registers.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const
Returns true if register Reg and no aliasing register is in the set.
void stepBackward(const MachineInstr &MI)
Simulates liveness when stepping backwards over an instruction(bundle).
void removeReg(MCPhysReg Reg)
Removes a physical register, all its sub-registers, and all its super-registers from the set.
void addLiveIns(const MachineBasicBlock &MBB)
Adds all live-in registers of basic block MBB.
void addLiveOuts(const MachineBasicBlock &MBB)
Adds all live-out registers of basic block MBB.
void addReg(MCPhysReg Reg)
Adds a physical register and all its sub-registers to the set.
bool usesWindowsCFI() const
static MCCFIInstruction createDefCfaRegister(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_def_cfa_register modifies a rule for computing CFA.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createRestore(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_restore says that the rule for Register is now the same as it was at the beginning of the functi...
static MCCFIInstruction createNegateRAState(MCSymbol *L, SMLoc Loc={})
.cfi_negate_ra_state AArch64 negate RA state.
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
static MCCFIInstruction createSameValue(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_same_value Current value of Register is the same as in the previous frame.
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Describe properties that are true of each instruction in the target description file.
Wrapper class representing physical registers. Should be passed by value.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator instr_begin()
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MachineInstr & instr_back()
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
instr_iterator instr_end()
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
reverse_iterator rbegin()
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasCalls() const
Return true if the current function has any function calls.
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
Align getMaxAlign() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
void setObjectOffset(int ObjectIdx, int64_t SPOffset)
Set the stack frame offset of the specified object.
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
int getStackProtectorIndex() const
Return the index for the stack protector object.
uint64_t estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isMaxCallFrameSizeComputed() const
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
unsigned getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
bool hasStackProtectorIndex() const
uint8_t getStackID(int ObjectIdx) const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
int getObjectIndexBegin() const
Return the minimum frame object index.
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
unsigned addFrameInst(const MCCFIInstruction &Inst)
void setHasWinCFI(bool v)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
bool hasEHFunclets() const
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
void setFlags(unsigned flags)
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
uint32_t getFlags() const
Return the MI flags bitvector.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
const MCContext & getContext() const
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
static MachineOperand CreateImm(int64_t Val)
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool isLiveIn(Register Reg) const
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
void backward()
Update internal register state and move MBB iterator backwards.
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
Wrapper class representing virtual and physical registers.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
StringRef - Represent a constant reference to a string, i.e.
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool enableCFIFixup(MachineFunction &MF) const
Returns true if we may need to fix the unwind information for the function.
TargetInstrInfo - Interface to description of machine instruction set.
CodeModel::Model getCodeModel() const
Returns the code model.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
SwiftAsyncFramePointerMode SwiftAsyncFramePointer
Control when and how the Swift async frame pointer bit should be set.
bool DisableFramePointerElim(const MachineFunction &MF) const
DisableFramePointerElim - This returns true if frame pointer elimination optimization should be disab...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const TargetRegisterClass * getMinimalPhysRegClass(MCRegister Reg, MVT VT=MVT::Other) const
Returns the Register Class of a physical register of the given type, picking the most sub register cl...
Align getSpillAlign(const TargetRegisterClass &RC) const
Return the minimum required alignment in bytes for a spill slot for a register of this class.
bool hasStackRealignment(const MachineFunction &MF) const
True if stack realignment is required and still possible.
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
StringRef getArchName() const
Get the architecture (first) component of the triple.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
const unsigned StackProbeMaxLoopUnroll
Maximum number of iterations to unroll for a constant size probing loop.
const unsigned StackProbeMaxUnprobedStack
Maximum allowed number of unprobed bytes above SP at an ABI boundary.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
NodeAddr< InstrNode * > Instr
This is an optimization pass for GlobalISel generic memory operations.
void stable_sort(R &&Range)
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
auto reverse(ContainerTy &&C)
@ Always
Always set the bit.
@ Never
Never set the bit.
@ DeploymentBased
Determine whether to set the bit statically or dynamically based on the deployment target.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
unsigned getDefRegState(bool B)
unsigned getKillRegState(bool B)
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
static bool recomputeLiveIns(MachineBasicBlock &MBB)
Convenience function for recomputing live-in's for a MBB.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Description of the encoding of one expression Op.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.