255#define DEBUG_TYPE "frame-info"
258 cl::desc(
"enable use of redzone on AArch64"),
262 "stack-tagging-merge-settag",
272 cl::desc(
"Emit homogeneous prologue and epilogue for the size "
273 "optimization (default = off)"));
283STATISTIC(NumRedZoneFunctions,
"Number of functions using red zone");
299 int64_t ArgumentPopSize = 0;
300 if (IsTailCallReturn) {
306 ArgumentPopSize = StackAdjust.
getImm();
315 return ArgumentPopSize;
326bool AArch64FrameLowering::homogeneousPrologEpilog(
351 if (AFI->hasSwiftAsyncContext() || AFI->hasStreamingModeChanges())
358 unsigned NumGPRs = 0;
359 for (
unsigned I = 0; CSRegs[
I]; ++
I) {
361 if (Reg == AArch64::LR) {
362 assert(CSRegs[
I + 1] == AArch64::FP);
363 if (NumGPRs % 2 != 0)
367 if (AArch64::GPR64RegClass.
contains(Reg))
375bool AArch64FrameLowering::producePairRegisters(
MachineFunction &MF)
const {
394 if (
MI.isDebugInstr() ||
MI.isPseudo() ||
395 MI.getOpcode() == AArch64::ADDXri ||
396 MI.getOpcode() == AArch64::ADDSXri)
423 if (!IsWin64 || IsFunclet) {
428 Attribute::SwiftAsync))
433 const unsigned UnwindHelpObject = (MF.
hasEHFunclets() ? 8 : 0);
435 alignTo(VarArgsArea + UnwindHelpObject, 16);
452 const unsigned RedZoneSize =
465 bool LowerQRegCopyThroughMem = Subtarget.hasFPARMv8() &&
469 return !(MFI.
hasCalls() ||
hasFP(MF) || NumBytes > RedZoneSize ||
530 unsigned Opc =
I->getOpcode();
531 bool IsDestroy = Opc ==
TII->getCallFrameDestroyOpcode();
532 uint64_t CalleePopAmount = IsDestroy ?
I->getOperand(1).getImm() : 0;
535 int64_t Amount =
I->getOperand(0).getImm();
543 if (CalleePopAmount == 0) {
554 assert(Amount > -0xffffff && Amount < 0xffffff &&
"call frame too large");
565 "non-reserved call frame without var sized objects?");
574 }
else if (CalleePopAmount != 0) {
577 assert(CalleePopAmount < 0xffffff &&
"call frame too large");
584void AArch64FrameLowering::emitCalleeSavedGPRLocations(
590 bool LocallyStreaming =
591 Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface();
602 for (
const auto &Info : CSI) {
603 unsigned FrameIdx =
Info.getFrameIdx();
607 assert(!
Info.isSpilledToReg() &&
"Spilling to registers not implemented");
608 int64_t DwarfReg =
TRI.getDwarfRegNum(
Info.getReg(),
true);
615 (!LocallyStreaming &&
616 DwarfReg ==
TRI.getDwarfRegNum(AArch64::VG,
true)))
627void AArch64FrameLowering::emitCalleeSavedSVELocations(
643 for (
const auto &Info : CSI) {
649 assert(!
Info.isSpilledToReg() &&
"Spilling to registers not implemented");
684 const MCInstrDesc &CFIDesc =
TII.get(TargetOpcode::CFI_INSTRUCTION);
690 nullptr,
TRI.getDwarfRegNum(AArch64::SP,
true), 0));
694 if (MFI.shouldSignReturnAddress(MF)) {
700 if (MFI.needsShadowCallStackPrologueEpilogue(MF))
702 TRI.getDwarfRegNum(AArch64::X18,
true));
705 const std::vector<CalleeSavedInfo> &CSI =
707 for (
const auto &
Info : CSI) {
708 unsigned Reg =
Info.getReg();
709 if (!
TRI.regNeedsCFI(Reg, Reg))
712 TRI.getDwarfRegNum(Reg,
true));
731 for (
const auto &
Info : CSI) {
736 unsigned Reg =
Info.getReg();
741 if (!
Info.isRestored())
745 nullptr,
TRI.getDwarfRegNum(
Info.getReg(),
true)));
752void AArch64FrameLowering::emitCalleeSavedGPRRestores(
757void AArch64FrameLowering::emitCalleeSavedSVERestores(
765 static const int64_t MAX_BYTES_PER_SCALABLE_BYTE = 16;
766 return Size.getScalable() * MAX_BYTES_PER_SCALABLE_BYTE +
Size.getFixed();
769void AArch64FrameLowering::allocateStackSpace(
771 int64_t RealignmentPadding,
StackOffset AllocSize,
bool NeedsWinCFI,
772 bool *HasWinCFI,
bool EmitCFI,
StackOffset InitialOffset,
773 bool FollowupAllocs)
const {
786 const uint64_t AndMask = ~(MaxAlign - 1);
789 Register TargetReg = RealignmentPadding
795 EmitCFI, InitialOffset);
797 if (RealignmentPadding) {
818 if (AllocSize.
getScalable() == 0 && RealignmentPadding == 0) {
820 assert(ScratchReg != AArch64::NoRegister);
830 if (FollowupAllocs) {
847 if (
upperBound(AllocSize) + RealignmentPadding <= ProbeSize) {
848 Register ScratchReg = RealignmentPadding
851 assert(ScratchReg != AArch64::NoRegister);
855 EmitCFI, InitialOffset);
856 if (RealignmentPadding) {
864 if (FollowupAllocs ||
upperBound(AllocSize) + RealignmentPadding >
880 assert(TargetReg != AArch64::NoRegister);
884 EmitCFI, InitialOffset);
885 if (RealignmentPadding) {
905 if (RealignmentPadding)
918 case AArch64::W##n: \
919 case AArch64::X##n: \
944 case AArch64::B##n: \
945 case AArch64::H##n: \
946 case AArch64::S##n: \
947 case AArch64::D##n: \
948 case AArch64::Q##n: \
949 return HasSVE ? AArch64::Z##n : AArch64::Q##n
986void AArch64FrameLowering::emitZeroCallUsedRegs(
BitVector RegsToZero,
1002 bool HasSVE = STI.hasSVE();
1004 if (
TRI.isGeneralPurposeRegister(MF, Reg)) {
1007 GPRsToZero.set(XReg);
1011 FPRsToZero.set(XReg);
1027 {AArch64::P0, AArch64::P1, AArch64::P2, AArch64::P3, AArch64::P4,
1028 AArch64::P5, AArch64::P6, AArch64::P7, AArch64::P8, AArch64::P9,
1029 AArch64::P10, AArch64::P11, AArch64::P12, AArch64::P13, AArch64::P14,
1031 if (RegsToZero[PReg])
1043 for (
unsigned i = 0; CSRegs[i]; ++i)
1044 LiveRegs.
addReg(CSRegs[i]);
1078 for (
unsigned Reg : AArch64::GPR64RegClass) {
1082 return AArch64::NoRegister;
1128 StackSizeInBytes >=
uint64_t(MFI.getStackProbeSize());
1134 F.needsUnwindTableEntry();
1137bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
1143 if (homogeneousPrologEpilog(MF))
1166 if (MFI.hasVarSizedObjects())
1169 if (
RegInfo->hasStackRealignment(MF))
1186bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
1188 if (!shouldCombineCSRLocalStackBump(*
MBB.
getParent(), StackBumpBytes))
1198 while (LastI != Begin) {
1200 if (LastI->isTransient())
1205 switch (LastI->getOpcode()) {
1206 case AArch64::STGloop:
1207 case AArch64::STZGloop:
1209 case AArch64::STZGi:
1210 case AArch64::ST2Gi:
1211 case AArch64::STZ2Gi:
1224 unsigned Opc =
MBBI->getOpcode();
1228 unsigned ImmIdx =
MBBI->getNumOperands() - 1;
1229 int Imm =
MBBI->getOperand(ImmIdx).getImm();
1237 case AArch64::LDPDpost:
1240 case AArch64::STPDpre: {
1241 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1242 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(2).getReg());
1243 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFRegP_X))
1250 case AArch64::LDPXpost:
1253 case AArch64::STPXpre: {
1256 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1257 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFPLR_X))
1261 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveRegP_X))
1268 case AArch64::LDRDpost:
1271 case AArch64::STRDpre: {
1272 unsigned Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1273 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveFReg_X))
1279 case AArch64::LDRXpost:
1282 case AArch64::STRXpre: {
1283 unsigned Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1290 case AArch64::STPDi:
1291 case AArch64::LDPDi: {
1292 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1293 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1301 case AArch64::STPXi:
1302 case AArch64::LDPXi: {
1305 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1317 case AArch64::STRXui:
1318 case AArch64::LDRXui: {
1319 int Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1326 case AArch64::STRDui:
1327 case AArch64::LDRDui: {
1328 unsigned Reg =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1335 case AArch64::STPQi:
1336 case AArch64::LDPQi: {
1337 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(0).getReg());
1338 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1339 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegQP))
1346 case AArch64::LDPQpost:
1349 case AArch64::STPQpre: {
1350 unsigned Reg0 =
RegInfo->getSEHRegNum(
MBBI->getOperand(1).getReg());
1351 unsigned Reg1 =
RegInfo->getSEHRegNum(
MBBI->getOperand(2).getReg());
1352 MIB =
BuildMI(MF,
DL,
TII.get(AArch64::SEH_SaveAnyRegQPX))
1366 unsigned LocalStackSize) {
1368 unsigned ImmIdx =
MBBI->getNumOperands() - 1;
1369 switch (
MBBI->getOpcode()) {
1372 case AArch64::SEH_SaveFPLR:
1373 case AArch64::SEH_SaveRegP:
1374 case AArch64::SEH_SaveReg:
1375 case AArch64::SEH_SaveFRegP:
1376 case AArch64::SEH_SaveFReg:
1377 case AArch64::SEH_SaveAnyRegQP:
1378 case AArch64::SEH_SaveAnyRegQPX:
1379 ImmOpnd = &
MBBI->getOperand(ImmIdx);
1393 unsigned Opc =
MBBI->getOpcode();
1394 if (Opc == AArch64::CNTD_XPiI || Opc == AArch64::RDSVLI_XI ||
1395 Opc == AArch64::UBFMXri)
1399 if (Opc == AArch64::ORRXrr)
1402 if (Opc == AArch64::BL) {
1403 auto Op1 =
MBBI->getOperand(0);
1404 return Op1.isSymbol() &&
1405 (
StringRef(Op1.getSymbolName()) ==
"__arm_get_current_vg");
1418 bool NeedsWinCFI,
bool *HasWinCFI,
bool EmitCFI,
1420 int CFAOffset = 0) {
1433 switch (
MBBI->getOpcode()) {
1436 case AArch64::STPXi:
1437 NewOpc = AArch64::STPXpre;
1439 case AArch64::STPDi:
1440 NewOpc = AArch64::STPDpre;
1442 case AArch64::STPQi:
1443 NewOpc = AArch64::STPQpre;
1445 case AArch64::STRXui:
1446 NewOpc = AArch64::STRXpre;
1448 case AArch64::STRDui:
1449 NewOpc = AArch64::STRDpre;
1451 case AArch64::STRQui:
1452 NewOpc = AArch64::STRQpre;
1454 case AArch64::LDPXi:
1455 NewOpc = AArch64::LDPXpost;
1457 case AArch64::LDPDi:
1458 NewOpc = AArch64::LDPDpost;
1460 case AArch64::LDPQi:
1461 NewOpc = AArch64::LDPQpost;
1463 case AArch64::LDRXui:
1464 NewOpc = AArch64::LDRXpost;
1466 case AArch64::LDRDui:
1467 NewOpc = AArch64::LDRDpost;
1469 case AArch64::LDRQui:
1470 NewOpc = AArch64::LDRQpost;
1475 auto SEH = std::next(
MBBI);
1477 SEH->eraseFromParent();
1481 int64_t MinOffset, MaxOffset;
1483 NewOpc, Scale, Width, MinOffset, MaxOffset);
1489 if (
MBBI->getOperand(
MBBI->getNumOperands() - 1).getImm() != 0 ||
1490 CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) {
1497 false,
false,
nullptr, EmitCFI,
1500 return std::prev(
MBBI);
1507 unsigned OpndIdx = 0;
1508 for (
unsigned OpndEnd =
MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
1510 MIB.
add(
MBBI->getOperand(OpndIdx));
1512 assert(
MBBI->getOperand(OpndIdx).getImm() == 0 &&
1513 "Unexpected immediate offset in first/last callee-save save/restore "
1515 assert(
MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
1516 "Unexpected base register in callee-save save/restore instruction!");
1517 assert(CSStackSizeInc % Scale == 0);
1518 MIB.
addImm(CSStackSizeInc / (
int)Scale);
1549 unsigned Opc =
MI.getOpcode();
1552 case AArch64::STPXi:
1553 case AArch64::STRXui:
1554 case AArch64::STPDi:
1555 case AArch64::STRDui:
1556 case AArch64::LDPXi:
1557 case AArch64::LDRXui:
1558 case AArch64::LDPDi:
1559 case AArch64::LDRDui:
1562 case AArch64::STPQi:
1563 case AArch64::STRQui:
1564 case AArch64::LDPQi:
1565 case AArch64::LDRQui:
1572 unsigned OffsetIdx =
MI.getNumExplicitOperands() - 1;
1573 assert(
MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
1574 "Unexpected base register in callee-save save/restore instruction!");
1578 assert(LocalStackSize % Scale == 0);
1579 OffsetOpnd.
setImm(OffsetOpnd.
getImm() + LocalStackSize / Scale);
1584 assert(
MBBI !=
MI.getParent()->end() &&
"Expecting a valid instruction");
1586 "Expecting a SEH instruction");
1597 switch (
I->getOpcode()) {
1600 case AArch64::PTRUE_C_B:
1601 case AArch64::LD1B_2Z_IMM:
1602 case AArch64::ST1B_2Z_IMM:
1603 case AArch64::STR_ZXI:
1604 case AArch64::STR_PXI:
1605 case AArch64::LDR_ZXI:
1606 case AArch64::LDR_PXI:
1617 bool NeedsUnwindInfo) {
1633 if (NeedsUnwindInfo) {
1636 static const char CFIInst[] = {
1637 dwarf::DW_CFA_val_expression,
1640 static_cast<char>(
unsigned(dwarf::DW_OP_breg18)),
1641 static_cast<char>(-8) & 0x7f,
1644 nullptr,
StringRef(CFIInst,
sizeof(CFIInst))));
1682 const int OffsetToFirstCalleeSaveFromFP =
1686 unsigned Reg =
TRI->getDwarfRegNum(
FramePtr,
true);
1688 nullptr, Reg, FixedObject - OffsetToFirstCalleeSaveFromFP));
1720 bool HasFP =
hasFP(MF);
1722 bool HasWinCFI =
false;
1731 while (NonFrameStart !=
End &&
1736 if (NonFrameStart !=
MBB.
end()) {
1752 if (NonFrameStart ==
MBB.
end())
1757 for (auto &Op : MI.operands())
1758 if (Op.isReg() && Op.isDef())
1759 assert(!LiveRegs.contains(Op.getReg()) &&
1760 "live register clobbered by inserted prologue instructions");
1777 if (MFnI.needsShadowCallStackPrologueEpilogue(MF))
1779 MFnI.needsDwarfUnwindInfo(MF));
1781 if (MFnI.shouldSignReturnAddress(MF)) {
1788 if (EmitCFI && MFnI.isMTETagged()) {
1866 assert(!HasFP &&
"unexpected function without stack frame but with FP");
1868 "unexpected function without stack frame but with SVE objects");
1877 ++NumRedZoneFunctions;
1909 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1910 bool HomPrologEpilog = homogeneousPrologEpilog(MF);
1911 if (CombineSPBump) {
1912 assert(!SVEStackSize &&
"Cannot combine SP bump with SVE");
1918 }
else if (HomPrologEpilog) {
1920 NumBytes -= PrologueSaveSize;
1921 }
else if (PrologueSaveSize != 0) {
1923 MBB,
MBBI,
DL,
TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI,
1925 NumBytes -= PrologueSaveSize;
1927 assert(NumBytes >= 0 &&
"Negative stack allocation size!?");
1941 NeedsWinCFI, &HasWinCFI);
1946 if (!IsFunclet && HasFP) {
1958 bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync);
1959 if (HaveInitialContext)
1961 Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR;
1977 if (HomPrologEpilog) {
1990 if (NeedsWinCFI && HasWinCFI) {
1995 NeedsWinCFI =
false;
2006 emitCalleeSavedGPRLocations(
MBB,
MBBI);
2009 const bool NeedsRealignment =
2010 NumBytes && !IsFunclet && RegInfo->hasStackRealignment(MF);
2011 const int64_t RealignmentPadding =
2017 uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4;
2025 if (NumBytes >= (1 << 28))
2027 "unwinding purposes");
2029 uint32_t LowNumWords = NumWords & 0xFFFF;
2036 if ((NumWords & 0xFFFF0000) != 0) {
2039 .
addImm((NumWords & 0xFFFF0000) >> 16)
2110 if (RealignmentPadding > 0) {
2111 if (RealignmentPadding >= 4096) {
2114 .
addImm(RealignmentPadding)
2124 .
addImm(RealignmentPadding)
2141 StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize;
2147 LLVM_DEBUG(
dbgs() <<
"SVECalleeSavedStackSize = " << CalleeSavedSize
2150 CalleeSavesBegin =
MBBI;
2154 CalleeSavesEnd =
MBBI;
2157 SVELocalsSize = SVEStackSize - SVECalleeSavesSize;
2164 allocateStackSpace(
MBB, CalleeSavesBegin, 0, SVECalleeSavesSize,
false,
2165 nullptr, EmitAsyncCFI && !HasFP, CFAOffset,
2167 CFAOffset += SVECalleeSavesSize;
2170 emitCalleeSavedSVELocations(
MBB, CalleeSavesEnd);
2175 "Cannot use redzone with stack realignment");
2180 allocateStackSpace(
MBB, CalleeSavesEnd, RealignmentPadding,
2182 NeedsWinCFI, &HasWinCFI, EmitAsyncCFI && !HasFP,
2194 if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
2206 if (NeedsWinCFI && HasWinCFI) {
2214 if (IsFunclet &&
F.hasPersonalityFn()) {
2224 if (EmitCFI && !EmitAsyncCFI) {
2231 *RegInfo, AArch64::SP, AArch64::SP, TotalSize,
2237 emitCalleeSavedGPRLocations(
MBB,
MBBI);
2238 emitCalleeSavedSVELocations(
MBB,
MBBI);
2243 switch (
MI.getOpcode()) {
2246 case AArch64::CATCHRET:
2247 case AArch64::CLEANUPRET:
2262 bool HasWinCFI =
false;
2263 bool IsFunclet =
false;
2266 DL =
MBBI->getDebugLoc();
2274 BuildMI(MBB, MBB.getFirstTerminator(), DL,
2275 TII->get(AArch64::PAUTH_EPILOGUE))
2276 .setMIFlag(MachineInstr::FrameDestroy);
2286 TII->get(AArch64::SEH_EpilogEnd))
2313 int64_t AfterCSRPopSize = ArgumentStackToRestore;
2321 if (homogeneousPrologEpilog(MF, &
MBB)) {
2325 auto HomogeneousEpilog = std::prev(LastPopI);
2326 if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog)
2327 LastPopI = HomogeneousEpilog;
2337 assert(AfterCSRPopSize == 0);
2340 bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(
MBB, NumBytes);
2343 bool CombineAfterCSRBump =
false;
2344 if (!CombineSPBump && PrologueSaveSize != 0) {
2346 while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
2348 Pop = std::prev(Pop);
2351 const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
2355 if (OffsetOp.
getImm() == 0 && AfterCSRPopSize >= 0) {
2357 MBB, Pop,
DL,
TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, EmitCFI,
2364 AfterCSRPopSize += PrologueSaveSize;
2365 CombineAfterCSRBump =
true;
2374 while (LastPopI != Begin) {
2380 }
else if (CombineSPBump)
2382 NeedsWinCFI, &HasWinCFI);
2394 EpilogStartI = LastPopI;
2430 if (CombineSPBump) {
2431 assert(!SVEStackSize &&
"Cannot combine SP bump with SVE");
2434 if (EmitCFI &&
hasFP(MF)) {
2436 unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP,
true);
2451 NumBytes -= PrologueSaveSize;
2452 assert(NumBytes >= 0 &&
"Negative stack allocation size!?");
2456 StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
2459 RestoreBegin = std::prev(RestoreEnd);
2460 while (RestoreBegin !=
MBB.
begin() &&
2469 DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
2470 DeallocateAfter = CalleeSavedSizeAsOffset;
2492 MBB, RestoreBegin,
DL, AArch64::SP, AArch64::SP,
2494 false,
false,
nullptr, EmitCFI && !
hasFP(MF),
2501 false,
nullptr, EmitCFI && !
hasFP(MF),
2507 false,
nullptr, EmitCFI && !
hasFP(MF),
2512 emitCalleeSavedSVERestores(
MBB, RestoreEnd);
2519 if (RedZone && AfterCSRPopSize == 0)
2526 bool NoCalleeSaveRestore = PrologueSaveSize == 0;
2527 int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
2528 if (NoCalleeSaveRestore)
2529 StackRestoreBytes += AfterCSRPopSize;
2532 MBB, LastPopI,
DL, AArch64::SP, AArch64::SP,
2539 if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
2552 MBB, LastPopI,
DL, AArch64::SP, AArch64::FP,
2555 }
else if (NumBytes)
2561 if (EmitCFI &&
hasFP(MF)) {
2563 unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP,
true);
2574 if (AfterCSRPopSize) {
2575 assert(AfterCSRPopSize > 0 &&
"attempting to reallocate arg stack that an "
2576 "interrupt may have clobbered");
2581 false, NeedsWinCFI, &HasWinCFI, EmitCFI,
2613 int64_t ObjectOffset) {
2617 bool IsWin64 = Subtarget.isCallingConvWin64(
F.getCallingConv(),
F.isVarArg());
2618 unsigned FixedObject =
2627 int64_t ObjectOffset) {
2638 return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
2645 bool ForSimm)
const {
2648 bool isFixed = MFI.isFixedObjectIndex(FI);
2655 const MachineFunction &MF, int64_t ObjectOffset,
bool isFixed,
bool isSVE,
2656 Register &FrameReg,
bool PreferFP,
bool ForSimm)
const {
2679 PreferFP &= !SVEStackSize;
2687 }
else if (isCSR && RegInfo->hasStackRealignment(MF)) {
2691 assert(
hasFP(MF) &&
"Re-aligned stack must have frame pointer");
2693 }
else if (
hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
2698 bool FPOffsetFits = !ForSimm || FPOffset >= -256;
2699 PreferFP |=
Offset > -FPOffset && !SVEStackSize;
2701 if (MFI.hasVarSizedObjects()) {
2705 bool CanUseBP = RegInfo->hasBasePointer(MF);
2706 if (FPOffsetFits && CanUseBP)
2713 }
else if (FPOffset >= 0) {
2718 }
else if (MF.
hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
2725 "Funclets should only be present on Win64");
2729 if (FPOffsetFits && PreferFP)
2736 ((isFixed || isCSR) || !RegInfo->hasStackRealignment(MF) || !UseFP) &&
2737 "In the presence of dynamic stack pointer realignment, "
2738 "non-argument/CSR objects cannot be accessed through the frame pointer");
2750 RegInfo->hasStackRealignment(MF))) {
2751 FrameReg = RegInfo->getFrameRegister(MF);
2755 FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
2761 if (UseFP && !(isFixed || isCSR))
2762 ScalableOffset = -SVEStackSize;
2763 if (!UseFP && (isFixed || isCSR))
2764 ScalableOffset = SVEStackSize;
2767 FrameReg = RegInfo->getFrameRegister(MF);
2772 if (RegInfo->hasBasePointer(MF))
2773 FrameReg = RegInfo->getBaseRegister();
2775 assert(!MFI.hasVarSizedObjects() &&
2776 "Can't use SP when we have var sized objects.");
2777 FrameReg = AArch64::SP;
2803 Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
2808 bool NeedsWinCFI,
bool IsFirst,
2817 if (Reg2 == AArch64::FP)
2821 if (
TRI->getEncodingValue(Reg2) ==
TRI->getEncodingValue(Reg1) + 1)
2828 if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
2829 (Reg1 - AArch64::X19) % 2 == 0 && Reg2 == AArch64::LR && !IsFirst)
2839 bool UsesWinAAPCS,
bool NeedsWinCFI,
2840 bool NeedsFrameRecord,
bool IsFirst,
2848 if (NeedsFrameRecord)
2849 return Reg2 == AArch64::LR;
2857 unsigned Reg1 = AArch64::NoRegister;
2858 unsigned Reg2 = AArch64::NoRegister;
2861 enum RegType { GPR, FPR64, FPR128, PPR, ZPR, VG }
Type;
2863 RegPairInfo() =
default;
2865 bool isPaired()
const {
return Reg2 != AArch64::NoRegister; }
2867 unsigned getScale()
const {
2882 bool isScalable()
const {
return Type == PPR ||
Type == ZPR; }
2888 for (
unsigned PReg = AArch64::P8; PReg <= AArch64::P15; ++PReg) {
2889 if (SavedRegs.
test(PReg)) {
2890 unsigned PNReg = PReg - AArch64::P0 + AArch64::PN0;
2894 return AArch64::NoRegister;
2900 bool NeedsFrameRecord) {
2910 unsigned Count = CSI.
size();
2917 "Odd number of callee-saved regs to spill!");
2919 int StackFillDir = -1;
2921 unsigned FirstReg = 0;
2929 FirstReg = Count - 1;
2936 for (
unsigned i = FirstReg; i < Count; i += RegInc) {
2938 RPI.Reg1 = CSI[i].getReg();
2940 if (AArch64::GPR64RegClass.
contains(RPI.Reg1))
2941 RPI.Type = RegPairInfo::GPR;
2942 else if (AArch64::FPR64RegClass.
contains(RPI.Reg1))
2943 RPI.Type = RegPairInfo::FPR64;
2944 else if (AArch64::FPR128RegClass.
contains(RPI.Reg1))
2945 RPI.Type = RegPairInfo::FPR128;
2946 else if (AArch64::ZPRRegClass.
contains(RPI.Reg1))
2947 RPI.Type = RegPairInfo::ZPR;
2948 else if (AArch64::PPRRegClass.
contains(RPI.Reg1))
2949 RPI.Type = RegPairInfo::PPR;
2950 else if (RPI.Reg1 == AArch64::VG)
2951 RPI.Type = RegPairInfo::VG;
2964 Register NextReg = CSI[i + RegInc].getReg();
2965 bool IsFirst = i == FirstReg;
2967 case RegPairInfo::GPR:
2968 if (AArch64::GPR64RegClass.
contains(NextReg) &&
2970 NeedsWinCFI, NeedsFrameRecord, IsFirst,
2974 case RegPairInfo::FPR64:
2975 if (AArch64::FPR64RegClass.
contains(NextReg) &&
2980 case RegPairInfo::FPR128:
2981 if (AArch64::FPR128RegClass.
contains(NextReg))
2984 case RegPairInfo::PPR:
2986 case RegPairInfo::ZPR:
2988 if (((RPI.Reg1 - AArch64::Z0) & 1) == 0 && (NextReg == RPI.Reg1 + 1))
2991 case RegPairInfo::VG:
3002 assert((!RPI.isPaired() ||
3003 (CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
3004 "Out of order callee saved regs!");
3006 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
3007 RPI.Reg1 == AArch64::LR) &&
3008 "FrameRecord must be allocated together with LR");
3011 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
3012 RPI.Reg2 == AArch64::LR) &&
3013 "FrameRecord must be allocated together with LR");
3021 ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
3022 RPI.Reg1 + 1 == RPI.Reg2))) &&
3023 "Callee-save registers not saved as adjacent register pair!");
3025 RPI.FrameIdx = CSI[i].getFrameIdx();
3028 RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
3029 int Scale = RPI.getScale();
3031 int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
3032 assert(OffsetPre % Scale == 0);
3034 if (RPI.isScalable())
3035 ScalableByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
3037 ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
3042 ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
3043 (IsWindows && RPI.Reg2 == AArch64::LR)))
3044 ByteOffset += StackFillDir * 8;
3048 if (NeedGapToAlignStack && !NeedsWinCFI && !RPI.isScalable() &&
3049 RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired() &&
3050 ByteOffset % 16 != 0) {
3051 ByteOffset += 8 * StackFillDir;
3052 assert(MFI.getObjectAlign(RPI.FrameIdx) <=
Align(16));
3056 MFI.setObjectAlignment(RPI.FrameIdx,
Align(16));
3057 NeedGapToAlignStack =
false;
3060 int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
3061 assert(OffsetPost % Scale == 0);
3064 int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
3069 ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
3070 (IsWindows && RPI.Reg2 == AArch64::LR)))
3072 RPI.Offset =
Offset / Scale;
3074 assert((!RPI.isPaired() ||
3075 (!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
3076 (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
3077 "Offset out of bounds for LDP/STP immediate");
3081 if (NeedsFrameRecord &&
3082 ((!IsWindows && RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
3083 (IsWindows && RPI.Reg1 == AArch64::FP && RPI.Reg2 == AArch64::LR)))
3097 MFI.setObjectAlignment(CSI[0].getFrameIdx(),
Align(16));
3100 std::reverse(RegPairs.
begin(), RegPairs.
end());
3119 MRI.freezeReservedRegs();
3121 if (homogeneousPrologEpilog(MF)) {
3125 for (
auto &RPI : RegPairs) {
3130 if (!
MRI.isReserved(RPI.Reg1))
3132 if (RPI.isPaired() && !
MRI.isReserved(RPI.Reg2))
3137 bool PTrueCreated =
false;
3139 unsigned Reg1 = RPI.Reg1;
3140 unsigned Reg2 = RPI.Reg2;
3156 case RegPairInfo::GPR:
3157 StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
3159 Alignment =
Align(8);
3161 case RegPairInfo::FPR64:
3162 StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
3164 Alignment =
Align(8);
3166 case RegPairInfo::FPR128:
3167 StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
3169 Alignment =
Align(16);
3171 case RegPairInfo::ZPR:
3172 StrOpc = RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI;
3174 Alignment =
Align(16);
3176 case RegPairInfo::PPR:
3177 StrOpc = AArch64::STR_PXI;
3179 Alignment =
Align(2);
3181 case RegPairInfo::VG:
3182 StrOpc = AArch64::STRXui;
3184 Alignment =
Align(8);
3188 unsigned X0Scratch = AArch64::NoRegister;
3189 if (Reg1 == AArch64::VG) {
3192 assert(Reg1 != AArch64::NoRegister);
3195 if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface() &&
3220 return STI.getRegisterInfo()->isSuperOrSubRegisterEq(
3221 AArch64::X0, LiveIn.PhysReg);
3225 if (X0Scratch != AArch64::NoRegister)
3232 const uint32_t *RegMask =
TRI->getCallPreservedMask(
3247 dbgs() <<
") -> fi#(" << RPI.FrameIdx;
3248 if (RPI.isPaired())
dbgs() <<
", " << RPI.FrameIdx + 1;
3251 assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
3252 "Windows unwdinding requires a consecutive (FP,LR) pair");
3256 unsigned FrameIdxReg1 = RPI.FrameIdx;
3257 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
3258 if (NeedsWinCFI && RPI.isPaired()) {
3263 if (RPI.isPaired() && RPI.isScalable()) {
3268 assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
3269 "Expects SVE2.1 or SME2 target and a predicate register");
3270#ifdef EXPENSIVE_CHECKS
3271 auto IsPPR = [](
const RegPairInfo &c) {
3272 return c.Reg1 == RegPairInfo::PPR;
3274 auto PPRBegin = std::find_if(RegPairs.
begin(), RegPairs.
end(), IsPPR);
3275 auto IsZPR = [](
const RegPairInfo &c) {
3276 return c.Type == RegPairInfo::ZPR;
3278 auto ZPRBegin = std::find_if(RegPairs.
begin(), RegPairs.
end(), IsZPR);
3279 assert(!(PPRBegin < ZPRBegin) &&
3280 "Expected callee save predicate to be handled first");
3282 if (!PTrueCreated) {
3283 PTrueCreated =
true;
3288 if (!
MRI.isReserved(Reg1))
3290 if (!
MRI.isReserved(Reg2))
3292 MIB.
addReg( AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0));
3308 if (!
MRI.isReserved(Reg1))
3310 if (RPI.isPaired()) {
3311 if (!
MRI.isReserved(Reg2))
3331 if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR) {
3337 if (X0Scratch != AArch64::NoRegister)
3357 DL =
MBBI->getDebugLoc();
3360 if (homogeneousPrologEpilog(MF, &
MBB)) {
3363 for (
auto &RPI : RegPairs) {
3371 auto IsPPR = [](
const RegPairInfo &c) {
return c.Type == RegPairInfo::PPR; };
3372 auto PPRBegin = std::find_if(RegPairs.
begin(), RegPairs.
end(), IsPPR);
3373 auto PPREnd = std::find_if_not(PPRBegin, RegPairs.
end(), IsPPR);
3374 std::reverse(PPRBegin, PPREnd);
3375 auto IsZPR = [](
const RegPairInfo &c) {
return c.Type == RegPairInfo::ZPR; };
3376 auto ZPRBegin = std::find_if(RegPairs.
begin(), RegPairs.
end(), IsZPR);
3377 auto ZPREnd = std::find_if_not(ZPRBegin, RegPairs.
end(), IsZPR);
3378 std::reverse(ZPRBegin, ZPREnd);
3380 bool PTrueCreated =
false;
3381 for (
const RegPairInfo &RPI : RegPairs) {
3382 unsigned Reg1 = RPI.Reg1;
3383 unsigned Reg2 = RPI.Reg2;
3397 case RegPairInfo::GPR:
3398 LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
3400 Alignment =
Align(8);
3402 case RegPairInfo::FPR64:
3403 LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
3405 Alignment =
Align(8);
3407 case RegPairInfo::FPR128:
3408 LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
3410 Alignment =
Align(16);
3412 case RegPairInfo::ZPR:
3413 LdrOpc = RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI;
3415 Alignment =
Align(16);
3417 case RegPairInfo::PPR:
3418 LdrOpc = AArch64::LDR_PXI;
3420 Alignment =
Align(2);
3422 case RegPairInfo::VG:
3427 dbgs() <<
") -> fi#(" << RPI.FrameIdx;
3428 if (RPI.isPaired())
dbgs() <<
", " << RPI.FrameIdx + 1;
3434 unsigned FrameIdxReg1 = RPI.FrameIdx;
3435 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
3436 if (NeedsWinCFI && RPI.isPaired()) {
3442 if (RPI.isPaired() && RPI.isScalable()) {
3446 assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
3447 "Expects SVE2.1 or SME2 target and a predicate register");
3448#ifdef EXPENSIVE_CHECKS
3449 assert(!(PPRBegin < ZPRBegin) &&
3450 "Expected callee save predicate to be handled first");
3452 if (!PTrueCreated) {
3453 PTrueCreated =
true;
3458 MIB.
addReg( AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0),
3475 if (RPI.isPaired()) {
3499 if (!
MI.mayLoadOrStore() ||
MI.getNumMemOperands() < 1)
3500 return std::nullopt;
3504 dyn_cast_or_null<FixedStackPseudoSourceValue>(MMO->
getPseudoValue());
3506 return std::optional<int>(PSV->getFrameIndex());
3517 return std::nullopt;
3523void AArch64FrameLowering::determineStackHazardSlot(
3538 bool HasFPRCSRs =
any_of(SavedRegs.
set_bits(), [](
unsigned Reg) {
3539 return AArch64::FPR64RegClass.contains(Reg) ||
3540 AArch64::FPR128RegClass.contains(Reg) ||
3541 AArch64::ZPRRegClass.contains(Reg) ||
3542 AArch64::PPRRegClass.contains(Reg);
3544 bool HasFPRStackObjects =
false;
3547 for (
auto &
MBB : MF) {
3548 for (
auto &
MI :
MBB) {
3550 if (FI && *FI >= 0 && *FI < (
int)FrameObjects.size()) {
3553 FrameObjects[*FI] |= 2;
3555 FrameObjects[*FI] |= 1;
3559 HasFPRStackObjects =
3560 any_of(FrameObjects, [](
unsigned B) {
return (
B & 3) == 2; });
3563 if (HasFPRCSRs || HasFPRStackObjects) {
3584 unsigned UnspilledCSGPR = AArch64::NoRegister;
3585 unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
3594 unsigned ExtraCSSpill = 0;
3595 bool HasUnpairedGPR64 =
false;
3596 bool HasPairZReg =
false;
3598 for (
unsigned i = 0; CSRegs[i]; ++i) {
3599 const unsigned Reg = CSRegs[i];
3602 if (Reg == BasePointerReg)
3605 bool RegUsed = SavedRegs.
test(Reg);
3606 unsigned PairedReg = AArch64::NoRegister;
3607 const bool RegIsGPR64 = AArch64::GPR64RegClass.contains(Reg);
3608 if (RegIsGPR64 || AArch64::FPR64RegClass.
contains(Reg) ||
3609 AArch64::FPR128RegClass.contains(Reg)) {
3612 if (HasUnpairedGPR64)
3613 PairedReg = CSRegs[i % 2 == 0 ? i - 1 : i + 1];
3615 PairedReg = CSRegs[i ^ 1];
3622 if (RegIsGPR64 && !AArch64::GPR64RegClass.
contains(PairedReg)) {
3623 PairedReg = AArch64::NoRegister;
3624 HasUnpairedGPR64 =
true;
3626 assert(PairedReg == AArch64::NoRegister ||
3627 AArch64::GPR64RegClass.
contains(Reg, PairedReg) ||
3628 AArch64::FPR64RegClass.
contains(Reg, PairedReg) ||
3629 AArch64::FPR128RegClass.
contains(Reg, PairedReg));
3632 if (AArch64::GPR64RegClass.
contains(Reg) &&
3634 UnspilledCSGPR = Reg;
3635 UnspilledCSGPRPaired = PairedReg;
3643 if (producePairRegisters(MF) && PairedReg != AArch64::NoRegister &&
3644 !SavedRegs.
test(PairedReg)) {
3645 SavedRegs.
set(PairedReg);
3646 if (AArch64::GPR64RegClass.
contains(PairedReg) &&
3648 ExtraCSSpill = PairedReg;
3651 HasPairZReg |= (AArch64::ZPRRegClass.contains(Reg, CSRegs[i ^ 1]) &&
3652 SavedRegs.
test(CSRegs[i ^ 1]));
3655 if (HasPairZReg && (Subtarget.hasSVE2p1() || Subtarget.hasSME2())) {
3660 if (PnReg != AArch64::NoRegister)
3666 SavedRegs.
set(AArch64::P8);
3671 "Predicate cannot be a reserved register");
3681 SavedRegs.
set(AArch64::X18);
3685 unsigned CSStackSize = 0;
3686 unsigned SVECSStackSize = 0;
3689 for (
unsigned Reg : SavedRegs.
set_bits()) {
3691 if (AArch64::PPRRegClass.
contains(Reg) ||
3692 AArch64::ZPRRegClass.
contains(Reg))
3704 if (AFI->hasStreamingModeChanges()) {
3705 if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
3713 determineStackHazardSlot(MF, SavedRegs);
3714 if (AFI->hasStackHazardSlotIndex())
3718 unsigned NumSavedRegs = SavedRegs.
count();
3724 SavedRegs.
set(AArch64::FP);
3725 SavedRegs.
set(AArch64::LR);
3729 dbgs() <<
"*** determineCalleeSaves\nSaved CSRs:";
3730 for (
unsigned Reg : SavedRegs.
set_bits())
3736 int64_t SVEStackSize =
3737 alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
3738 bool CanEliminateFrame = (SavedRegs.
count() == 0) && !SVEStackSize;
3747 int64_t CalleeStackUsed = 0;
3750 if (FixedOff > CalleeStackUsed)
3751 CalleeStackUsed = FixedOff;
3755 bool BigStack = SVEStackSize || (EstimatedStackSize + CSStackSize +
3756 CalleeStackUsed) > EstimatedStackSizeLimit;
3758 AFI->setHasStackFrame(
true);
3767 if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
3769 <<
" to get a scratch register.\n");
3770 SavedRegs.
set(UnspilledCSGPR);
3771 ExtraCSSpill = UnspilledCSGPR;
3776 if (producePairRegisters(MF)) {
3777 if (UnspilledCSGPRPaired == AArch64::NoRegister) {
3780 SavedRegs.
reset(UnspilledCSGPR);
3781 ExtraCSSpill = AArch64::NoRegister;
3784 SavedRegs.
set(UnspilledCSGPRPaired);
3793 unsigned Size =
TRI->getSpillSize(RC);
3794 Align Alignment =
TRI->getSpillAlign(RC);
3797 LLVM_DEBUG(
dbgs() <<
"No available CS registers, allocated fi#" << FI
3798 <<
" as the emergency spill slot.\n");
3803 CSStackSize += 8 * (SavedRegs.
count() - NumSavedRegs);
3807 if (
hasFP(MF) && AFI->hasSwiftAsyncContext())
3812 << EstimatedStackSize + AlignedCSStackSize <<
" bytes.\n");
3815 AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
3816 "Should not invalidate callee saved info");
3820 AFI->setCalleeSavedStackSize(AlignedCSStackSize);
3821 AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
3822 AFI->setSVECalleeSavedStackSize(
alignTo(SVECSStackSize, 16));
3827 std::vector<CalleeSavedInfo> &CSI,
unsigned &MinCSFrameIndex,
3828 unsigned &MaxCSFrameIndex)
const {
3836 std::reverse(CSI.begin(), CSI.end());
3850 if ((
unsigned)FrameIdx < MinCSFrameIndex)
3851 MinCSFrameIndex = FrameIdx;
3852 if ((
unsigned)FrameIdx > MaxCSFrameIndex)
3853 MaxCSFrameIndex = FrameIdx;
3858 std::vector<CalleeSavedInfo> VGSaves;
3862 VGInfo.setRestored(
false);
3863 VGSaves.push_back(VGInfo);
3867 if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
3868 VGSaves.push_back(VGInfo);
3870 bool InsertBeforeLR =
false;
3872 for (
unsigned I = 0;
I < CSI.size();
I++)
3873 if (CSI[
I].
getReg() == AArch64::LR) {
3874 InsertBeforeLR =
true;
3875 CSI.insert(CSI.begin() +
I, VGSaves.begin(), VGSaves.end());
3879 if (!InsertBeforeLR)
3880 CSI.insert(CSI.end(), VGSaves.begin(), VGSaves.end());
3884 int HazardSlotIndex = std::numeric_limits<int>::max();
3885 for (
auto &CS : CSI) {
3893 assert(HazardSlotIndex == std::numeric_limits<int>::max() &&
3894 "Unexpected register order for hazard slot");
3896 LLVM_DEBUG(
dbgs() <<
"Created CSR Hazard at slot " << HazardSlotIndex
3899 if ((
unsigned)HazardSlotIndex < MinCSFrameIndex)
3900 MinCSFrameIndex = HazardSlotIndex;
3901 if ((
unsigned)HazardSlotIndex > MaxCSFrameIndex)
3902 MaxCSFrameIndex = HazardSlotIndex;
3908 CS.setFrameIdx(FrameIdx);
3910 if ((
unsigned)FrameIdx < MinCSFrameIndex)
3911 MinCSFrameIndex = FrameIdx;
3912 if ((
unsigned)FrameIdx > MaxCSFrameIndex)
3913 MaxCSFrameIndex = FrameIdx;
3917 Reg == AArch64::FP) {
3920 if ((
unsigned)FrameIdx < MinCSFrameIndex)
3921 MinCSFrameIndex = FrameIdx;
3922 if ((
unsigned)FrameIdx > MaxCSFrameIndex)
3923 MaxCSFrameIndex = FrameIdx;
3930 HazardSlotIndex == std::numeric_limits<int>::max()) {
3932 LLVM_DEBUG(
dbgs() <<
"Created CSR Hazard at slot " << HazardSlotIndex
3935 if ((
unsigned)HazardSlotIndex < MinCSFrameIndex)
3936 MinCSFrameIndex = HazardSlotIndex;
3937 if ((
unsigned)HazardSlotIndex > MaxCSFrameIndex)
3938 MaxCSFrameIndex = HazardSlotIndex;
3962 int &Min,
int &Max) {
3963 Min = std::numeric_limits<int>::max();
3964 Max = std::numeric_limits<int>::min();
3970 for (
auto &CS : CSI) {
3971 if (AArch64::ZPRRegClass.
contains(CS.getReg()) ||
3972 AArch64::PPRRegClass.contains(CS.getReg())) {
3973 assert((Max == std::numeric_limits<int>::min() ||
3974 Max + 1 == CS.getFrameIdx()) &&
3975 "SVE CalleeSaves are not consecutive");
3977 Min = std::min(Min, CS.getFrameIdx());
3978 Max = std::max(Max, CS.getFrameIdx());
3981 return Min != std::numeric_limits<int>::max();
3990 int &MinCSFrameIndex,
3991 int &MaxCSFrameIndex,
3992 bool AssignOffsets) {
3997 "SVE vectors should never be passed on the stack by value, only by "
4001 auto Assign = [&MFI](
int FI, int64_t
Offset) {
4011 for (
int I = MinCSFrameIndex;
I <= MaxCSFrameIndex; ++
I) {
4027 int StackProtectorFI = -1;
4031 ObjectsToAllocate.
push_back(StackProtectorFI);
4037 if (
I == StackProtectorFI)
4039 if (MaxCSFrameIndex >=
I &&
I >= MinCSFrameIndex)
4048 for (
unsigned FI : ObjectsToAllocate) {
4053 if (Alignment >
Align(16))
4055 "Alignment of scalable vectors > 16 bytes is not yet supported");
4065int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
4067 int MinCSFrameIndex, MaxCSFrameIndex;
4071int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
4082 "Upwards growing stack unsupported");
4084 int MinCSFrameIndex, MaxCSFrameIndex;
4085 int64_t SVEStackSize =
4086 assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
4106 int64_t FixedObject =
4119 assert(DstReg &&
"There must be a free register after frame setup");
4128struct TagStoreInstr {
4151 std::optional<int64_t> FrameRegUpdate;
4153 unsigned FrameRegUpdateFlags;
4164 :
MBB(
MBB), ZeroData(ZeroData) {
4170 void addInstruction(TagStoreInstr
I) {
4172 TagStores.
back().Offset + TagStores.
back().Size ==
I.Offset) &&
4173 "Non-adjacent tag store instructions.");
4188 const int64_t kMinOffset = -256 * 16;
4189 const int64_t kMaxOffset = 255 * 16;
4192 int64_t BaseRegOffsetBytes = FrameRegOffset.
getFixed();
4193 if (BaseRegOffsetBytes < kMinOffset ||
4194 BaseRegOffsetBytes + (
Size -
Size % 32) > kMaxOffset ||
4198 BaseRegOffsetBytes % 16 != 0) {
4199 Register ScratchReg =
MRI->createVirtualRegister(&AArch64::GPR64RegClass);
4202 BaseReg = ScratchReg;
4203 BaseRegOffsetBytes = 0;
4208 int64_t InstrSize = (
Size > 16) ? 32 : 16;
4211 ? (ZeroData ? AArch64::STZGi : AArch64::STGi)
4212 : (ZeroData ? AArch64::STZ2Gi : AArch64::ST2Gi);
4213 assert(BaseRegOffsetBytes % 16 == 0);
4217 .
addImm(BaseRegOffsetBytes / 16)
4221 if (BaseRegOffsetBytes == 0)
4223 BaseRegOffsetBytes += InstrSize;
4237 :
MRI->createVirtualRegister(&AArch64::GPR64RegClass);
4238 Register SizeReg =
MRI->createVirtualRegister(&AArch64::GPR64RegClass);
4242 int64_t LoopSize =
Size;
4245 if (FrameRegUpdate && *FrameRegUpdate)
4246 LoopSize -= LoopSize % 32;
4248 TII->get(ZeroData ? AArch64::STZGloop_wback
4249 : AArch64::STGloop_wback))
4256 LoopI->
setFlags(FrameRegUpdateFlags);
4258 int64_t ExtraBaseRegUpdate =
4259 FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.
getFixed() -
Size) : 0;
4260 if (LoopSize <
Size) {
4265 TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
4269 .
addImm(1 + ExtraBaseRegUpdate / 16)
4272 }
else if (ExtraBaseRegUpdate) {
4276 TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
4279 .
addImm(std::abs(ExtraBaseRegUpdate))
4289 int64_t
Size, int64_t *TotalOffset) {
4291 if ((
MI.getOpcode() == AArch64::ADDXri ||
4292 MI.getOpcode() == AArch64::SUBXri) &&
4293 MI.getOperand(0).getReg() == Reg &&
MI.getOperand(1).getReg() == Reg) {
4295 int64_t
Offset =
MI.getOperand(2).getImm() << Shift;
4296 if (
MI.getOpcode() == AArch64::SUBXri)
4298 int64_t AbsPostOffset = std::abs(
Offset -
Size);
4299 const int64_t kMaxOffset =
4301 if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
4312 for (
auto &TS : TSE) {
4316 if (
MI->memoperands_empty()) {
4320 MemRefs.
append(
MI->memoperands_begin(),
MI->memoperands_end());
4326 bool TryMergeSPUpdate) {
4327 if (TagStores.
empty())
4329 TagStoreInstr &FirstTagStore = TagStores[0];
4330 TagStoreInstr &LastTagStore = TagStores[TagStores.
size() - 1];
4331 Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
4332 DL = TagStores[0].MI->getDebugLoc();
4336 *MF, FirstTagStore.Offset,
false ,
false , Reg,
4339 FrameRegUpdate = std::nullopt;
4341 mergeMemRefs(TagStores, CombinedMemRefs);
4344 dbgs() <<
"Replacing adjacent STG instructions:\n";
4345 for (
const auto &Instr : TagStores) {
4354 if (TagStores.
size() < 2)
4356 emitUnrolled(InsertI);
4359 int64_t TotalOffset = 0;
4360 if (TryMergeSPUpdate) {
4366 if (InsertI !=
MBB->
end() &&
4367 canMergeRegUpdate(InsertI, FrameReg, FrameRegOffset.
getFixed() +
Size,
4369 UpdateInstr = &*InsertI++;
4375 if (!UpdateInstr && TagStores.
size() < 2)
4379 FrameRegUpdate = TotalOffset;
4380 FrameRegUpdateFlags = UpdateInstr->
getFlags();
4387 for (
auto &TS : TagStores)
4388 TS.MI->eraseFromParent();
4392 int64_t &
Size,
bool &ZeroData) {
4396 unsigned Opcode =
MI.getOpcode();
4397 ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGi ||
4398 Opcode == AArch64::STZ2Gi);
4400 if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
4401 if (!
MI.getOperand(0).isDead() || !
MI.getOperand(1).isDead())
4403 if (!
MI.getOperand(2).isImm() || !
MI.getOperand(3).isFI())
4406 Size =
MI.getOperand(2).getImm();
4410 if (Opcode == AArch64::STGi || Opcode == AArch64::STZGi)
4412 else if (Opcode == AArch64::ST2Gi || Opcode == AArch64::STZ2Gi)
4417 if (
MI.getOperand(0).getReg() != AArch64::SP || !
MI.getOperand(1).isFI())
4421 16 *
MI.getOperand(2).getImm();
4441 if (!isMergeableStackTaggingInstruction(
MI,
Offset,
Size, FirstZeroData))
4447 constexpr int kScanLimit = 10;
4450 NextI != E && Count < kScanLimit; ++NextI) {
4459 if (isMergeableStackTaggingInstruction(
MI,
Offset,
Size, ZeroData)) {
4460 if (ZeroData != FirstZeroData)
4468 if (!
MI.isTransient())
4477 if (
MI.mayLoadOrStore() ||
MI.hasUnmodeledSideEffects())
4493 LiveRegs.addLiveOuts(*
MBB);
4498 LiveRegs.stepBackward(*
I);
4501 if (LiveRegs.contains(AArch64::NZCV))
4505 [](
const TagStoreInstr &
Left,
const TagStoreInstr &
Right) {
4510 int64_t CurOffset = Instrs[0].Offset;
4511 for (
auto &Instr : Instrs) {
4512 if (CurOffset >
Instr.Offset)
4519 TagStoreEdit TSE(
MBB, FirstZeroData);
4520 std::optional<int64_t> EndOffset;
4521 for (
auto &Instr : Instrs) {
4522 if (EndOffset && *EndOffset !=
Instr.Offset) {
4524 TSE.emitCode(InsertI, TFI,
false);
4528 TSE.addInstruction(Instr);
4548 if (
MI.getOpcode() != AArch64::VGSavePseudo &&
4549 MI.getOpcode() != AArch64::VGRestorePseudo)
4553 bool LocallyStreaming =
4560 int64_t VGFrameIdx =
4562 assert(VGFrameIdx != std::numeric_limits<int>::max() &&
4563 "Expected FrameIdx for VG");
4566 if (
MI.getOpcode() == AArch64::VGSavePseudo) {
4571 nullptr,
TRI->getDwarfRegNum(AArch64::VG,
true),
Offset));
4574 nullptr,
TRI->getDwarfRegNum(AArch64::VG,
true)));
4577 TII->get(TargetOpcode::CFI_INSTRUCTION))
4580 MI.eraseFromParent();
4592 II = tryMergeAdjacentSTG(
II,
this, RS);
4601 bool IgnoreSPUpdates)
const {
4603 if (IgnoreSPUpdates) {
4606 FrameReg = AArch64::SP;
4616 FrameReg = AArch64::SP;
4641 bool IsValid =
false;
4643 int ObjectIndex = 0;
4645 int GroupIndex = -1;
4647 bool ObjectFirst =
false;
4650 bool GroupFirst =
false;
4654 unsigned Accesses = 0;
4655 enum { AccessFPR = 1, AccessHazard = 2, AccessGPR = 4 };
4660 int NextGroupIndex = 0;
4661 std::vector<FrameObject> &Objects;
4664 GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
4666 void EndCurrentGroup() {
4667 if (CurrentMembers.
size() > 1) {
4672 for (
int Index : CurrentMembers) {
4673 Objects[
Index].GroupIndex = NextGroupIndex;
4679 CurrentMembers.clear();
4683bool FrameObjectCompare(
const FrameObject &
A,
const FrameObject &
B) {
4705 return std::make_tuple(!
A.IsValid,
A.Accesses,
A.ObjectFirst,
A.GroupFirst,
4706 A.GroupIndex,
A.ObjectIndex) <
4707 std::make_tuple(!
B.IsValid,
B.Accesses,
B.ObjectFirst,
B.GroupFirst,
4708 B.GroupIndex,
B.ObjectIndex);
4719 std::vector<FrameObject> FrameObjects(MFI.getObjectIndexEnd());
4720 for (
auto &Obj : ObjectsToAllocate) {
4721 FrameObjects[Obj].IsValid =
true;
4722 FrameObjects[Obj].ObjectIndex = Obj;
4727 GroupBuilder GB(FrameObjects);
4728 for (
auto &
MBB : MF) {
4729 for (
auto &
MI :
MBB) {
4730 if (
MI.isDebugInstr())
4735 if (FI && *FI >= 0 && *FI < (
int)FrameObjects.size()) {
4738 FrameObjects[*FI].Accesses |= FrameObject::AccessFPR;
4740 FrameObjects[*FI].Accesses |= FrameObject::AccessGPR;
4745 switch (
MI.getOpcode()) {
4746 case AArch64::STGloop:
4747 case AArch64::STZGloop:
4751 case AArch64::STZGi:
4752 case AArch64::ST2Gi:
4753 case AArch64::STZ2Gi:
4765 if (FI >= 0 && FI < MFI.getObjectIndexEnd() &&
4766 FrameObjects[FI].IsValid)
4774 GB.AddMember(TaggedFI);
4776 GB.EndCurrentGroup();
4779 GB.EndCurrentGroup();
4784 FrameObject::AccessHazard;
4786 for (
auto &Obj : FrameObjects)
4787 if (!Obj.Accesses ||
4788 Obj.Accesses == (FrameObject::AccessGPR | FrameObject::AccessFPR))
4789 Obj.Accesses = FrameObject::AccessGPR;
4798 FrameObjects[*TBPI].ObjectFirst =
true;
4799 FrameObjects[*TBPI].GroupFirst =
true;
4800 int FirstGroupIndex = FrameObjects[*TBPI].GroupIndex;
4801 if (FirstGroupIndex >= 0)
4802 for (FrameObject &Object : FrameObjects)
4803 if (Object.GroupIndex == FirstGroupIndex)
4804 Object.GroupFirst =
true;
4810 for (
auto &Obj : FrameObjects) {
4814 ObjectsToAllocate[i++] = Obj.ObjectIndex;
4818 dbgs() <<
"Final frame order:\n";
4819 for (
auto &Obj : FrameObjects) {
4822 dbgs() <<
" " << Obj.ObjectIndex <<
": group " << Obj.GroupIndex;
4823 if (Obj.ObjectFirst)
4824 dbgs() <<
", first";
4826 dbgs() <<
", group-first";
4837AArch64FrameLowering::inlineStackProbeLoopExactMultiple(
4848 MF.
insert(MBBInsertPoint, LoopMBB);
4850 MF.
insert(MBBInsertPoint, ExitMBB);
4885 return ExitMBB->
begin();
4888void AArch64FrameLowering::inlineStackProbeFixed(
4901 int64_t NumBlocks = FrameSize / ProbeSize;
4902 int64_t ResidualSize = FrameSize % ProbeSize;
4904 LLVM_DEBUG(
dbgs() <<
"Stack probing: total " << FrameSize <<
" bytes, "
4905 << NumBlocks <<
" blocks of " << ProbeSize
4906 <<
" bytes, plus " << ResidualSize <<
" bytes\n");
4911 for (
int i = 0; i < NumBlocks; ++i) {
4917 EmitAsyncCFI && !HasFP, CFAOffset);
4926 }
else if (NumBlocks != 0) {
4932 EmitAsyncCFI && !HasFP, CFAOffset);
4934 MBBI = inlineStackProbeLoopExactMultiple(
MBBI, ProbeSize, ScratchReg);
4936 if (EmitAsyncCFI && !HasFP) {
4940 unsigned Reg =
RegInfo.getDwarfRegNum(AArch64::SP,
true);
4949 if (ResidualSize != 0) {
4955 EmitAsyncCFI && !HasFP, CFAOffset);
4974 if (
MI.getOpcode() == AArch64::PROBED_STACKALLOC ||
4975 MI.getOpcode() == AArch64::PROBED_STACKALLOC_VAR)
4979 if (
MI->getOpcode() == AArch64::PROBED_STACKALLOC) {
4980 Register ScratchReg =
MI->getOperand(0).getReg();
4981 int64_t FrameSize =
MI->getOperand(1).getImm();
4983 MI->getOperand(3).getImm());
4984 inlineStackProbeFixed(
MI->getIterator(), ScratchReg, FrameSize,
4987 assert(
MI->getOpcode() == AArch64::PROBED_STACKALLOC_VAR &&
4988 "Stack probe pseudo-instruction expected");
4991 Register TargetReg =
MI->getOperand(0).getReg();
4992 (void)
TII->probedStackAlloc(
MI->getIterator(), TargetReg,
true);
4994 MI->eraseFromParent();
unsigned const MachineRegisterInfo * MRI
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
static int64_t getArgumentStackToRestore(MachineFunction &MF, MachineBasicBlock &MBB)
Returns how much of the incoming argument stack area (in bytes) we should clean up in an epilogue.
static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL)
static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB)
static void emitCalleeSavedRestores(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool SVE)
static void computeCalleeSaveRegisterPairs(MachineFunction &MF, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI, SmallVectorImpl< RegPairInfo > &RegPairs, bool NeedsFrameRecord)
static const unsigned DefaultSafeSPDisplacement
This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using ...
static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned FixedObject)
static bool needsWinCFI(const MachineFunction &MF)
static void insertCFISameValue(const MCInstrDesc &Desc, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, unsigned DwarfReg)
static cl::opt< bool > StackTaggingMergeSetTag("stack-tagging-merge-settag", cl::desc("merge settag instruction in function epilog"), cl::init(true), cl::Hidden)
static cl::opt< unsigned > StackHazardSize("aarch64-stack-hazard-size", cl::init(0), cl::Hidden)
bool requiresGetVGCall(MachineFunction &MF)
bool isVGInstruction(MachineBasicBlock::iterator MBBI)
static std::optional< int > getLdStFrameID(const MachineInstr &MI, const MachineFrameInfo &MFI)
static bool produceCompactUnwindFrame(MachineFunction &MF)
static cl::opt< bool > StackHazardInNonStreaming("aarch64-stack-hazard-in-non-streaming", cl::init(false), cl::Hidden)
static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex, bool AssignOffsets)
static cl::opt< bool > OrderFrameObjects("aarch64-order-frame-objects", cl::desc("sort stack allocations"), cl::init(true), cl::Hidden)
static bool windowsRequiresStackProbe(MachineFunction &MF, uint64_t StackSizeInBytes)
static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, uint64_t LocalStackSize, bool NeedsWinCFI, bool *HasWinCFI)
static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2, bool NeedsWinCFI, bool IsFirst, const TargetRegisterInfo *TRI)
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, MachineInstr::MIFlag FrameFlag=MachineInstr::FrameSetup, int CFAOffset=0)
static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, unsigned LocalStackSize)
static StackOffset getSVEStackSize(const MachineFunction &MF)
Returns the size of the entire SVE stackframe (calleesaves + spills).
static cl::opt< bool > EnableRedZone("aarch64-redzone", cl::desc("enable use of redzone on AArch64"), cl::init(false), cl::Hidden)
static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI, const TargetInstrInfo &TII, MachineInstr::MIFlag Flag)
static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB)
static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI, LivePhysRegs &LiveRegs)
Collect live registers from the end of MI's parent up to (including) MI in LiveRegs.
cl::opt< bool > EnableHomogeneousPrologEpilog("homogeneous-prolog-epilog", cl::Hidden, cl::desc("Emit homogeneous prologue and epilogue for the size " "optimization (default = off)"))
MachineBasicBlock::iterator emitVGSaveRestore(MachineBasicBlock::iterator II, const AArch64FrameLowering *TFI)
static bool IsSVECalleeSave(MachineBasicBlock::iterator I)
static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2, bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord, bool IsFirst, const TargetRegisterInfo *TRI)
Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
unsigned findFreePredicateReg(BitVector &SavedRegs)
static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg)
static StackOffset getFPOffset(const MachineFunction &MF, int64_t ObjectOffset)
static bool isTargetWindows(const MachineFunction &MF)
static StackOffset getStackOffset(const MachineFunction &MF, int64_t ObjectOffset)
static int64_t upperBound(StackOffset Size)
static unsigned estimateRSStackSizeLimit(MachineFunction &MF)
Look at each instruction that references stack frames and return the stack size limit beyond which so...
static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI, int &Min, int &Max)
returns true if there are any SVE callee saves.
static MCRegister getRegisterOrZero(MCRegister Reg, bool HasSVE)
static bool isFuncletReturnInstr(const MachineInstr &MI)
static void emitShadowCallStackPrologue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool NeedsWinCFI, bool NeedsUnwindInfo)
static unsigned getFixedObjectSize(const MachineFunction &MF, const AArch64FunctionInfo *AFI, bool IsWin64, bool IsFunclet)
Returns the size of the fixed object area (allocated next to sp on entry) On Win64 this may include a...
static const int kSetTagLoopThreshold
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
This file contains the simple types necessary to represent the attributes associated with functions a...
#define CASE(ATTRNAME, AANAME,...)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static void clear(coro::Shape &Shape)
static const HTTPClientCleanup Cleanup
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
uint64_t IntrinsicInst * II
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static const unsigned FramePtr
void processFunctionBeforeFrameIndicesReplaced(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameIndicesReplaced - This method is called immediately before MO_FrameIndex op...
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
bool enableStackSlotScavenging(const MachineFunction &MF) const override
Returns true if the stack slot holes in the fixed and callee-save stack area should be used when allo...
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
StackOffset getNonLocalFrameIndexReference(const MachineFunction &MF, int FI) const override
getNonLocalFrameIndexReference - This method returns the offset used to reference a frame index locat...
TargetStackID::Value getStackIDForScalableVectors() const override
Returns the StackID that scalable vectors should be associated with.
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register.
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
bool enableCFIFixup(MachineFunction &MF) const override
Returns true if we may need to fix the unwind information for the function.
void resetCFIToInitialState(MachineBasicBlock &MBB) const override
Emit CFI instructions that recreate the state of the unwind information upon fucntion entry.
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
bool canUseRedZone(const MachineFunction &MF) const
Can this function use the red zone for local allocations.
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
int getSEHFrameIndexOffset(const MachineFunction &MF, int FI) const
unsigned getWinEHFuncletFrameSize(const MachineFunction &MF) const
Funclets only need to account for space for the callee saved registers, as the locals are accounted f...
void orderFrameObjects(const MachineFunction &MF, SmallVectorImpl< int > &ObjectsToAllocate) const override
Order the symbols in the local stack frame.
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - Provide a base+offset reference to an FI slot for debug info.
StackOffset resolveFrameOffsetReference(const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE, Register &FrameReg, bool PreferFP, bool ForSimm) const
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI, unsigned &MinCSFrameIndex, unsigned &MaxCSFrameIndex) const override
assignCalleeSavedSpillSlots - Allows target to override spill slot assignment logic.
StackOffset getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI, Register &FrameReg, bool IgnoreSPUpdates) const override
For Win64 AArch64 EH, the offset to the Unwind object is from the SP before the update.
StackOffset resolveFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP, bool ForSimm) const
unsigned getWinEHParentFrameOffset(const MachineFunction &MF) const override
The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve the parent's frame pointer...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF) const
void setSwiftAsyncContextFrameIdx(int FI)
unsigned getTailCallReservedStack() const
unsigned getCalleeSavedStackSize(const MachineFrameInfo &MFI) const
void setCalleeSaveBaseToFrameRecordOffset(int Offset)
bool hasStackProbing() const
unsigned getArgumentStackToRestore() const
void setLocalStackSize(uint64_t Size)
void setVGIdx(unsigned Idx)
int getCalleeSaveBaseToFrameRecordOffset() const
bool hasStreamingModeChanges() const
bool shouldSignReturnAddress(const MachineFunction &MF) const
void setPredicateRegForFillSpill(unsigned Reg)
int getStackHazardSlotIndex() const
void setStreamingVGIdx(unsigned FrameIdx)
int64_t getStackProbeSize() const
uint64_t getStackSizeSVE() const
void setHasRedZone(bool s)
bool hasStackFrame() const
std::optional< int > getTaggedBasePointerIndex() const
uint64_t getLocalStackSize() const
void setStackRealigned(bool s)
bool needsDwarfUnwindInfo(const MachineFunction &MF) const
unsigned getVarArgsGPRSize() const
void setStackSizeSVE(uint64_t S)
bool isStackRealigned() const
bool hasSwiftAsyncContext() const
bool hasStackHazardSlotIndex() const
void setTaggedBasePointerOffset(unsigned Offset)
void setStackHazardCSRSlotIndex(int Index)
unsigned getPredicateRegForFillSpill() const
unsigned getSVECalleeSavedStackSize() const
bool needsAsyncDwarfUnwindInfo(const MachineFunction &MF) const
int64_t getStreamingVGIdx() const
void setMinMaxSVECSFrameIndex(int Min, int Max)
bool hasCalleeSaveStackFreeSpace() const
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
static bool isFpOrNEON(Register Reg)
Returns whether the physical register is FP or NEON.
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
bool hasBasePointer(const MachineFunction &MF) const
bool cannotEliminateFrame(const MachineFunction &MF) const
unsigned getBaseRegister() const
bool isTargetWindows() const
const AArch64RegisterInfo * getRegisterInfo() const override
bool isNeonAvailable() const
Returns true if the target has NEON and the function at runtime is known to have NEON enabled (e....
const AArch64InstrInfo * getInstrInfo() const override
bool isTargetILP32() const
const AArch64TargetLowering * getTargetLowering() const override
bool isTargetMachO() const
const Triple & getTargetTriple() const
const char * getChkStkName() const
bool isCallingConvWin64(CallingConv::ID CC, bool IsVarArg) const
bool swiftAsyncContextIsDynamicallySet() const
Return whether FrameLowering should always set the "extended frame present" bit in FP,...
bool hasInlineStackProbe(const MachineFunction &MF) const override
True if stack clash protection is enabled for this functions.
unsigned getRedZoneSize(const Function &F) const
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
bool hasAttrSomewhere(Attribute::AttrKind Kind, unsigned *Index=nullptr) const
Return true if the specified attribute is set for at least one parameter or for the return value.
bool test(unsigned Idx) const
size_type count() const
count - Returns the number of bits which are set.
iterator_range< const_set_bits_iterator > set_bits() const
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
Emit instructions to copy a pair of physical registers.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const
Returns true if register Reg and no aliasing register is in the set.
void stepBackward(const MachineInstr &MI)
Simulates liveness when stepping backwards over an instruction(bundle).
void removeReg(MCPhysReg Reg)
Removes a physical register, all its sub-registers, and all its super-registers from the set.
void addLiveIns(const MachineBasicBlock &MBB)
Adds all live-in registers of basic block MBB.
void addLiveOuts(const MachineBasicBlock &MBB)
Adds all live-out registers of basic block MBB.
void addReg(MCPhysReg Reg)
Adds a physical register and all its sub-registers to the set.
bool usesWindowsCFI() const
static MCCFIInstruction createDefCfaRegister(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_def_cfa_register modifies a rule for computing CFA.
static MCCFIInstruction createRestore(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_restore says that the rule for Register is now the same as it was at the beginning of the functi...
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
static MCCFIInstruction createNegateRAState(MCSymbol *L, SMLoc Loc={})
.cfi_negate_ra_state AArch64 negate RA state.
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
static MCCFIInstruction createSameValue(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_same_value Current value of Register is the same as in the previous frame.
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Describe properties that are true of each instruction in the target description file.
Wrapper class representing physical registers. Should be passed by value.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
instr_iterator instr_begin()
iterator_range< livein_iterator > liveins() const
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MachineInstr & instr_back()
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
instr_iterator instr_end()
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
reverse_iterator rbegin()
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
const AllocaInst * getObjectAllocation(int ObjectIdx) const
Return the underlying Alloca of the specified stack object if it exists.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasCalls() const
Return true if the current function has any function calls.
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
Align getMaxAlign() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
void setObjectOffset(int ObjectIdx, int64_t SPOffset)
Set the stack frame offset of the specified object.
uint64_t getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
int getStackProtectorIndex() const
Return the index for the stack protector object.
uint64_t estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
void setStackID(int ObjectIdx, uint8_t ID)
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isMaxCallFrameSizeComputed() const
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
bool hasStackProtectorIndex() const
uint8_t getStackID(int ObjectIdx) const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
int getObjectIndexBegin() const
Return the minimum frame object index.
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
unsigned addFrameInst(const MCCFIInstruction &Inst)
void setHasWinCFI(bool v)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
bool hasEHFunclets() const
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
void setFlags(unsigned flags)
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
uint32_t getFlags() const
Return the MI flags bitvector.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const Value * getValue() const
Return the base address of the memory access.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
static MachineOperand CreateImm(int64_t Val)
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool isLiveIn(Register Reg) const
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
void backward()
Update internal register state and move MBB iterator backwards.
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
Wrapper class representing virtual and physical registers.
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
bool hasStreamingInterface() const
bool hasStreamingBody() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
static StackOffset get(int64_t Fixed, int64_t Scalable)
static StackOffset getScalable(int64_t Scalable)
static StackOffset getFixed(int64_t Fixed)
StringRef - Represent a constant reference to a string, i.e.
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool enableCFIFixup(MachineFunction &MF) const
Returns true if we may need to fix the unwind information for the function.
TargetInstrInfo - Interface to description of machine instruction set.
CodeModel::Model getCodeModel() const
Returns the code model.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
SwiftAsyncFramePointerMode SwiftAsyncFramePointer
Control when and how the Swift async frame pointer bit should be set.
bool DisableFramePointerElim(const MachineFunction &MF) const
DisableFramePointerElim - This returns true if frame pointer elimination optimization should be disab...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const TargetRegisterClass * getMinimalPhysRegClass(MCRegister Reg, MVT VT=MVT::Other) const
Returns the Register Class of a physical register of the given type, picking the most sub register cl...
Align getSpillAlign(const TargetRegisterClass &RC) const
Return the minimum required alignment in bytes for a spill slot for a register of this class.
bool hasStackRealignment(const MachineFunction &MF) const
True if stack realignment is required and still possible.
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
StringRef getArchName() const
Get the architecture (first) component of the triple.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
const unsigned StackProbeMaxLoopUnroll
Maximum number of iterations to unroll for a constant size probing loop.
const unsigned StackProbeMaxUnprobedStack
Maximum allowed number of unprobed bytes above SP at an ABI boundary.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
@ AArch64_SVE_VectorCall
Used between AArch64 SVE functions.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ CXX_FAST_TLS
Used for access functions.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1
Preserve X1-X15, X19-X29, SP, Z0-Z31, P0-P15.
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ PreserveNone
Used for runtime calls that preserves none general registers.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
NodeAddr< InstrNode * > Instr
This is an optimization pass for GlobalISel generic memory operations.
void stable_sort(R &&Range)
MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg, unsigned Reg, const StackOffset &Offset, bool LastAdjustmentWasScalable=true)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
@ Always
Always set the bit.
@ Never
Never set the bit.
@ DeploymentBased
Determine whether to set the bit statically or dynamically based on the deployment target.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
unsigned getDefRegState(bool B)
unsigned getKillRegState(bool B)
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Description of the encoding of one expression Op.
Pair of physical register and lane mask.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.