40#define DEBUG_TYPE "x86-fl"
42STATISTIC(NumFrameLoopProbe,
"Number of loop stack probes used in prologue");
44 "Number of extra stack probes generated in prologue");
52 STI(STI),
TII(*STI.getInstrInfo()),
TRI(STI.getRegisterInfo()) {
76 (
hasFP(MF) && !
TRI->hasStackRealignment(MF)) ||
111 return X86::SUB64ri8;
112 return X86::SUB64ri32;
115 return X86::SUB32ri8;
123 return X86::ADD64ri8;
124 return X86::ADD64ri32;
127 return X86::ADD32ri8;
133 return IsLP64 ? X86::SUB64rr : X86::SUB32rr;
137 return IsLP64 ? X86::ADD64rr : X86::ADD32rr;
143 return X86::AND64ri8;
144 return X86::AND64ri32;
147 return X86::AND32ri8;
152 return IsLP64 ? X86::LEA64r : X86::LEA32r;
158 return X86::MOV32ri64;
160 return X86::MOV64ri32;
168 unsigned Reg = RegMask.PhysReg;
170 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
171 Reg == X86::AH || Reg == X86::AL)
185 bool BreakNext =
false;
190 if (Reg != X86::EFLAGS)
211 if (Succ->isLiveIn(X86::EFLAGS))
222 int64_t NumBytes,
bool InEpilogue)
const {
223 bool isSub = NumBytes < 0;
237 if (EmitInlineStackProbe && !InEpilogue) {
243 }
else if (
Offset > Chunk) {
254 unsigned AddSubRROpc =
263 MI->getOperand(3).setIsDead();
265 }
else if (
Offset > 8 * Chunk) {
289 MI->getOperand(3).setIsDead();
311 ? (
Is64Bit ? X86::PUSH64r : X86::PUSH32r)
312 : (
Is64Bit ? X86::POP64r : X86::POP32r);
321 BuildStackAdjustment(
MBB,
MBBI,
DL, isSub ? -ThisVal : ThisVal, InEpilogue)
331 assert(
Offset != 0 &&
"zero offset stack adjustment requested");
349 if (UseLEA && !
STI.useLeaForSP())
354 "We shouldn't have allowed this insertion point");
371 MI->getOperand(3).setIsDead();
378 bool doMergeWithPrevious)
const {
397 if (doMergeWithPrevious && PI !=
MBB.
begin() && PI->isCFIInstruction())
400 unsigned Opc = PI->getOpcode();
403 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 ||
404 Opc == X86::ADD32ri || Opc == X86::ADD32ri8) &&
405 PI->getOperand(0).getReg() ==
StackPtr){
407 Offset = PI->getOperand(2).getImm();
408 }
else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
409 PI->getOperand(0).getReg() ==
StackPtr &&
410 PI->getOperand(1).getReg() ==
StackPtr &&
411 PI->getOperand(2).getImm() == 1 &&
412 PI->getOperand(3).getReg() == X86::NoRegister &&
413 PI->getOperand(5).getReg() == X86::NoRegister) {
415 Offset = PI->getOperand(4).getImm();
416 }
else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 ||
417 Opc == X86::SUB32ri || Opc == X86::SUB32ri8) &&
418 PI->getOperand(0).getReg() ==
StackPtr) {
420 Offset = -PI->getOperand(2).getImm();
425 if (PI !=
MBB.
end() && PI->isCFIInstruction()) {
432 if (!doMergeWithPrevious)
465 unsigned DwarfReg =
MRI->getDwarfRegNum(MachineFramePtr,
true);
489 unsigned DwarfReg =
MRI->getDwarfRegNum(Reg,
true);
500 CfaExpr.
push_back(dwarf::DW_CFA_expression);
509 unsigned DwarfFramePtr =
MRI->getDwarfRegNum(MachineFramePtr,
true);
510 CfaExpr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
525 int FI =
MI->getOperand(1).getIndex();
533 unsigned DwarfFramePtr =
MRI->getDwarfRegNum(MachineFramePtr,
true);
534 CfaExpr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfFramePtr));
540 DefCfaExpr.
push_back(dwarf::DW_CFA_def_cfa_expression);
550void X86FrameLowering::emitZeroCallUsedRegs(
BitVector RegsToZero,
566 if (!X86::RFP80RegClass.
contains(Reg))
569 unsigned NumFPRegs = ST.is64Bit() ? 8 : 7;
570 for (
unsigned i = 0; i != NumFPRegs; ++i)
573 for (
unsigned i = 0; i != NumFPRegs; ++i)
581 if (
TRI->isGeneralPurposeRegister(MF, Reg)) {
583 RegsToZero.
reset(Reg);
593 if (
ST.hasMMX() && X86::VR64RegClass.contains(Reg))
598 if (X86::VR128RegClass.
contains(Reg)) {
603 }
else if (X86::VR256RegClass.
contains(Reg)) {
607 XorOp = X86::VPXORrr;
608 }
else if (X86::VR512RegClass.
contains(Reg)) {
612 XorOp = X86::VPXORYrr;
613 }
else if (X86::VK1RegClass.
contains(Reg) ||
620 XorOp =
ST.hasBWI() ? X86::KXORQrr : X86::KXORWrr;
634 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum)
const {
641 emitStackProbeInline(MF,
MBB,
MBBI,
DL,
false);
644 emitStackProbeCall(MF,
MBB,
MBBI,
DL, InProlog, InstrNum);
655 return MI.getOpcode() == X86::STACKALLOC_W_PROBING;
657 if (Where != PrologMBB.
end()) {
659 emitStackProbeInline(MF, PrologMBB, Where,
DL,
true);
660 Where->eraseFromParent();
668 bool InProlog)
const {
671 emitStackProbeInlineWindowsCoreCLR64(MF,
MBB,
MBBI,
DL, InProlog);
673 emitStackProbeInlineGeneric(MF,
MBB,
MBBI,
DL, InProlog);
676void X86FrameLowering::emitStackProbeInlineGeneric(
685 "different expansion expected for CoreCLR 64 bit");
687 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
688 uint64_t ProbeChunk = StackProbeSize * 8;
691 TRI->hasStackRealignment(MF) ? calculateMaxStackAlign(MF) : 0;
696 if (
Offset > ProbeChunk) {
698 MaxAlign % StackProbeSize);
701 MaxAlign % StackProbeSize);
705void X86FrameLowering::emitStackProbeInlineGenericBlock(
710 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
714 const unsigned MovMIOpc =
Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
715 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
719 assert(AlignOffset < StackProbeSize);
722 if (StackProbeSize <
Offset + AlignOffset) {
725 BuildStackAdjustment(
MBB,
MBBI,
DL, -StackAdjustment,
false)
727 if (!HasFP && NeedsDwarfCFI) {
738 NumFrameExtraProbe++;
739 CurrentOffset = StackProbeSize - AlignOffset;
745 while (CurrentOffset + StackProbeSize <
Offset) {
746 BuildStackAdjustment(
MBB,
MBBI,
DL, -StackProbeSize,
false)
749 if (!HasFP && NeedsDwarfCFI) {
759 NumFrameExtraProbe++;
760 CurrentOffset += StackProbeSize;
769 unsigned Opc =
Is64Bit ? X86::PUSH64r : X86::PUSH32r;
774 BuildStackAdjustment(
MBB,
MBBI,
DL, -ChunkSize,
false)
781void X86FrameLowering::emitStackProbeInlineGenericLoop(
789 "Inline stack probe loop will clobber live EFLAGS.");
791 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
795 const unsigned MovMIOpc =
Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
796 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
799 if (AlignOffset < StackProbeSize) {
801 BuildStackAdjustment(
MBB,
MBBI,
DL, -AlignOffset,
false)
809 NumFrameExtraProbe++;
822 MF.
insert(MBBIter, testMBB);
823 MF.
insert(MBBIter, tailMBB);
844 if (!HasFP && NeedsDwarfCFI) {
847 const Register DwarfFinalStackProbed =
854 nullptr,
TRI->getDwarfRegNum(DwarfFinalStackProbed,
true)));
861 BuildStackAdjustment(*testMBB, testMBB->
end(),
DL, -StackProbeSize,
895 BuildStackAdjustment(*tailMBB, TailMBBIter,
DL, -TailOffset,
901 if (!HasFP && NeedsDwarfCFI) {
911 nullptr,
TRI->getDwarfRegNum(DwarfStackPtr,
true)));
919void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64(
923 assert(
STI.is64Bit() &&
"different expansion needed for 32 bit");
930 "Inline stack probe loop will clobber live EFLAGS.");
965 MF.
insert(MBBIter, RoundMBB);
966 MF.
insert(MBBIter, LoopMBB);
967 MF.
insert(MBBIter, ContinueMBB);
975 const int64_t ThreadEnvironmentStackLimit = 0x10;
977 const int64_t PageMask = ~(
PageSize - 1);
983 const Register SizeReg = InProlog ? X86::RAX
984 :
MRI.createVirtualRegister(RegClass),
985 ZeroReg = InProlog ? X86::RCX
986 :
MRI.createVirtualRegister(RegClass),
987 CopyReg = InProlog ? X86::RDX
988 :
MRI.createVirtualRegister(RegClass),
989 TestReg = InProlog ? X86::RDX
990 :
MRI.createVirtualRegister(RegClass),
991 FinalReg = InProlog ? X86::RDX
992 :
MRI.createVirtualRegister(RegClass),
993 RoundedReg = InProlog ? X86::RDX
994 :
MRI.createVirtualRegister(RegClass),
995 LimitReg = InProlog ? X86::RCX
996 :
MRI.createVirtualRegister(RegClass),
997 JoinReg = InProlog ? X86::RCX
998 :
MRI.createVirtualRegister(RegClass),
999 ProbeReg = InProlog ? X86::RCX
1000 :
MRI.createVirtualRegister(RegClass);
1003 int64_t RCXShadowSlot = 0;
1004 int64_t RDXShadowSlot = 0;
1020 int64_t InitSlot = 8 + CalleeSaveSize + (
HasFP ? 8 : 0);
1024 RCXShadowSlot = InitSlot;
1026 RDXShadowSlot = InitSlot;
1027 if (IsRDXLiveIn && IsRCXLiveIn)
1068 .
addImm(ThreadEnvironmentStackLimit)
1076 BuildMI(RoundMBB,
DL,
TII.get(X86::AND64ri32), RoundedReg)
1117 TII.get(X86::MOV64rm), X86::RCX),
1118 X86::RSP,
false, RCXShadowSlot);
1121 TII.get(X86::MOV64rm), X86::RDX),
1122 X86::RSP,
false, RDXShadowSlot);
1128 BuildMI(*ContinueMBB, ContinueMBBI,
DL,
TII.get(X86::SUB64rr), X86::RSP)
1141 for (++BeforeMBBI; BeforeMBBI !=
MBB.
end(); ++BeforeMBBI) {
1157void X86FrameLowering::emitStackProbeCall(
1160 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum)
const {
1166 "code model and indirect thunks not yet implemented.");
1170 "Stack probe calls will clobber live EFLAGS.");
1174 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
1176 CallOp = X86::CALLpcrel32;
1237 for (++ExpansionMBBI; ExpansionMBBI !=
MBBI; ++ExpansionMBBI)
1245 const uint64_t Win64MaxSEHOffset = 128;
1246 uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset);
1248 return SEHFrameOffset & -16;
1261 MaxAlign = (
StackAlign > MaxAlign) ? StackAlign : MaxAlign;
1265 return MaxAlign.
value();
1278 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
1279 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
1284 if (Reg ==
StackPtr && EmitInlineStackProbe && MaxAlign >= StackProbeSize) {
1286 NumFrameLoopProbe++;
1297 MF.
insert(MBBIter, entryMBB);
1298 MF.
insert(MBBIter, headMBB);
1299 MF.
insert(MBBIter, bodyMBB);
1300 MF.
insert(MBBIter, footMBB);
1301 const unsigned MovMIOpc =
Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
1310 BuildMI(entryMBB,
DL,
TII.get(TargetOpcode::COPY), FinalStackProbed)
1314 BuildMI(entryMBB,
DL,
TII.get(AndOp), FinalStackProbed)
1315 .
addReg(FinalStackProbed)
1320 MI->getOperand(3).setIsDead();
1324 .
addReg(FinalStackProbed)
1338 const unsigned SUBOpc =
1348 .
addReg(FinalStackProbed)
1369 const unsigned SUBOpc =
1379 .
addReg(FinalStackProbed)
1395 .
addReg(FinalStackProbed)
1417 MI->getOperand(3).setIsDead();
1425 "MF used frame lowering for wrong subtarget");
1434bool X86FrameLowering::isWin64Prologue(
const MachineFunction &MF)
const {
1438bool X86FrameLowering::needsDwarfCFI(
const MachineFunction &MF)
const {
1530 "MF used frame lowering for wrong subtarget");
1536 uint64_t MaxAlign = calculateMaxStackAlign(MF);
1542 bool FnHasClrFunclet =
1544 bool IsClrFunclet = IsFunclet && FnHasClrFunclet;
1545 bool HasFP =
hasFP(MF);
1546 bool IsWin64Prologue = isWin64Prologue(MF);
1551 bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO;
1552 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1558 bool HasWinCFI =
false;
1569 ArgBaseReg =
MI->getOperand(0).getReg();
1582 if (NeedsDwarfCFI) {
1584 unsigned DwarfStackPtr =
TRI->getDwarfRegNum(ArgBaseReg,
true);
1603 if (TailCallArgReserveSize && IsWin64Prologue)
1606 const bool EmitStackProbeCall =
1623 .
addUse(X86::NoRegister);
1658 !EmitStackProbeCall &&
1665 StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0);
1672 if (TailCallArgReserveSize != 0) {
1673 BuildStackAdjustment(
MBB,
MBBI,
DL, -(
int)TailCallArgReserveSize,
1696 Register Establisher = X86::NoRegister;
1702 if (IsWin64Prologue && IsFunclet && !IsClrFunclet) {
1718 NumBytes = FrameSize -
1722 if (
TRI->hasStackRealignment(MF) && !IsWin64Prologue)
1723 NumBytes =
alignTo(NumBytes, MaxAlign);
1730 if (NeedsDwarfCFI && !ArgBaseReg.
isValid()) {
1736 nullptr, -2 * stackGrowth + (
int)TailCallArgReserveSize),
1740 unsigned DwarfFramePtr =
TRI->getDwarfRegNum(MachineFramePtr,
true);
1744 (
int)TailCallArgReserveSize),
1762 if (Attrs.hasAttrSomewhere(Attribute::SwiftAsync)) {
1797 if (!IsWin64Prologue && !IsFunclet) {
1806 if (NeedsDwarfCFI) {
1809 CfaExpr.
push_back(dwarf::DW_CFA_expression);
1811 unsigned DwarfReg =
TRI->getDwarfRegNum(MachineFramePtr,
true);
1814 CfaExpr.
push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
1823 unsigned DwarfFramePtr =
TRI->getDwarfRegNum(MachineFramePtr,
true);
1842 assert(!IsFunclet &&
"funclets without FPs not yet implemented");
1843 NumBytes = StackSize -
1850 if (HasFP &&
TRI->hasStackRealignment(MF))
1858 unsigned ParentFrameNumBytes = NumBytes;
1860 NumBytes = getWinEHFuncletFrameSize(MF);
1863 bool PushedRegs =
false;
1868 (
MBBI->getOpcode() == X86::PUSH32r ||
1869 MBBI->getOpcode() == X86::PUSH64r)) {
1874 if (!HasFP && NeedsDwarfCFI) {
1895 if (!IsWin64Prologue && !IsFunclet &&
TRI->hasStackRealignment(MF) &&
1897 assert(HasFP &&
"There should be a frame pointer if stack is realigned.");
1923 uint64_t AlignedNumBytes = NumBytes;
1924 if (IsWin64Prologue && !IsFunclet &&
TRI->hasStackRealignment(MF))
1925 AlignedNumBytes =
alignTo(AlignedNumBytes, MaxAlign);
1926 if (AlignedNumBytes >= StackProbeSize && EmitStackProbeCall) {
1928 "The Red Zone is not accounted for in stack probes");
1950 int64_t
Alloc = isEAXAlive ? NumBytes - 8 : NumBytes;
1958 .
addImm(isEAXAlive ? NumBytes - 4 : NumBytes)
1977 }
else if (NumBytes) {
1981 if (NeedsWinCFI && NumBytes) {
1988 int SEHFrameOffset = 0;
1989 unsigned SPOrEstablisher;
1996 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2000 Establisher,
false, PSPSlotOffset)
2007 false, PSPSlotOffset)
2014 SPOrEstablisher = Establisher;
2019 if (IsWin64Prologue && HasFP) {
2026 SPOrEstablisher,
false, SEHFrameOffset);
2029 .
addReg(SPOrEstablisher);
2032 if (NeedsWinCFI && !IsFunclet) {
2033 assert(!NeedsWinFPO &&
"this setframe incompatible with FPO data");
2042 }
else if (IsFunclet &&
STI.is32Bit()) {
2066 if (X86::FR64RegClass.
contains(Reg)) {
2069 if (IsWin64Prologue && IsFunclet)
2077 assert(!NeedsWinFPO &&
"SEH_SaveXMM incompatible with FPO data");
2087 if (NeedsWinCFI && HasWinCFI)
2091 if (FnHasClrFunclet && !IsFunclet) {
2095 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2109 if (IsWin64Prologue &&
TRI->hasStackRealignment(MF)) {
2110 assert(HasFP &&
"There should be a frame pointer if stack is realigned.");
2111 BuildStackAlignAND(
MBB,
MBBI,
DL, SPOrEstablisher, MaxAlign);
2115 if (IsFunclet &&
STI.is32Bit())
2148 assert(UsedReg == BasePtr);
2157 int FI =
MI->getOperand(1).getIndex();
2158 unsigned MOVmr =
Is64Bit ? X86::MOV64mr : X86::MOV32mr;
2165 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
2167 if (!HasFP && NumBytes) {
2208 switch (
MI.getOpcode()) {
2210 case X86::CLEANUPRET:
2232X86FrameLowering::getPSPSlotOffsetFromSP(
const MachineFunction &MF)
const {
2239 return static_cast<unsigned>(
Offset);
2243X86FrameLowering::getWinEHFuncletFrameSize(
const MachineFunction &MF)
const {
2249 unsigned XMMSize = WinEHXMMSlotInfo.
size() *
2250 TRI->getSpillSize(X86::VR128RegClass);
2259 UsedSize = getPSPSlotOffsetFromSP(MF) +
SlotSize;
2270 return FrameSizeMinusRBP + XMMSize - CSSize;
2274 return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi ||
2275 Opc == X86::TCRETURNmi ||
2276 Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNdi64 ||
2277 Opc == X86::TCRETURNmi64;
2288 DL =
MBBI->getDebugLoc();
2296 bool NeedsWin64CFI =
2302 uint64_t MaxAlign = calculateMaxStackAlign(MF);
2305 bool HasFP =
hasFP(MF);
2314 unsigned Opc = X86::LEA32r;
2316 ArgBaseReg =
MI->getOperand(0).getReg();
2317 if (
STI.is64Bit()) {
2319 StackReg = X86::RSP;
2330 if (NeedsDwarfCFI) {
2331 unsigned DwarfStackPtr =
TRI->getDwarfRegNum(StackReg,
true);
2341 assert(HasFP &&
"EH funclets without FP not yet implemented");
2342 NumBytes = getWinEHFuncletFrameSize(MF);
2346 NumBytes = FrameSize - CSSize - TailCallArgReserveSize;
2350 if (
TRI->hasStackRealignment(MF) && !IsWin64Prologue)
2351 NumBytes =
alignTo(FrameSize, MaxAlign);
2353 NumBytes = StackSize - CSSize - TailCallArgReserveSize;
2355 uint64_t SEHStackAllocAmt = NumBytes;
2380 if (NeedsDwarfCFI) {
2382 unsigned DwarfStackPtr =
2383 TRI->getDwarfRegNum(
Is64Bit ? X86::RSP : X86::ESP,
true);
2389 unsigned DwarfFramePtr =
TRI->getDwarfRegNum(MachineFramePtr,
true);
2404 unsigned Opc = PI->getOpcode();
2406 if (Opc != X86::DBG_VALUE && !PI->isTerminator()) {
2420 int FI =
MI->getOperand(1).getIndex();
2421 unsigned MOVrm =
Is64Bit ? X86::MOV64rm : X86::MOV32rm;
2428 if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET)
2429 emitCatchRetReturnValue(
MBB, FirstCSPop, &*Terminator);
2432 DL =
MBBI->getDebugLoc();
2444 if (
TRI->hasStackRealignment(MF))
2448 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
2460 if (LEAAmount != 0) {
2471 }
else if (NumBytes) {
2474 if (!HasFP && NeedsDwarfCFI) {
2478 nullptr, CSSize + TailCallArgReserveSize +
SlotSize),
2493 if (!HasFP && NeedsDwarfCFI) {
2500 unsigned Opc = PI->getOpcode();
2502 if (Opc == X86::POP32r || Opc == X86::POP64r) {
2520 assert(
Offset >= 0 &&
"TCDelta should never be positive");
2544 else if (
TRI->hasStackRealignment(MF))
2558 int64_t FPDelta = 0;
2569 if (IsWin64Prologue) {
2577 uint64_t NumBytes = FrameSize - CSSize;
2587 FPDelta = FrameSize - SEHFrameOffset;
2589 "FPDelta isn't aligned per the Win64 ABI!");
2601 if (TailCallReturnAddrDelta < 0)
2602 Offset -= TailCallReturnAddrDelta;
2620 const auto it = WinEHXMMSlotInfo.find(FI);
2622 if (it == WinEHXMMSlotInfo.end())
2633 int Adjustment)
const {
2643 bool IgnoreSPUpdates)
const {
2693 "we don't handle this case!");
2725 std::vector<CalleeSavedInfo> &CSI)
const {
2729 unsigned CalleeSavedFrameSize = 0;
2730 unsigned XMMCalleeSavedFrameSize = 0;
2736 if (TailCallReturnAddrDelta < 0) {
2747 TailCallReturnAddrDelta -
SlotSize,
true);
2751 if (this->TRI->hasBasePointer(MF)) {
2777 for (
unsigned i = 0; i < CSI.size(); ++i) {
2778 if (
TRI->regsOverlap(CSI[i].getReg(),FPReg)) {
2779 CSI.erase(CSI.begin() + i);
2820 if (X86::VK16RegClass.
contains(Reg))
2824 unsigned Size =
TRI->getSpillSize(*RC);
2825 Align Alignment =
TRI->getSpillAlign(*RC);
2827 assert(SpillSlotOffset < 0 &&
"SpillSlotOffset should always < 0 on X86");
2828 SpillSlotOffset = -
alignTo(-SpillSlotOffset, Alignment);
2831 SpillSlotOffset -=
Size;
2837 if (X86::VR128RegClass.
contains(Reg)) {
2838 WinEHXMMSlotInfo[
SlotIndex] = XMMCalleeSavedFrameSize;
2839 XMMCalleeSavedFrameSize +=
Size;
2858 unsigned Opc =
STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
2866 bool isLiveIn =
MRI.isLiveIn(Reg);
2871 bool CanKill = !isLiveIn;
2875 if (
MRI.isLiveIn(*AReg)) {
2893 unsigned Opc =
STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
2894 Register BaseReg = this->TRI->getBaseRegister();
2909 if (X86::VK16RegClass.
contains(Reg))
2932 "SEH should not use CATCHRET");
2937 if (
STI.is64Bit()) {
2969 if (
MI->getOpcode() == X86::CATCHRET) {
2983 if (X86::GR64RegClass.
contains(Reg) ||
2989 if (X86::VK16RegClass.
contains(Reg))
3001 unsigned Opc =
STI.is64Bit() ? X86::POP64r : X86::POP32r;
3002 Register BaseReg = this->TRI->getBaseRegister();
3008 unsigned Opc =
STI.is64Bit() ? X86::POP64r : X86::POP32r;
3011 if (!X86::GR64RegClass.
contains(Reg) &&
3031 SavedRegs.
set(BasePtr);
3040 if (
I->hasNestAttr() && !
I->use_empty())
3057 return Primary ? X86::R14 : X86::R13;
3059 return Primary ? X86::EBX : X86::EDI;
3064 return Primary ? X86::R11 : X86::R12;
3066 return Primary ? X86::R11D : X86::R12D;
3076 "nested function.");
3077 return Primary ? X86::EAX : X86::ECX;
3080 return Primary ? X86::EDX : X86::EAX;
3081 return Primary ? X86::ECX : X86::EAX;
3092 unsigned TlsReg, TlsOffset;
3097 assert(&(*MF.
begin()) == &PrologueMBB &&
"Shrink-wrapping not supported yet");
3101 "Scratch register is live-in");
3121 bool IsNested =
false;
3130 for (
const auto &LI : PrologueMBB.
liveins()) {
3149 TlsOffset =
IsLP64 ? 0x70 : 0x40;
3152 TlsOffset = 0x60 + 90*8;
3166 if (CompareStackPointer)
3167 ScratchReg =
IsLP64 ? X86::RSP : X86::ESP;
3180 TlsOffset = 0x48 + 90*4;
3193 if (CompareStackPointer)
3194 ScratchReg = X86::ESP;
3206 unsigned ScratchReg2;
3208 if (CompareStackPointer) {
3211 SaveScratch2 =
false;
3223 "Scratch register is live-in and not saved");
3229 BuildMI(checkMBB,
DL,
TII.get(X86::MOV32ri), ScratchReg2)
3238 BuildMI(checkMBB,
DL,
TII.get(X86::POP32r), ScratchReg2);
3252 const unsigned RegAX =
IsLP64 ? X86::RAX : X86::EAX;
3253 const unsigned Reg10 =
IsLP64 ? X86::R10 : X86::R10D;
3254 const unsigned Reg11 =
IsLP64 ? X86::R11 : X86::R11D;
3255 const unsigned MOVrr =
IsLP64 ? X86::MOV64rr : X86::MOV32rr;
3292 "code model and thunks not yet implemented.");
3309 BuildMI(allocMBB,
DL,
TII.get(X86::MORESTACK_RET_RESTORE_R10));
3318#ifdef EXPENSIVE_CHECKS
3329 for (
int i = 0, e = HiPELiteralsMD->
getNumOperands(); i != e; ++i) {
3331 if (
Node->getNumOperands() != 2)
continue;
3332 MDString *NodeName = dyn_cast<MDString>(
Node->getOperand(0));
3334 if (!NodeName || !NodeVal)
continue;
3336 if (ValConst && NodeName->
getString() == LiteralName) {
3342 +
" required but not provided");
3353 return MI.isMetaInstruction();
3379 assert(&(*MF.
begin()) == &PrologueMBB &&
"Shrink-wrapping not supported yet");
3384 if (!HiPELiteralsMD)
3386 "Can't generate HiPE prologue without runtime parameters");
3387 const unsigned HipeLeafWords
3389 Is64Bit ?
"AMD64_LEAF_WORDS" :
"X86_LEAF_WORDS");
3390 const unsigned CCRegisteredArgs =
Is64Bit ? 6 : 5;
3391 const unsigned Guaranteed = HipeLeafWords *
SlotSize;
3397 "HiPE prologue is only supported on Linux operating systems.");
3407 unsigned MoreStackForCalls = 0;
3409 for (
auto &
MBB : MF) {
3410 for (
auto &
MI :
MBB) {
3430 if (
F->getName().contains(
"erlang.") ||
F->getName().contains(
"bif_") ||
3434 unsigned CalleeStkArity =
3435 F->arg_size() > CCRegisteredArgs ?
F->arg_size()-CCRegisteredArgs : 0;
3436 if (HipeLeafWords - 1 > CalleeStkArity)
3437 MoreStackForCalls = std::max(MoreStackForCalls,
3438 (HipeLeafWords - 1 - CalleeStkArity) *
SlotSize);
3441 MaxStack += MoreStackForCalls;
3446 if (MaxStack > Guaranteed) {
3450 for (
const auto &LI : PrologueMBB.
liveins()) {
3458 unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
3459 unsigned LEAop, CMPop, CALLop;
3464 LEAop = X86::LEA64r;
3465 CMPop = X86::CMP64rm;
3466 CALLop = X86::CALL64pcrel32;
3470 LEAop = X86::LEA32r;
3471 CMPop = X86::CMP32rm;
3472 CALLop = X86::CALLpcrel32;
3477 "HiPE prologue scratch register is live-in");
3481 SPReg,
false, -MaxStack);
3484 .
addReg(ScratchReg), PReg,
false, SPLimitOffset);
3489 addExternalSymbol(
"inc_stack_0");
3491 SPReg,
false, -MaxStack);
3493 .
addReg(ScratchReg), PReg,
false, SPLimitOffset);
3501#ifdef EXPENSIVE_CHECKS
3518 if (NumPops != 1 && NumPops != 2)
3526 if (!Prev->isCall() || !Prev->getOperand(1).isRegMask())
3530 unsigned FoundRegs = 0;
3536 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass;
3538 for (
auto Candidate : RegClass) {
3546 if (
MRI.isReserved(Candidate))
3551 if (MO.isReg() && MO.isDef() &&
3552 TRI->isSuperOrSubRegisterEq(MO.getReg(), Candidate)) {
3561 Regs[FoundRegs++] = Candidate;
3562 if (FoundRegs == (
unsigned)NumPops)
3570 while (FoundRegs < (
unsigned)NumPops)
3571 Regs[FoundRegs++] = Regs[0];
3573 for (
int i = 0; i < NumPops; ++i)
3575 TII.get(
STI.is64Bit() ? X86::POP64r : X86::POP32r), Regs[i]);
3584 unsigned Opcode =
I->getOpcode();
3585 bool isDestroy = Opcode ==
TII.getCallFrameDestroyOpcode();
3598 if (!reserveCallFrame) {
3619 bool HasDwarfEHHandlers = !WindowsCFI && !MF.
getLandingPads().empty();
3621 if (HasDwarfEHHandlers && !isDestroy &&
3631 Amount -= InternalAmt;
3641 int64_t StackAdjustment = isDestroy ? Amount : -Amount;
3643 if (StackAdjustment) {
3650 if (StackAdjustment) {
3651 if (!(
F.hasMinSize() &&
3652 adjustStackWithPops(
MBB, InsertPos,
DL, StackAdjustment)))
3653 BuildStackAdjustment(
MBB, InsertPos,
DL, StackAdjustment,
3666 int64_t CfaAdjustment = -StackAdjustment;
3669 if (CfaAdjustment) {
3682 while (CI !=
B && !std::prev(CI)->isCall())
3684 BuildStackAdjustment(
MBB, CI,
DL, -InternalAmt,
false);
3700 if (TLI.hasInlineStackProbe(MF) || TLI.hasStackProbeSymbol(MF))
3736 bool CompactUnwind =
3756 "restoring EBP/ESI on non-32-bit target");
3767 int FI =
FuncInfo.EHRegNodeFrameIndex;
3773 X86::EBP,
true, -EHRegSize)
3779 int EndOffset = -EHRegOffset - EHRegSize;
3780 FuncInfo.EHRegNodeEndOffset = EndOffset;
3792 "end of registration object above normal EBP position!");
3793 }
else if (UsedReg == BasePtr) {
3803 assert(UsedReg == BasePtr);
3824struct X86FrameSortingObject {
3825 bool IsValid =
false;
3826 unsigned ObjectIndex = 0;
3827 unsigned ObjectSize = 0;
3829 unsigned ObjectNumUses = 0;
3845struct X86FrameSortingComparator {
3846 inline bool operator()(
const X86FrameSortingObject &
A,
3847 const X86FrameSortingObject &
B)
const {
3848 uint64_t DensityAScaled, DensityBScaled;
3868 DensityAScaled =
static_cast<uint64_t>(
A.ObjectNumUses) *
3870 DensityBScaled =
static_cast<uint64_t>(
B.ObjectNumUses) *
3881 if (DensityAScaled == DensityBScaled)
3882 return A.ObjectAlignment <
B.ObjectAlignment;
3884 return DensityAScaled < DensityBScaled;
3898 if (ObjectsToAllocate.
empty())
3910 for (
auto &Obj : ObjectsToAllocate) {
3911 SortingObjects[Obj].IsValid =
true;
3912 SortingObjects[Obj].ObjectIndex = Obj;
3916 if (ObjectSize == 0)
3918 SortingObjects[Obj].ObjectSize = 4;
3920 SortingObjects[Obj].ObjectSize = ObjectSize;
3924 for (
auto &
MBB : MF) {
3925 for (
auto &
MI :
MBB) {
3926 if (
MI.isDebugInstr())
3932 int Index = MO.getIndex();
3936 SortingObjects[
Index].IsValid)
3937 SortingObjects[
Index].ObjectNumUses++;
3952 for (
auto &Obj : SortingObjects) {
3956 ObjectsToAllocate[i++] = Obj.ObjectIndex;
3960 if (!
TRI->hasStackRealignment(MF) &&
hasFP(MF))
3961 std::reverse(ObjectsToAllocate.
begin(), ObjectsToAllocate.
end());
3973 Offset += getWinEHFuncletFrameSize(MF);
3993 adjustFrameForMsvcCxxEh(MF);
3997void X86FrameLowering::adjustFrameForMsvcCxxEh(
MachineFunction &MF)
const {
4005 int64_t MinFixedObjOffset = -
SlotSize;
4011 int FrameIndex =
H.CatchObj.FrameIndex;
4012 if (FrameIndex != INT_MAX) {
4015 MinFixedObjOffset -= std::abs(MinFixedObjOffset) %
Align;
4023 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % 8;
4024 int64_t UnwindHelpOffset = MinFixedObjOffset -
SlotSize;
4051 MI->eraseFromParent();
unsigned const MachineRegisterInfo * MRI
static bool isFuncletReturnInstr(const MachineInstr &MI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static const uint64_t kSplitStackAvailable
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Given that RA is a live value
const HexagonInstrInfo * TII
static cl::opt< int > PageSize("imp-null-check-page-size", cl::desc("The page size of the target in bytes"), cl::init(4096), cl::Hidden)
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
static bool isTailCallOpcode(unsigned Opc)
unsigned const TargetRegisterInfo * TRI
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static bool is64Bit(const char *name)
static unsigned calculateSetFPREG(uint64_t SPAdjust)
static unsigned GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Primary)
GetScratchRegister - Get a temp register for performing work in the segmented stack and the Erlang/Hi...
static unsigned getMOVriOpcode(bool Use64BitReg, int64_t Imm)
static unsigned getLEArOpcode(bool IsLP64)
static unsigned getADDriOpcode(bool IsLP64, int64_t Imm)
static bool flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB)
Check if the flags need to be preserved before the terminators.
static unsigned getSUBriOpcode(bool IsLP64, int64_t Imm)
static unsigned getANDriOpcode(bool IsLP64, int64_t Imm)
static bool isEAXLiveIn(MachineBasicBlock &MBB)
static unsigned getADDrrOpcode(bool IsLP64)
static bool HasNestArgument(const MachineFunction *MF)
static unsigned getHiPELiteral(NamedMDNode *HiPELiteralsMD, const StringRef LiteralName)
Lookup an ERTS parameter in the !hipe.literals named metadata node.
static bool blockEndIsUnreachable(const MachineBasicBlock &MBB, MachineBasicBlock::const_iterator MBBI)
static unsigned getSUBrrOpcode(bool IsLP64)
static const unsigned FramePtr
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool empty() const
empty - Check if the array is empty.
LLVM Basic Block Representation.
iterator_range< const_set_bits_iterator > set_bits() const
static BranchProbability getOne()
static BranchProbability getZero()
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
bool hasPersonalityFn() const
Check whether this function has a personality function.
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
bool needsUnwindTableEntry() const
True if this function needs an unwind table.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
bool usesWindowsCFI() const
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int Offset)
.cfi_def_cfa_offset modifies a rule for computing CFA.
static MCCFIInstruction createRestore(MCSymbol *L, unsigned Register)
.cfi_restore says that the rule for Register is now the same as it was at the beginning of the functi...
static MCCFIInstruction createGnuArgsSize(MCSymbol *L, int Size)
A special wrapper for .cfi_escape that indicates GNU_ARGS_SIZE.
static MCCFIInstruction createDefCfaRegister(MCSymbol *L, unsigned Register)
.cfi_def_cfa_register modifies a rule for computing CFA.
OpType getOperation() const
static MCCFIInstruction createAdjustCfaOffset(MCSymbol *L, int Adjustment)
.cfi_adjust_cfa_offset Same as .cfi_def_cfa_offset, but Offset is a relative value that is added/subt...
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int Offset)
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset)
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
const MCObjectFileInfo * getObjectFileInfo() const
const MCRegisterInfo * getRegisterInfo() const
MCSection * getCompactUnwindSection() const
MCRegAliasIterator enumerates all registers aliasing Reg.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
StringRef getString() const
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
bool isEHPad() const
Returns true if the block is a landing pad.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator_range< livein_iterator > liveins() const
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
LivenessQueryResult computeRegisterLiveness(const TargetRegisterInfo *TRI, MCRegister Reg, const_iterator Before, unsigned Neighborhood=10) const
Return whether (physical) register Reg has been defined and not killed as of just before Before.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
bool isReturnBlock() const
Convenience function that returns true if the block ends in a return instruction.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any DBG_VALUE and DBG_LABEL instructions.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.