LLVM 22.0.0git
SILowerSGPRSpills.cpp
Go to the documentation of this file.
1//===-- SILowerSGPRSPills.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Handle SGPR spills. This pass takes the place of PrologEpilogInserter for all
10// SGPR spills, so must insert CSR SGPR spills as well as expand them.
11//
12// This pass must never create new SGPR virtual registers.
13//
14// FIXME: Must stop RegScavenger spills in later passes.
15//
16//===----------------------------------------------------------------------===//
17
18#include "SILowerSGPRSpills.h"
19#include "AMDGPU.h"
20#include "GCNSubtarget.h"
28
29using namespace llvm;
30
31#define DEBUG_TYPE "si-lower-sgpr-spills"
32
34
35namespace {
36
37static cl::opt<unsigned> MaxNumVGPRsForWwmAllocation(
38 "amdgpu-num-vgprs-for-wwm-alloc",
39 cl::desc("Max num VGPRs for whole-wave register allocation."),
41
42class SILowerSGPRSpills {
43private:
44 const SIRegisterInfo *TRI = nullptr;
45 const SIInstrInfo *TII = nullptr;
46 LiveIntervals *LIS = nullptr;
47 SlotIndexes *Indexes = nullptr;
48 MachineDominatorTree *MDT = nullptr;
49
50 // Save and Restore blocks of the current function. Typically there is a
51 // single save block, unless Windows EH funclets are involved.
52 MBBVector SaveBlocks;
53 MBBVector RestoreBlocks;
54
55public:
56 SILowerSGPRSpills(LiveIntervals *LIS, SlotIndexes *Indexes,
58 : LIS(LIS), Indexes(Indexes), MDT(MDT) {}
59 bool run(MachineFunction &MF);
60 void calculateSaveRestoreBlocks(MachineFunction &MF);
61 bool spillCalleeSavedRegs(MachineFunction &MF,
62 SmallVectorImpl<int> &CalleeSavedFIs);
63 void updateLaneVGPRDomInstr(
66 void determineRegsForWWMAllocation(MachineFunction &MF, BitVector &RegMask);
67};
68
69class SILowerSGPRSpillsLegacy : public MachineFunctionPass {
70public:
71 static char ID;
72
73 SILowerSGPRSpillsLegacy() : MachineFunctionPass(ID) {}
74
75 bool runOnMachineFunction(MachineFunction &MF) override;
76
77 void getAnalysisUsage(AnalysisUsage &AU) const override {
79 AU.setPreservesAll();
81 }
82
83 MachineFunctionProperties getClearedProperties() const override {
84 // SILowerSGPRSpills introduces new Virtual VGPRs for spilling SGPRs.
85 return MachineFunctionProperties().setIsSSA().setNoVRegs();
86 }
87};
88
89} // end anonymous namespace
90
91char SILowerSGPRSpillsLegacy::ID = 0;
92
93INITIALIZE_PASS_BEGIN(SILowerSGPRSpillsLegacy, DEBUG_TYPE,
94 "SI lower SGPR spill instructions", false, false)
98INITIALIZE_PASS_END(SILowerSGPRSpillsLegacy, DEBUG_TYPE,
99 "SI lower SGPR spill instructions", false, false)
100
101char &llvm::SILowerSGPRSpillsLegacyID = SILowerSGPRSpillsLegacy::ID;
102
105 for (MCRegAliasIterator R(Reg, TRI, true); R.isValid(); ++R) {
106 if (MBB.isLiveIn(*R)) {
107 return true;
108 }
109 }
110 return false;
111}
112
113/// Insert spill code for the callee-saved registers used in the function.
114static void insertCSRSaves(MachineBasicBlock &SaveBlock,
116 LiveIntervals *LIS) {
117 MachineFunction &MF = *SaveBlock.getParent();
121 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
122 const SIRegisterInfo *RI = ST.getRegisterInfo();
123
124 MachineBasicBlock::iterator I = SaveBlock.begin();
125 if (!TFI->spillCalleeSavedRegisters(SaveBlock, I, CSI, TRI)) {
126 for (const CalleeSavedInfo &CS : CSI) {
127 // Insert the spill to the stack frame.
128 MCRegister Reg = CS.getReg();
129
130 MachineInstrSpan MIS(I, &SaveBlock);
131 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(
132 Reg, Reg == RI->getReturnAddressReg(MF) ? MVT::i64 : MVT::i32);
133
134 // If this value was already livein, we probably have a direct use of the
135 // incoming register value, so don't kill at the spill point. This happens
136 // since we pass some special inputs (workgroup IDs) in the callee saved
137 // range.
138 const bool IsLiveIn = isLiveIntoMBB(Reg, SaveBlock, TRI);
139 TII.storeRegToStackSlot(SaveBlock, I, Reg, !IsLiveIn, CS.getFrameIdx(),
140 RC, TRI, Register());
141
142 if (Indexes) {
143 assert(std::distance(MIS.begin(), I) == 1);
144 MachineInstr &Inst = *std::prev(I);
145 Indexes->insertMachineInstrInMaps(Inst);
146 }
147
148 if (LIS)
150 }
151 } else {
152 // TFI doesn't update Indexes and LIS, so we have to do it separately.
153 if (Indexes)
154 Indexes->repairIndexesInRange(&SaveBlock, SaveBlock.begin(), I);
155
156 if (LIS)
157 for (const CalleeSavedInfo &CS : CSI)
158 LIS->removeAllRegUnitsForPhysReg(CS.getReg());
159 }
160}
161
162/// Insert restore code for the callee-saved registers used in the function.
163static void insertCSRRestores(MachineBasicBlock &RestoreBlock,
165 SlotIndexes *Indexes, LiveIntervals *LIS) {
166 MachineFunction &MF = *RestoreBlock.getParent();
170 // Restore all registers immediately before the return and any
171 // terminators that precede it.
173 const MachineBasicBlock::iterator BeforeRestoresI =
174 I == RestoreBlock.begin() ? I : std::prev(I);
175
176 // FIXME: Just emit the readlane/writelane directly
177 if (!TFI->restoreCalleeSavedRegisters(RestoreBlock, I, CSI, TRI)) {
178 for (const CalleeSavedInfo &CI : reverse(CSI)) {
179 // Insert in reverse order. loadRegFromStackSlot can insert
180 // multiple instructions.
181 TFI->restoreCalleeSavedRegister(RestoreBlock, I, CI, &TII, TRI);
182
183 if (Indexes) {
184 MachineInstr &Inst = *std::prev(I);
185 Indexes->insertMachineInstrInMaps(Inst);
186 }
187
188 if (LIS)
189 LIS->removeAllRegUnitsForPhysReg(CI.getReg());
190 }
191 } else {
192 // TFI doesn't update Indexes and LIS, so we have to do it separately.
193 if (Indexes)
194 Indexes->repairIndexesInRange(&RestoreBlock, BeforeRestoresI,
195 RestoreBlock.getFirstTerminator());
196
197 if (LIS)
198 for (const CalleeSavedInfo &CS : CSI)
199 LIS->removeAllRegUnitsForPhysReg(CS.getReg());
200 }
201}
202
203/// Compute the sets of entry and return blocks for saving and restoring
204/// callee-saved registers, and placing prolog and epilog code.
205void SILowerSGPRSpills::calculateSaveRestoreBlocks(MachineFunction &MF) {
206 const MachineFrameInfo &MFI = MF.getFrameInfo();
207
208 // Even when we do not change any CSR, we still want to insert the
209 // prologue and epilogue of the function.
210 // So set the save points for those.
211
212 // Use the points found by shrink-wrapping, if any.
213 if (!MFI.getSavePoints().empty()) {
214 assert(MFI.getSavePoints().size() == 1 &&
215 "Multiple save points not yet supported!");
216 const auto &SavePoint = *MFI.getSavePoints().begin();
217 SaveBlocks.push_back(SavePoint.first);
218 assert(MFI.getRestorePoints().size() == 1 &&
219 "Multiple restore points not yet supported!");
220 const auto &RestorePoint = *MFI.getRestorePoints().begin();
221 MachineBasicBlock *RestoreBlock = RestorePoint.first;
222 // If RestoreBlock does not have any successor and is not a return block
223 // then the end point is unreachable and we do not need to insert any
224 // epilogue.
225 if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
226 RestoreBlocks.push_back(RestoreBlock);
227 return;
228 }
229
230 // Save refs to entry and return blocks.
231 SaveBlocks.push_back(&MF.front());
232 for (MachineBasicBlock &MBB : MF) {
233 if (MBB.isEHFuncletEntry())
234 SaveBlocks.push_back(&MBB);
235 if (MBB.isReturnBlock())
236 RestoreBlocks.push_back(&MBB);
237 }
238}
239
240// TODO: To support shrink wrapping, this would need to copy
241// PrologEpilogInserter's updateLiveness.
243 MachineBasicBlock &EntryBB = MF.front();
244
245 for (const CalleeSavedInfo &CSIReg : CSI)
246 EntryBB.addLiveIn(CSIReg.getReg());
247 EntryBB.sortUniqueLiveIns();
248}
249
250bool SILowerSGPRSpills::spillCalleeSavedRegs(
251 MachineFunction &MF, SmallVectorImpl<int> &CalleeSavedFIs) {
252 MachineRegisterInfo &MRI = MF.getRegInfo();
253 const Function &F = MF.getFunction();
254 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
255 const SIFrameLowering *TFI = ST.getFrameLowering();
256 MachineFrameInfo &MFI = MF.getFrameInfo();
257 RegScavenger *RS = nullptr;
258
259 // Determine which of the registers in the callee save list should be saved.
260 BitVector SavedRegs;
261 TFI->determineCalleeSavesSGPR(MF, SavedRegs, RS);
262
263 // Add the code to save and restore the callee saved registers.
264 if (!F.hasFnAttribute(Attribute::Naked)) {
265 // FIXME: This is a lie. The CalleeSavedInfo is incomplete, but this is
266 // necessary for verifier liveness checks.
267 MFI.setCalleeSavedInfoValid(true);
268
269 std::vector<CalleeSavedInfo> CSI;
270 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
271
272 for (unsigned I = 0; CSRegs[I]; ++I) {
273 MCRegister Reg = CSRegs[I];
274
275 if (SavedRegs.test(Reg)) {
276 const TargetRegisterClass *RC =
277 TRI->getMinimalPhysRegClass(Reg, MVT::i32);
278 int JunkFI = MFI.CreateStackObject(TRI->getSpillSize(*RC),
279 TRI->getSpillAlign(*RC), true);
280
281 CSI.emplace_back(Reg, JunkFI);
282 CalleeSavedFIs.push_back(JunkFI);
283 }
284 }
285
286 if (!CSI.empty()) {
287 for (MachineBasicBlock *SaveBlock : SaveBlocks)
288 insertCSRSaves(*SaveBlock, CSI, Indexes, LIS);
289
290 // Add live ins to save blocks.
291 assert(SaveBlocks.size() == 1 && "shrink wrapping not fully implemented");
292 updateLiveness(MF, CSI);
293
294 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
295 insertCSRRestores(*RestoreBlock, CSI, Indexes, LIS);
296 return true;
297 }
298 }
299
300 return false;
301}
302
303void SILowerSGPRSpills::updateLaneVGPRDomInstr(
304 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
305 DenseMap<Register, MachineBasicBlock::iterator> &LaneVGPRDomInstr) {
306 // For the Def of a virtual LaneVPGR to dominate all its uses, we should
307 // insert an IMPLICIT_DEF before the dominating spill. Switching to a
308 // depth first order doesn't really help since the machine function can be in
309 // the unstructured control flow post-SSA. For each virtual register, hence
310 // finding the common dominator to get either the dominating spill or a block
311 // dominating all spills.
312 SIMachineFunctionInfo *FuncInfo =
313 MBB->getParent()->getInfo<SIMachineFunctionInfo>();
315 FuncInfo->getSGPRSpillToVirtualVGPRLanes(FI);
316 Register PrevLaneVGPR;
317 for (auto &Spill : VGPRSpills) {
318 if (PrevLaneVGPR == Spill.VGPR)
319 continue;
320
321 PrevLaneVGPR = Spill.VGPR;
322 auto I = LaneVGPRDomInstr.find(Spill.VGPR);
323 if (Spill.Lane == 0 && I == LaneVGPRDomInstr.end()) {
324 // Initially add the spill instruction itself for Insertion point.
325 LaneVGPRDomInstr[Spill.VGPR] = InsertPt;
326 } else {
327 assert(I != LaneVGPRDomInstr.end());
328 auto PrevInsertPt = I->second;
329 MachineBasicBlock *DomMBB = PrevInsertPt->getParent();
330 if (DomMBB == MBB) {
331 // The insertion point earlier selected in a predecessor block whose
332 // spills are currently being lowered. The earlier InsertPt would be
333 // the one just before the block terminator and it should be changed
334 // if we insert any new spill in it.
335 if (MDT->dominates(&*InsertPt, &*PrevInsertPt))
336 I->second = InsertPt;
337
338 continue;
339 }
340
341 // Find the common dominator block between PrevInsertPt and the
342 // current spill.
343 DomMBB = MDT->findNearestCommonDominator(DomMBB, MBB);
344 if (DomMBB == MBB)
345 I->second = InsertPt;
346 else if (DomMBB != PrevInsertPt->getParent())
347 I->second = &(*DomMBB->getFirstTerminator());
348 }
349 }
350}
351
352void SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF,
353 BitVector &RegMask) {
354 // Determine an optimal number of VGPRs for WWM allocation. The complement
355 // list will be available for allocating other VGPR virtual registers.
356 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
357 MachineRegisterInfo &MRI = MF.getRegInfo();
358 BitVector ReservedRegs = TRI->getReservedRegs(MF);
359 BitVector NonWwmAllocMask(TRI->getNumRegs());
360 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
361
362 // FIXME: MaxNumVGPRsForWwmAllocation might need to be adjusted in the future
363 // to have a balanced allocation between WWM values and per-thread vector
364 // register operands.
365 unsigned NumRegs = MaxNumVGPRsForWwmAllocation;
366 NumRegs =
367 std::min(static_cast<unsigned>(MFI->getSGPRSpillVGPRs().size()), NumRegs);
368
369 auto [MaxNumVGPRs, MaxNumAGPRs] = ST.getMaxNumVectorRegs(MF.getFunction());
370 // Try to use the highest available registers for now. Later after
371 // vgpr-regalloc, they can be shifted to the lowest range.
372 unsigned I = 0;
373 for (unsigned Reg = AMDGPU::VGPR0 + MaxNumVGPRs - 1;
374 (I < NumRegs) && (Reg >= AMDGPU::VGPR0); --Reg) {
375 if (!ReservedRegs.test(Reg) &&
376 !MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true)) {
377 TRI->markSuperRegs(RegMask, Reg);
378 ++I;
379 }
380 }
381
382 if (I != NumRegs) {
383 // Reserve an arbitrary register and report the error.
384 TRI->markSuperRegs(RegMask, AMDGPU::VGPR0);
386 "cannot find enough VGPRs for wwm-regalloc");
387 }
388}
389
390bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) {
391 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
392 LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
393 auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();
394 SlotIndexes *Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
395 MachineDominatorTree *MDT =
396 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
397 return SILowerSGPRSpills(LIS, Indexes, MDT).run(MF);
398}
399
400bool SILowerSGPRSpills::run(MachineFunction &MF) {
401 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
402 TII = ST.getInstrInfo();
403 TRI = &TII->getRegisterInfo();
404
405 assert(SaveBlocks.empty() && RestoreBlocks.empty());
406
407 // First, expose any CSR SGPR spills. This is mostly the same as what PEI
408 // does, but somewhat simpler.
409 calculateSaveRestoreBlocks(MF);
410 SmallVector<int> CalleeSavedFIs;
411 bool HasCSRs = spillCalleeSavedRegs(MF, CalleeSavedFIs);
412
413 MachineFrameInfo &MFI = MF.getFrameInfo();
414 MachineRegisterInfo &MRI = MF.getRegInfo();
415 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
416
417 if (!MFI.hasStackObjects() && !HasCSRs) {
418 SaveBlocks.clear();
419 RestoreBlocks.clear();
420 return false;
421 }
422
423 bool MadeChange = false;
424 bool SpilledToVirtVGPRLanes = false;
425
426 // TODO: CSR VGPRs will never be spilled to AGPRs. These can probably be
427 // handled as SpilledToReg in regular PrologEpilogInserter.
428 const bool HasSGPRSpillToVGPR = TRI->spillSGPRToVGPR() &&
429 (HasCSRs || FuncInfo->hasSpilledSGPRs());
430 if (HasSGPRSpillToVGPR) {
431 // Process all SGPR spills before frame offsets are finalized. Ideally SGPRs
432 // are spilled to VGPRs, in which case we can eliminate the stack usage.
433 //
434 // This operates under the assumption that only other SGPR spills are users
435 // of the frame index.
436
437 // To track the spill frame indices handled in this pass.
438 BitVector SpillFIs(MFI.getObjectIndexEnd(), false);
439
440 // To track the IMPLICIT_DEF insertion point for the lane vgprs.
441 DenseMap<Register, MachineBasicBlock::iterator> LaneVGPRDomInstr;
442
443 for (MachineBasicBlock &MBB : MF) {
444 for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
445 if (!TII->isSGPRSpill(MI))
446 continue;
447
448 if (MI.getOperand(0).isUndef()) {
449 if (Indexes)
451 MI.eraseFromParent();
452 continue;
453 }
454
455 int FI = TII->getNamedOperand(MI, AMDGPU::OpName::addr)->getIndex();
457
458 bool IsCalleeSaveSGPRSpill = llvm::is_contained(CalleeSavedFIs, FI);
459 if (IsCalleeSaveSGPRSpill) {
460 // Spill callee-saved SGPRs into physical VGPR lanes.
461
462 // TODO: This is to ensure the CFIs are static for efficient frame
463 // unwinding in the debugger. Spilling them into virtual VGPR lanes
464 // involve regalloc to allocate the physical VGPRs and that might
465 // cause intermediate spill/split of such liveranges for successful
466 // allocation. This would result in broken CFI encoding unless the
467 // regalloc aware CFI generation to insert new CFIs along with the
468 // intermediate spills is implemented. There is no such support
469 // currently exist in the LLVM compiler.
470 if (FuncInfo->allocateSGPRSpillToVGPRLane(
471 MF, FI, /*SpillToPhysVGPRLane=*/true)) {
472 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
473 MI, FI, nullptr, Indexes, LIS, true);
474 if (!Spilled)
476 "failed to spill SGPR to physical VGPR lane when allocated");
477 }
478 } else {
479 MachineInstrSpan MIS(&MI, &MBB);
480 if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) {
481 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
482 MI, FI, nullptr, Indexes, LIS);
483 if (!Spilled)
485 "failed to spill SGPR to virtual VGPR lane when allocated");
486 SpillFIs.set(FI);
487 updateLaneVGPRDomInstr(FI, &MBB, MIS.begin(), LaneVGPRDomInstr);
488 SpilledToVirtVGPRLanes = true;
489 }
490 }
491 }
492 }
493
494 for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
495 auto InsertPt = LaneVGPRDomInstr[Reg];
496 // Insert the IMPLICIT_DEF at the identified points.
497 MachineBasicBlock &Block = *InsertPt->getParent();
498 DebugLoc DL = Block.findDebugLoc(InsertPt);
499 auto MIB =
500 BuildMI(Block, *InsertPt, DL, TII->get(AMDGPU::IMPLICIT_DEF), Reg);
501
502 // Add WWM flag to the virtual register.
504
505 // Set SGPR_SPILL asm printer flag
506 MIB->setAsmPrinterFlag(AMDGPU::SGPR_SPILL);
507 if (LIS) {
508 LIS->InsertMachineInstrInMaps(*MIB);
510 }
511 }
512
513 // Determine the registers for WWM allocation and also compute the register
514 // mask for non-wwm VGPR allocation.
515 if (FuncInfo->getSGPRSpillVGPRs().size()) {
516 BitVector WwmRegMask(TRI->getNumRegs());
517
518 determineRegsForWWMAllocation(MF, WwmRegMask);
519
520 BitVector NonWwmRegMask(WwmRegMask);
521 NonWwmRegMask.flip().clearBitsNotInMask(TRI->getAllVGPRRegMask());
522
523 // The complement set will be the registers for non-wwm (per-thread) vgpr
524 // allocation.
525 FuncInfo->updateNonWWMRegMask(NonWwmRegMask);
526 }
527
528 for (MachineBasicBlock &MBB : MF) {
529 // FIXME: The dead frame indices are replaced with a null register from
530 // the debug value instructions. We should instead, update it with the
531 // correct register value. But not sure the register value alone is
532 // adequate to lower the DIExpression. It should be worked out later.
533 for (MachineInstr &MI : MBB) {
534 if (MI.isDebugValue()) {
535 uint32_t StackOperandIdx = MI.isDebugValueList() ? 2 : 0;
536 if (MI.getOperand(StackOperandIdx).isFI() &&
538 MI.getOperand(StackOperandIdx).getIndex()) &&
539 SpillFIs[MI.getOperand(StackOperandIdx).getIndex()]) {
540 MI.getOperand(StackOperandIdx)
541 .ChangeToRegister(Register(), false /*isDef*/);
542 }
543 }
544 }
545 }
546
547 // All those frame indices which are dead by now should be removed from the
548 // function frame. Otherwise, there is a side effect such as re-mapping of
549 // free frame index ids by the later pass(es) like "stack slot coloring"
550 // which in turn could mess-up with the book keeping of "frame index to VGPR
551 // lane".
552 FuncInfo->removeDeadFrameIndices(MFI, /*ResetSGPRSpillStackIDs*/ false);
553
554 MadeChange = true;
555 }
556
557 if (SpilledToVirtVGPRLanes) {
558 const TargetRegisterClass *RC = TRI->getWaveMaskRegClass();
559 // Shift back the reserved SGPR for EXEC copy into the lowest range.
560 // This SGPR is reserved to handle the whole-wave spill/copy operations
561 // that might get inserted during vgpr regalloc.
562 Register UnusedLowSGPR = TRI->findUnusedRegister(MRI, RC, MF);
563 if (UnusedLowSGPR && TRI->getHWRegIndex(UnusedLowSGPR) <
564 TRI->getHWRegIndex(FuncInfo->getSGPRForEXECCopy()))
565 FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
566 } else {
567 // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
568 // spills/copies. Reset the SGPR reserved for EXEC copy.
569 FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);
570 }
571
572 SaveBlocks.clear();
573 RestoreBlocks.clear();
574
575 return MadeChange;
576}
577
578PreservedAnalyses
581 MFPropsModifier _(*this, MF);
582 auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(MF);
583 auto *Indexes = MFAM.getCachedResult<SlotIndexesAnalysis>(MF);
585 SILowerSGPRSpills(LIS, Indexes, MDT).run(MF);
586 return PreservedAnalyses::all();
587}
unsigned const MachineRegisterInfo * MRI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
aarch64 promote const
Provides AMDGPU specific target descriptions.
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
AMD GCN specific subclass of TargetSubtarget.
#define DEBUG_TYPE
const HexagonInstrInfo * TII
#define _
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
static void insertCSRRestores(MachineBasicBlock &RestoreBlock, std::vector< CalleeSavedInfo > &CSI)
Insert restore code for the callee-saved registers used in the function.
SmallVector< MachineBasicBlock *, 4 > MBBVector
static void insertCSRSaves(MachineBasicBlock &SaveBlock, ArrayRef< CalleeSavedInfo > CSI)
Insert spill code for the callee-saved registers used in the function.
static void updateLiveness(MachineFunction &MF)
Helper function to update the liveness information for the callee-saved registers.
This file declares the machine register scavenger class.
static bool isLiveIntoMBB(MCRegister Reg, MachineBasicBlock &MBB, const TargetRegisterInfo *TRI)
static void insertCSRRestores(MachineBasicBlock &RestoreBlock, MutableArrayRef< CalleeSavedInfo > CSI, SlotIndexes *Indexes, LiveIntervals *LIS)
Insert restore code for the callee-saved registers used in the function.
static void insertCSRSaves(MachineBasicBlock &SaveBlock, ArrayRef< CalleeSavedInfo > CSI, SlotIndexes *Indexes, LiveIntervals *LIS)
Insert spill code for the callee-saved registers used in the function.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
bool test(unsigned Idx) const
Definition BitVector.h:480
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:167
iterator end()
Definition DenseMap.h:81
NodeT * findNearestCommonDominator(NodeT *A, NodeT *B) const
Find nearest common dominator basic block for basic block A and B.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:359
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
void removeAllRegUnitsForPhysReg(MCRegister Reg)
Remove associated live ranges for the register units associated with Reg.
SlotIndex InsertMachineInstrInMaps(MachineInstr &MI)
LiveInterval & createAndComputeVirtRegInterval(Register Reg)
MCRegAliasIterator enumerates all registers aliasing Reg.
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:33
An RAII based helper class to modify MachineFunctionProperties when running pass.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
bool isReturnBlock() const
Convenience function that returns true if the block ends in a return instruction.
LLVM_ABI void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
LLVM_ABI DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
Analysis pass which computes a MachineDominatorTree.
LLVM_ABI Result run(MachineFunction &MF, MachineFunctionAnalysisManager &)
Analysis pass which computes a MachineDominatorTree.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
bool dominates(const MachineInstr *A, const MachineInstr *B) const
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setCalleeSavedInfoValid(bool v)
int getObjectIndexEnd() const
Return one past the maximum frame object index.
bool hasStackObjects() const
Return true if there are any stack objects in this function.
uint8_t getStackID(int ObjectIdx) const
const SaveRestorePoints & getRestorePoints() const
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
const SaveRestorePoints & getSavePoints() const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Properties which a MachineFunction may have at a given point in time.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
MachineInstrSpan provides an interface to get an iteration range containing the instruction it was in...
MachineBasicBlock::iterator begin()
Representation of each machine instruction.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition ArrayRef.h:303
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
Wrapper class representing virtual and physical registers.
Definition Register.h:19
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
void setFlag(Register Reg, uint8_t Flag)
ArrayRef< SIRegisterInfo::SpilledReg > getSGPRSpillToVirtualVGPRLanes(int FrameIndex) const
bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, bool SpillToPhysVGPRLane=false, bool IsPrologEpilog=false)
bool removeDeadFrameIndices(MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs)
If ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill to the default stack.
void updateNonWWMRegMask(BitVector &RegMask)
ArrayRef< Register > getSGPRSpillVGPRs() const
SlotIndexes pass.
SlotIndex insertMachineInstrInMaps(MachineInstr &MI, bool Late=false)
Insert the given machine instruction into the mapping.
LLVM_ABI void removeMachineInstrFromMaps(MachineInstr &MI, bool AllowBundled=false)
Removes machine instruction (bundle) MI from the mapping.
LLVM_ABI void repairIndexesInRange(MachineBasicBlock *MBB, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End)
Repair indexes after adding and removing instructions.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Information about stack frame layout on the target.
void restoreCalleeSavedRegister(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const CalleeSavedInfo &CS, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const
virtual bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
virtual bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:634
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
char & SILowerSGPRSpillsLegacyID
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1877