31#define DEBUG_TYPE "si-lower-sgpr-spills"
38 "amdgpu-num-vgprs-for-wwm-alloc",
39 cl::desc(
"Max num VGPRs for whole-wave register allocation."),
42class SILowerSGPRSpills {
58 : LIS(LIS), Indexes(Indexes), MDT(MDT) {}
63 void updateLaneVGPRDomInstr(
91char SILowerSGPRSpillsLegacy::ID = 0;
94 "SI lower SGPR spill instructions",
false,
false)
106 if (
MBB.isLiveIn(*R)) {
132 Reg,
Reg == RI->getReturnAddressReg(MF) ? MVT::i64 : MVT::i32);
139 TII.storeRegToStackSlot(SaveBlock,
I,
Reg, !IsLiveIn, CS.getFrameIdx(),
174 I == RestoreBlock.
begin() ?
I : std::prev(
I);
205void SILowerSGPRSpills::calculateSaveRestoreBlocks(
MachineFunction &MF) {
215 "Multiple save points not yet supported!");
219 "Multiple restore points not yet supported!");
221 MachineBasicBlock *RestoreBlock = RestorePoint.first;
232 for (MachineBasicBlock &
MBB : MF) {
250bool SILowerSGPRSpills::spillCalleeSavedRegs(
251 MachineFunction &MF, SmallVectorImpl<int> &CalleeSavedFIs) {
255 const SIFrameLowering *TFI =
ST.getFrameLowering();
257 RegScavenger *RS =
nullptr;
261 TFI->determineCalleeSavesSGPR(MF, SavedRegs, RS);
264 if (!
F.hasFnAttribute(Attribute::Naked)) {
269 std::vector<CalleeSavedInfo> CSI;
272 for (
unsigned I = 0; CSRegs[
I]; ++
I) {
273 MCRegister
Reg = CSRegs[
I];
276 const TargetRegisterClass *RC =
277 TRI->getMinimalPhysRegClass(
Reg, MVT::i32);
279 TRI->getSpillAlign(*RC),
true);
281 CSI.emplace_back(
Reg, JunkFI);
287 for (MachineBasicBlock *SaveBlock : SaveBlocks)
291 assert(SaveBlocks.size() == 1 &&
"shrink wrapping not fully implemented");
294 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
303void SILowerSGPRSpills::updateLaneVGPRDomInstr(
305 DenseMap<Register, MachineBasicBlock::iterator> &LaneVGPRDomInstr) {
312 SIMachineFunctionInfo *FuncInfo =
317 for (
auto &Spill : VGPRSpills) {
318 if (PrevLaneVGPR ==
Spill.VGPR)
321 PrevLaneVGPR =
Spill.VGPR;
323 if (
Spill.Lane == 0 &&
I == LaneVGPRDomInstr.
end()) {
325 LaneVGPRDomInstr[
Spill.VGPR] = InsertPt;
328 auto PrevInsertPt =
I->second;
329 MachineBasicBlock *DomMBB = PrevInsertPt->getParent();
335 if (MDT->
dominates(&*InsertPt, &*PrevInsertPt))
336 I->second = InsertPt;
345 I->second = InsertPt;
346 else if (DomMBB != PrevInsertPt->getParent())
352void SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF,
353 BitVector &RegMask) {
356 SIMachineFunctionInfo *MFI = MF.
getInfo<SIMachineFunctionInfo>();
358 BitVector ReservedRegs =
TRI->getReservedRegs(MF);
359 BitVector NonWwmAllocMask(
TRI->getNumRegs());
365 unsigned NumRegs = MaxNumVGPRsForWwmAllocation;
369 auto [MaxNumVGPRs, MaxNumAGPRs] =
ST.getMaxNumVectorRegs(MF.
getFunction());
373 for (
unsigned Reg = AMDGPU::VGPR0 + MaxNumVGPRs - 1;
374 (
I < NumRegs) && (
Reg >= AMDGPU::VGPR0); --
Reg) {
376 !
MRI.isPhysRegUsed(
Reg,
true)) {
377 TRI->markSuperRegs(RegMask,
Reg);
384 TRI->markSuperRegs(RegMask, AMDGPU::VGPR0);
386 "cannot find enough VGPRs for wwm-regalloc");
390bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) {
391 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
392 LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() :
nullptr;
393 auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();
394 SlotIndexes *Indexes = SIWrapper ? &SIWrapper->getSI() :
nullptr;
395 MachineDominatorTree *MDT =
396 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
397 return SILowerSGPRSpills(LIS, Indexes, MDT).run(MF);
400bool SILowerSGPRSpills::run(MachineFunction &MF) {
402 TII =
ST.getInstrInfo();
403 TRI = &
TII->getRegisterInfo();
409 calculateSaveRestoreBlocks(MF);
410 SmallVector<int> CalleeSavedFIs;
411 bool HasCSRs = spillCalleeSavedRegs(MF, CalleeSavedFIs);
415 SIMachineFunctionInfo *FuncInfo = MF.
getInfo<SIMachineFunctionInfo>();
419 RestoreBlocks.
clear();
423 bool MadeChange =
false;
424 bool SpilledToVirtVGPRLanes =
false;
428 const bool HasSGPRSpillToVGPR =
TRI->spillSGPRToVGPR() &&
430 if (HasSGPRSpillToVGPR) {
441 DenseMap<Register, MachineBasicBlock::iterator> LaneVGPRDomInstr;
443 for (MachineBasicBlock &
MBB : MF) {
445 if (!
TII->isSGPRSpill(
MI))
448 if (
MI.getOperand(0).isUndef()) {
451 MI.eraseFromParent();
455 int FI =
TII->getNamedOperand(
MI, AMDGPU::OpName::addr)->getIndex();
459 if (IsCalleeSaveSGPRSpill) {
472 bool Spilled =
TRI->eliminateSGPRToVGPRSpillFrameIndex(
473 MI, FI,
nullptr, Indexes, LIS,
true);
476 "failed to spill SGPR to physical VGPR lane when allocated");
479 MachineInstrSpan MIS(&
MI, &
MBB);
481 bool Spilled =
TRI->eliminateSGPRToVGPRSpillFrameIndex(
482 MI, FI,
nullptr, Indexes, LIS);
485 "failed to spill SGPR to virtual VGPR lane when allocated");
487 updateLaneVGPRDomInstr(FI, &
MBB, MIS.
begin(), LaneVGPRDomInstr);
488 SpilledToVirtVGPRLanes =
true;
495 auto InsertPt = LaneVGPRDomInstr[
Reg];
497 MachineBasicBlock &
Block = *InsertPt->getParent();
516 BitVector WwmRegMask(
TRI->getNumRegs());
518 determineRegsForWWMAllocation(MF, WwmRegMask);
520 BitVector NonWwmRegMask(WwmRegMask);
521 NonWwmRegMask.flip().clearBitsNotInMask(
TRI->getAllVGPRRegMask());
528 for (MachineBasicBlock &
MBB : MF) {
533 for (MachineInstr &
MI :
MBB) {
534 if (
MI.isDebugValue()) {
535 uint32_t StackOperandIdx =
MI.isDebugValueList() ? 2 : 0;
536 if (
MI.getOperand(StackOperandIdx).isFI() &&
538 MI.getOperand(StackOperandIdx).getIndex()) &&
539 SpillFIs[
MI.getOperand(StackOperandIdx).getIndex()]) {
540 MI.getOperand(StackOperandIdx)
541 .ChangeToRegister(
Register(),
false );
557 if (SpilledToVirtVGPRLanes) {
558 const TargetRegisterClass *RC =
TRI->getWaveMaskRegClass();
563 if (UnusedLowSGPR &&
TRI->getHWRegIndex(UnusedLowSGPR) <
573 RestoreBlocks.
clear();
585 SILowerSGPRSpills(LIS, Indexes, MDT).
run(MF);
unsigned const MachineRegisterInfo * MRI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Provides AMDGPU specific target descriptions.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
Register const TargetRegisterInfo * TRI
Promote Memory to Register
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static void insertCSRRestores(MachineBasicBlock &RestoreBlock, std::vector< CalleeSavedInfo > &CSI)
Insert restore code for the callee-saved registers used in the function.
SmallVector< MachineBasicBlock *, 4 > MBBVector
static void insertCSRSaves(MachineBasicBlock &SaveBlock, ArrayRef< CalleeSavedInfo > CSI)
Insert spill code for the callee-saved registers used in the function.
static void updateLiveness(MachineFunction &MF)
Helper function to update the liveness information for the callee-saved registers.
This file declares the machine register scavenger class.
static bool isLiveIntoMBB(MCRegister Reg, MachineBasicBlock &MBB, const TargetRegisterInfo *TRI)
static void insertCSRRestores(MachineBasicBlock &RestoreBlock, MutableArrayRef< CalleeSavedInfo > CSI, SlotIndexes *Indexes, LiveIntervals *LIS)
Insert restore code for the callee-saved registers used in the function.
static void insertCSRSaves(MachineBasicBlock &SaveBlock, ArrayRef< CalleeSavedInfo > CSI, SlotIndexes *Indexes, LiveIntervals *LIS)
Insert spill code for the callee-saved registers used in the function.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool test(unsigned Idx) const
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
iterator find(const_arg_type_t< KeyT > Val)
NodeT * findNearestCommonDominator(NodeT *A, NodeT *B) const
Find nearest common dominator basic block for basic block A and B.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
void removeAllRegUnitsForPhysReg(MCRegister Reg)
Remove associated live ranges for the register units associated with Reg.
SlotIndex InsertMachineInstrInMaps(MachineInstr &MI)
LiveInterval & createAndComputeVirtRegInterval(Register Reg)
MCRegAliasIterator enumerates all registers aliasing Reg.
Wrapper class representing physical registers. Should be passed by value.
An RAII based helper class to modify MachineFunctionProperties when running pass.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
bool isReturnBlock() const
Convenience function that returns true if the block ends in a return instruction.
LLVM_ABI void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
LLVM_ABI DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
Analysis pass which computes a MachineDominatorTree.
LLVM_ABI Result run(MachineFunction &MF, MachineFunctionAnalysisManager &)
Analysis pass which computes a MachineDominatorTree.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
bool dominates(const MachineInstr *A, const MachineInstr *B) const
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setCalleeSavedInfoValid(bool v)
int getObjectIndexEnd() const
Return one past the maximum frame object index.
bool hasStackObjects() const
Return true if there are any stack objects in this function.
uint8_t getStackID(int ObjectIdx) const
const SaveRestorePoints & getRestorePoints() const
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
const SaveRestorePoints & getSavePoints() const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Properties which a MachineFunction may have at a given point in time.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
MachineInstrSpan provides an interface to get an iteration range containing the instruction it was in...
MachineBasicBlock::iterator begin()
Representation of each machine instruction.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Wrapper class representing virtual and physical registers.
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
void setSGPRForEXECCopy(Register Reg)
void setFlag(Register Reg, uint8_t Flag)
ArrayRef< SIRegisterInfo::SpilledReg > getSGPRSpillToVirtualVGPRLanes(int FrameIndex) const
Register getSGPRForEXECCopy() const
bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, bool SpillToPhysVGPRLane=false, bool IsPrologEpilog=false)
bool removeDeadFrameIndices(MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs)
If ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill to the default stack.
void updateNonWWMRegMask(BitVector &RegMask)
bool hasSpilledSGPRs() const
ArrayRef< Register > getSGPRSpillVGPRs() const
SlotIndex insertMachineInstrInMaps(MachineInstr &MI, bool Late=false)
Insert the given machine instruction into the mapping.
LLVM_ABI void removeMachineInstrFromMaps(MachineInstr &MI, bool AllowBundled=false)
Removes machine instruction (bundle) MI from the mapping.
LLVM_ABI void repairIndexesInRange(MachineBasicBlock *MBB, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End)
Repair indexes after adding and removing instructions.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Information about stack frame layout on the target.
void restoreCalleeSavedRegister(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const CalleeSavedInfo &CS, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const
virtual bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
virtual bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
auto reverse(ContainerTy &&C)
char & SILowerSGPRSpillsLegacyID
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.