35#define DEBUG_TYPE "fixup-statepoint-caller-saved"
36STATISTIC(NumSpilledRegisters,
"Number of spilled register");
37STATISTIC(NumSpillSlotsAllocated,
"Number of spill slots allocated");
38STATISTIC(NumSpillSlotsExtended,
"Number of spill slots extended");
42 cl::desc(
"Allow spill in spill slot of greater size than register size"),
47 cl::desc(
"Allow passing GC Pointer arguments in callee saved registers"));
51 cl::desc(
"Enable simple copy propagation during register reloading"));
57 cl::desc(
"Max number of statepoints allowed to pass GC Ptrs in registers"));
75 return "Fixup Statepoint Caller Saved";
83char FixupStatepointCallerSaved::ID = 0;
87 "Fixup Statepoint Caller Saved",
false,
false)
94 return TRI.getSpillSize(*RC);
115 int Idx = RI->findRegisterUseOperandIdx(Reg, &
TRI,
false);
127 for (
auto It = ++(RI.
getReverse()); It != E; ++It) {
128 if (It->readsRegister(Reg, &
TRI) && !
Use)
130 if (It->modifiesRegister(Reg, &
TRI)) {
139 auto DestSrc =
TII.isCopyInstr(*Def);
140 if (!DestSrc || DestSrc->Destination->getReg() != Reg)
143 Register SrcReg = DestSrc->Source->getReg();
154 IsKill = DestSrc->Source->isKill();
159 LLVM_DEBUG(
dbgs() <<
"spillRegisters: removing dead copy " << *Def);
160 Def->eraseFromParent();
172using RegSlotPair = std::pair<Register, int>;
175class RegReloadCache {
180 RegReloadCache() =
default;
184 RegSlotPair RSP(Reg, FI);
187 assert(Res.second &&
"reload already exists");
192 RegSlotPair RSP(Reg, FI);
202class FrameIndexesCache {
204 struct FrameIndexesPerSize {
228 FrameIndexesPerSize &getCacheBucket(
unsigned Size) {
241 for (
auto &It : Cache)
244 ReservedSlots.
clear();
245 if (EHPad && GlobalIndices.
count(EHPad))
246 for (
auto &RSP : GlobalIndices[EHPad])
247 ReservedSlots.
insert(RSP.second);
253 auto It = GlobalIndices.
find(EHPad);
254 if (It != GlobalIndices.
end()) {
255 auto &Vec = It->second;
257 Vec, [Reg](RegSlotPair &RSP) {
return Reg == RSP.first; });
258 if (
Idx != Vec.end()) {
259 int FI =
Idx->second;
263 assert(ReservedSlots.
count(FI) &&
"using unreserved slot");
269 FrameIndexesPerSize &
Line = getCacheBucket(
Size);
270 while (
Line.Index <
Line.Slots.size()) {
272 if (ReservedSlots.
count(FI))
279 NumSpillSlotsExtended++;
284 NumSpillSlotsAllocated++;
285 Line.Slots.push_back(FI);
290 GlobalIndices[EHPad].push_back(std::make_pair(Reg, FI));
292 <<
printReg(Reg, &TRI) <<
" at landing pad "
312class StatepointState {
325 FrameIndexesCache &CacheFI;
326 bool AllowGCPtrInCSR;
338 FrameIndexesCache &CacheFI,
bool AllowGCPtrInCSR)
339 :
MI(
MI), MF(*
MI.getMF()),
TRI(*MF.getSubtarget().getRegisterInfo()),
340 TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()),
341 Mask(
Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {
349 return I.getOpcode() == TargetOpcode::STATEPOINT;
367 bool isCalleeSaved(
Register Reg) {
return (Mask[Reg / 32] >> Reg % 32) & 1; }
372 bool findRegistersToSpill() {
376 for (
const auto &Def :
MI.defs())
381 EndIdx =
MI.getNumOperands();
389 assert(
Reg.isPhysical() &&
"Only physical regs are expected");
391 if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !GCRegs.
contains(Reg)))
397 if (VisitedRegs.
insert(Reg).second)
401 CacheFI.sortRegisters(RegsToSpill);
402 return !RegsToSpill.
empty();
407 void spillRegisters() {
409 int FI = CacheFI.getFrameIndex(Reg, EHPad);
411 NumSpilledRegisters++;
412 RegToSlotIdx[
Reg] = FI;
432 int FI = RegToSlotIdx[
Reg];
453 void insertReloads(
MachineInstr *NewStatepoint, RegReloadCache &RC) {
455 auto InsertPoint = std::next(NewStatepoint->
getIterator());
457 for (
auto Reg : RegsToReload) {
458 insertReloadBefore(Reg, InsertPoint,
MBB);
460 << RegToSlotIdx[Reg] <<
" after statepoint\n");
462 if (EHPad && !RC.hasReload(Reg, RegToSlotIdx[Reg], EHPad)) {
463 RC.recordReload(Reg, RegToSlotIdx[Reg], EHPad);
464 auto EHPadInsertPoint =
466 insertReloadBefore(Reg, EHPadInsertPoint, EHPad);
480 unsigned NumOps =
MI.getNumOperands();
484 unsigned NumDefs =
MI.getNumDefs();
485 for (
unsigned I = 0;
I < NumDefs; ++
I) {
492 if (
MI.getOperand(
MI.findTiedOperandIdx(
I)).isUndef()) {
493 if (AllowGCPtrInCSR) {
499 if (!AllowGCPtrInCSR) {
501 RegsToReload.push_back(Reg);
503 if (isCalleeSaved(Reg)) {
508 RegsToReload.push_back(Reg);
515 unsigned CurOpIdx = 0;
517 for (
unsigned I = NumDefs;
I <
MI.getNumOperands(); ++
I) {
519 if (
I == OpsToSpill[CurOpIdx]) {
520 int FI = RegToSlotIdx[MO.
getReg()];
521 MIB.addImm(StackMaps::IndirectMemRefOp);
525 MIB.addFrameIndex(FI);
531 if (AllowGCPtrInCSR &&
MI.isRegTiedToDefOperand(
I, &OldDef)) {
533 assert(NewIndices[OldDef] < NumOps);
534 MIB->tieOperands(NewIndices[OldDef], MIB->getNumOperands() - 1);
538 assert(CurOpIdx == (OpsToSpill.
size() - 1) &&
"Not all operands processed");
541 for (
auto It : RegToSlotIdx) {
555 MI.getParent()->insert(MI, NewMI);
557 LLVM_DEBUG(
dbgs() <<
"rewritten statepoint to : " << *NewMI <<
"\n");
558 MI.eraseFromParent();
563class StatepointProcessor {
567 FrameIndexesCache CacheFI;
568 RegReloadCache ReloadCache;
572 : MF(MF),
TRI(*MF.getSubtarget().getRegisterInfo()),
573 CacheFI(MF.getFrameInfo(),
TRI) {}
579 if (Flags & (
uint64_t)StatepointFlags::DeoptLiveIn)
582 <<
MI.getParent()->getName() <<
" : process statepoint "
586 StatepointState
SS(
MI, Mask, CacheFI, AllowGCPtrInCSR);
587 CacheFI.reset(
SS.getEHPad());
589 if (!
SS.findRegistersToSpill())
593 auto *NewStatepoint =
SS.rewriteStatepoint();
594 SS.insertReloads(NewStatepoint, ReloadCache);
600bool FixupStatepointCallerSaved::runOnMachineFunction(
MachineFunction &MF) {
611 if (
I.getOpcode() == TargetOpcode::STATEPOINT)
614 if (Statepoints.
empty())
617 bool Changed =
false;
618 StatepointProcessor SPP(MF);
619 unsigned NumStatepoints = 0;
625 AllowGCPtrInCSR =
false;
626 Changed |= SPP.process(*
I, AllowGCPtrInCSR);
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static Register performCopyPropagation(Register Reg, MachineBasicBlock::iterator &RI, bool &IsKill, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI)
static cl::opt< bool > PassGCPtrInCSR("fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false), cl::desc("Allow passing GC Pointer arguments in callee saved registers"))
static cl::opt< unsigned > MaxStatepointsWithRegs("fixup-max-csr-statepoints", cl::Hidden, cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"))
Fixup Statepoint Caller static false unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg)
static cl::opt< bool > FixupSCSExtendSlotSize("fixup-scs-extend-slot-size", cl::Hidden, cl::init(false), cl::desc("Allow spill in spill slot of greater size than register size"), cl::Hidden)
Fixup Statepoint Caller Saved
static cl::opt< bool > EnableCopyProp("fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true), cl::desc("Enable simple copy propagation during register reloading"))
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
PowerPC TLS Dynamic Call Fixup
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Represent the analysis usage information of a pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
iterator find(const_arg_type_t< KeyT > Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
iterator SkipPHIsLabelsAndDebug(iterator I, Register Reg=Register(), bool SkipPseudoOp=true)
Return the first instruction in MBB after I that is not a PHI, label or debug.
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
iterator_range< succ_iterator > successors()
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
void setObjectSize(int ObjectIdx, int64_t Size)
Change the size of the specified stack object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImplicit=false)
CreateMachineInstr - Allocate a new MachineInstr.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Function & getFunction()
Return the LLVM function that this machine code represents.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
instr_iterator getInstrIterator() const
Representation of each machine instruction.
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
void setMemRefs(MachineFunction &MF, ArrayRef< MachineMemOperand * > MemRefs)
Assign this MachineInstr's memory reference descriptor list.
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
Flags
Flags values. These may be or'd together.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
bool contains(const T &V) const
Check if the SmallSet contains the given element.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level Statepoint operands.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A Use represents the edge between a Value definition and its users.
self_iterator getIterator()
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Define
Register definition.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
NodeAddr< DefNode * > Def
This is an optimization pass for GlobalISel generic memory operations.
char & FixupStatepointCallerSavedID
The pass fixups statepoint machine instruction to replace usage of caller saved registers with stack ...
void initializeFixupStatepointCallerSavedPass(PassRegistry &)
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.