Go to the documentation of this file.
35 #define DEBUG_TYPE "fixup-statepoint-caller-saved"
36 STATISTIC(NumSpilledRegisters,
"Number of spilled register");
37 STATISTIC(NumSpillSlotsAllocated,
"Number of spill slots allocated");
38 STATISTIC(NumSpillSlotsExtended,
"Number of spill slots extended");
42 cl::desc(
"Allow spill in spill slot of greater size than register size"),
47 cl::desc(
"Allow passing GC Pointer arguments in callee saved registers"));
51 cl::desc(
"Enable simple copy propagation during register reloading"));
57 cl::desc(
"Max number of statepoints allowed to pass GC Ptrs in registers"));
75 return "Fixup Statepoint Caller Saved";
87 "Fixup Statepoint Caller Saved",
false,
false)
115 int Idx = RI->findRegisterUseOperandIdx(
Reg,
false, &
TRI);
116 if (Idx >= 0 && (
unsigned)Idx <
StatepointOpers(&*RI).getNumDeoptArgsIdx()) {
127 for (
auto It = ++(RI.
getReverse()); It !=
E; ++It) {
128 if (It->readsRegister(
Reg, &
TRI) && !
Use)
130 if (It->modifiesRegister(
Reg, &
TRI)) {
139 auto DestSrc =
TII.isCopyInstr(*
Def);
140 if (!DestSrc || DestSrc->Destination->getReg() !=
Reg)
143 Register SrcReg = DestSrc->Source->getReg();
154 IsKill = DestSrc->Source->isKill();
160 Def->eraseFromParent();
167 using RegSlotPair = std::pair<Register, int>;
170 class RegReloadCache {
175 RegReloadCache() =
default;
179 RegSlotPair RSP(
Reg, FI);
182 assert(Res.second &&
"reload already exists");
187 RegSlotPair RSP(
Reg, FI);
197 class FrameIndexesCache {
199 struct FrameIndexesPerSize {
223 FrameIndexesPerSize &getCacheBucket(
unsigned Size) {
236 for (
auto &It : Cache)
239 ReservedSlots.
clear();
240 if (EHPad && GlobalIndices.
count(EHPad))
241 for (
auto &RSP : GlobalIndices[EHPad])
242 ReservedSlots.
insert(RSP.second);
248 auto It = GlobalIndices.
find(EHPad);
249 if (It != GlobalIndices.
end()) {
250 auto &Vec = It->second;
252 Vec, [
Reg](RegSlotPair &RSP) {
return Reg == RSP.first; });
253 if (Idx != Vec.end()) {
254 int FI = Idx->second;
258 assert(ReservedSlots.
count(FI) &&
"using unreserved slot");
264 FrameIndexesPerSize &Line = getCacheBucket(Size);
265 while (Line.Index < Line.Slots.size()) {
266 int FI = Line.Slots[Line.Index++];
267 if (ReservedSlots.
count(FI))
274 NumSpillSlotsExtended++;
279 NumSpillSlotsAllocated++;
280 Line.Slots.push_back(FI);
285 GlobalIndices[EHPad].push_back(std::make_pair(
Reg, FI));
307 class StatepointState {
320 FrameIndexesCache &CacheFI;
321 bool AllowGCPtrInCSR;
333 FrameIndexesCache &CacheFI,
bool AllowGCPtrInCSR)
334 :
MI(
MI), MF(*
MI.getMF()),
TRI(*MF.getSubtarget().getRegisterInfo()),
335 TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()),
336 Mask(
Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {
344 return I.getOpcode() == TargetOpcode::STATEPOINT;
367 bool findRegistersToSpill() {
371 for (
const auto &
Def :
MI.defs())
376 EndIdx =
MI.getNumOperands();
377 Idx < EndIdx; ++Idx) {
384 assert(
Reg.isPhysical() &&
"Only physical regs are expected");
393 RegsToSpill.push_back(
Reg);
394 OpsToSpill.push_back(Idx);
396 CacheFI.sortRegisters(RegsToSpill);
397 return !RegsToSpill.empty();
402 void spillRegisters() {
404 int FI = CacheFI.getFrameIndex(
Reg, EHPad);
407 NumSpilledRegisters++;
408 RegToSlotIdx[
Reg] = FI;
427 int FI = RegToSlotIdx[
Reg];
448 void insertReloads(
MachineInstr *NewStatepoint, RegReloadCache &RC) {
450 auto InsertPoint = std::next(NewStatepoint->
getIterator());
452 for (
auto Reg : RegsToReload) {
453 insertReloadBefore(
Reg, InsertPoint,
MBB);
455 << RegToSlotIdx[
Reg] <<
" after statepoint\n");
457 if (EHPad && !RC.hasReload(
Reg, RegToSlotIdx[
Reg], EHPad)) {
458 RC.recordReload(
Reg, RegToSlotIdx[
Reg], EHPad);
460 insertReloadBefore(
Reg, EHPadInsertPoint, EHPad);
474 unsigned NumOps =
MI.getNumOperands();
478 unsigned NumDefs =
MI.getNumDefs();
479 for (
unsigned I = 0;
I < NumDefs; ++
I) {
486 if (
MI.getOperand(
MI.findTiedOperandIdx(
I)).isUndef()) {
487 if (AllowGCPtrInCSR) {
493 if (!AllowGCPtrInCSR) {
495 RegsToReload.push_back(
Reg);
497 if (isCalleeSaved(
Reg)) {
501 NewIndices.push_back(NumOps);
502 RegsToReload.push_back(
Reg);
508 OpsToSpill.push_back(
MI.getNumOperands());
509 unsigned CurOpIdx = 0;
511 for (
unsigned I = NumDefs;
I <
MI.getNumOperands(); ++
I) {
513 if (
I == OpsToSpill[CurOpIdx]) {
514 int FI = RegToSlotIdx[MO.
getReg()];
515 MIB.addImm(StackMaps::IndirectMemRefOp);
519 MIB.addFrameIndex(FI);
525 if (AllowGCPtrInCSR &&
MI.isRegTiedToDefOperand(
I, &OldDef)) {
527 assert(NewIndices[OldDef] < NumOps);
528 MIB->tieOperands(NewIndices[OldDef], MIB->getNumOperands() - 1);
532 assert(CurOpIdx == (OpsToSpill.size() - 1) &&
"Not all operands processed");
535 for (
auto It : RegToSlotIdx) {
549 MI.getParent()->insert(
MI, NewMI);
551 LLVM_DEBUG(
dbgs() <<
"rewritten statepoint to : " << *NewMI <<
"\n");
552 MI.eraseFromParent();
557 class StatepointProcessor {
561 FrameIndexesCache CacheFI;
562 RegReloadCache ReloadCache;
566 : MF(MF),
TRI(*MF.getSubtarget().getRegisterInfo()),
567 CacheFI(MF.getFrameInfo(),
TRI) {}
576 <<
MI.getParent()->getName() <<
" : process statepoint "
580 StatepointState
SS(
MI,
Mask, CacheFI, AllowGCPtrInCSR);
581 CacheFI.reset(
SS.getEHPad());
583 if (!
SS.findRegistersToSpill())
587 auto *NewStatepoint =
SS.rewriteStatepoint();
588 SS.insertReloads(NewStatepoint, ReloadCache);
594 bool FixupStatepointCallerSaved::runOnMachineFunction(
MachineFunction &MF) {
605 if (
I.getOpcode() == TargetOpcode::STATEPOINT)
606 Statepoints.push_back(&
I);
608 if (Statepoints.empty())
611 bool Changed =
false;
612 StatepointProcessor SPP(MF);
613 unsigned NumStatepoints = 0;
619 AllowGCPtrInCSR =
false;
620 Changed |= SPP.process(*
I, AllowGCPtrInCSR);
This is an optimization pass for GlobalISel generic memory operations.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
static cl::opt< bool > PassGCPtrInCSR("fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false), cl::desc("Allow passing GC Pointer arguments in callee saved registers"))
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Reg
All possible values of the reg field in the ModR/M byte.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Fixup Statepoint Caller Saved
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImplicit=false)
CreateMachineInstr - Allocate a new MachineInstr.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
void setObjectSize(int ObjectIdx, int64_t Size)
Change the size of the specified stack object.
unsigned const TargetRegisterInfo * TRI
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
INITIALIZE_PASS_BEGIN(FixupStatepointCallerSaved, DEBUG_TYPE, "Fixup Statepoint Caller Saved", false, false) INITIALIZE_PASS_END(FixupStatepointCallerSaved
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
TargetInstrInfo - Interface to description of machine instruction set.
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
void setMemRefs(MachineFunction &MF, ArrayRef< MachineMemOperand * > MemRefs)
Assign this MachineInstr's memory reference descriptor list.
static Register performCopyPropagation(Register Reg, MachineBasicBlock::iterator &RI, bool &IsKill, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI)
Represent the analysis usage information of a pass.
const HexagonInstrInfo * TII
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
MachineOperand class - Representation of each machine instruction operand.
static cl::opt< bool > FixupSCSExtendSlotSize("fixup-scs-extend-slot-size", cl::Hidden, cl::init(false), cl::desc("Allow spill in spill slot of greater size than register size"), cl::Hidden)
STATISTIC(NumFunctions, "Total number of functions")
static cl::opt< bool > EnableCopyProp("fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true), cl::desc("Enable simple copy propagation during register reloading"))
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
instr_iterator getInstrIterator() const
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Representation of each machine instruction.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Fixup Statepoint Caller static false unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg)
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const override
Store the specified register of the given register class to the specified stack frame index.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
initializer< Ty > init(const Ty &Val)
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
iterator find(const_arg_type_t< KeyT > Val)
Flags
Flags values. These may be or'd together.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Register getReg() const
getReg - Returns the register number.
static cl::opt< unsigned > MaxStatepointsWithRegs("fixup-max-csr-statepoints", cl::Hidden, cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"))
MachineInstrBundleIterator< MachineInstr > iterator
iterator_range< succ_iterator > successors()
StringRef - Represent a constant reference to a string, i.e.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const override
Load the specified register of the given register class from the specified stack frame index.
self_iterator getIterator()
unsigned isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
const MachineBasicBlock * getParent() const
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
@ MOLoad
The memory access reads data.
Wrapper class representing virtual and physical registers.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
char & FixupStatepointCallerSavedID
The pass fixups statepoint machine instruction to replace usage of caller saved registers with stack ...
MI-level Statepoint operands.
Function & getFunction()
Return the LLVM function that this machine code represents.
@ Define
Register definition.
void sort(IteratorTy Start, IteratorTy End)
@ MOStore
The memory access writes data.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
unsigned getNumOperands() const
Retuns the total number of operands.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
@ DeoptLiveIn
Mark the deopt arguments associated with the statepoint as only being "live-in".
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
iterator SkipPHIsLabelsAndDebug(iterator I, bool SkipPseudoOp=true)
Return the first instruction in MBB after I that is not a PHI, label or debug.
const TargetRegisterClass * getMinimalPhysRegClass(MCRegister Reg, MVT VT=MVT::Other) const
Returns the Register Class of a physical register of the given type, picking the most sub register cl...
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
void initializeFixupStatepointCallerSavedPass(PassRegistry &)
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function.
A Use represents the edge between a Value definition and its users.