LLVM  13.0.0git
FixupStatepointCallerSaved.cpp
Go to the documentation of this file.
1 //===-- FixupStatepointCallerSaved.cpp - Fixup caller saved registers ----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// Statepoint instruction in deopt parameters contains values which are
12 /// meaningful to the runtime and should be able to be read at the moment the
13 /// call returns. So we can say that we need to encode the fact that these
14 /// values are "late read" by runtime. If we could express this notion for
15 /// register allocator it would produce the right form for us.
16 /// The need to fixup (i.e this pass) is specifically handling the fact that
17 /// we cannot describe such a late read for the register allocator.
18 /// Register allocator may put the value on a register clobbered by the call.
19 /// This pass forces the spill of such registers and replaces corresponding
20 /// statepoint operands to added spill slots.
21 ///
22 //===----------------------------------------------------------------------===//
23 
24 #include "llvm/ADT/SmallSet.h"
25 #include "llvm/ADT/Statistic.h"
29 #include "llvm/CodeGen/Passes.h"
30 #include "llvm/CodeGen/StackMaps.h"
33 #include "llvm/IR/Statepoint.h"
34 #include "llvm/InitializePasses.h"
35 #include "llvm/Support/Debug.h"
36 
37 using namespace llvm;
38 
39 #define DEBUG_TYPE "fixup-statepoint-caller-saved"
40 STATISTIC(NumSpilledRegisters, "Number of spilled register");
41 STATISTIC(NumSpillSlotsAllocated, "Number of spill slots allocated");
42 STATISTIC(NumSpillSlotsExtended, "Number of spill slots extended");
43 
45  "fixup-scs-extend-slot-size", cl::Hidden, cl::init(false),
46  cl::desc("Allow spill in spill slot of greater size than register size"),
47  cl::Hidden);
48 
50  "fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false),
51  cl::desc("Allow passing GC Pointer arguments in callee saved registers"));
52 
54  "fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true),
55  cl::desc("Enable simple copy propagation during register reloading"));
56 
57 // This is purely debugging option.
58 // It may be handy for investigating statepoint spilling issues.
60  "fixup-max-csr-statepoints", cl::Hidden,
61  cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"));
62 
63 namespace {
64 
65 class FixupStatepointCallerSaved : public MachineFunctionPass {
66 public:
67  static char ID;
68 
69  FixupStatepointCallerSaved() : MachineFunctionPass(ID) {
71  }
72 
73  void getAnalysisUsage(AnalysisUsage &AU) const override {
74  AU.setPreservesCFG();
76  }
77 
78  StringRef getPassName() const override {
79  return "Fixup Statepoint Caller Saved";
80  }
81 
82  bool runOnMachineFunction(MachineFunction &MF) override;
83 };
84 
85 } // End anonymous namespace.
86 
89 
90 INITIALIZE_PASS_BEGIN(FixupStatepointCallerSaved, DEBUG_TYPE,
91  "Fixup Statepoint Caller Saved", false, false)
92 INITIALIZE_PASS_END(FixupStatepointCallerSaved, DEBUG_TYPE,
93  "Fixup Statepoint Caller Saved", false, false)
94 
95 // Utility function to get size of the register.
98  return TRI.getSpillSize(*RC);
99 }
100 
101 // Try to eliminate redundant copy to register which we're going to
102 // spill, i.e. try to change:
103 // X = COPY Y
104 // SPILL X
105 // to
106 // SPILL Y
107 // If there are no uses of X between copy and STATEPOINT, that COPY
108 // may be eliminated.
109 // Reg - register we're about to spill
110 // RI - On entry points to statepoint.
111 // On successful copy propagation set to new spill point.
112 // IsKill - set to true if COPY is Kill (there are no uses of Y)
113 // Returns either found source copy register or original one.
116  bool &IsKill, const TargetInstrInfo &TII,
117  const TargetRegisterInfo &TRI) {
118  // First check if statepoint itself uses Reg in non-meta operands.
119  int Idx = RI->findRegisterUseOperandIdx(Reg, false, &TRI);
120  if (Idx >= 0 && (unsigned)Idx < StatepointOpers(&*RI).getNumDeoptArgsIdx()) {
121  IsKill = false;
122  return Reg;
123  }
124 
125  if (!EnableCopyProp)
126  return Reg;
127 
128  MachineBasicBlock *MBB = RI->getParent();
130  MachineInstr *Def = nullptr, *Use = nullptr;
131  for (auto It = ++(RI.getReverse()); It != E; ++It) {
132  if (It->readsRegister(Reg, &TRI) && !Use)
133  Use = &*It;
134  if (It->modifiesRegister(Reg, &TRI)) {
135  Def = &*It;
136  break;
137  }
138  }
139 
140  if (!Def)
141  return Reg;
142 
143  auto DestSrc = TII.isCopyInstr(*Def);
144  if (!DestSrc || DestSrc->Destination->getReg() != Reg)
145  return Reg;
146 
147  Register SrcReg = DestSrc->Source->getReg();
148 
149  if (getRegisterSize(TRI, Reg) != getRegisterSize(TRI, SrcReg))
150  return Reg;
151 
152  LLVM_DEBUG(dbgs() << "spillRegisters: perform copy propagation "
153  << printReg(Reg, &TRI) << " -> " << printReg(SrcReg, &TRI)
154  << "\n");
155 
156  // Insert spill immediately after Def
158  IsKill = DestSrc->Source->isKill();
159 
160  // There are no uses of original register between COPY and STATEPOINT.
161  // There can't be any after STATEPOINT, so we can eliminate Def.
162  if (!Use) {
163  LLVM_DEBUG(dbgs() << "spillRegisters: removing dead copy " << *Def);
164  Def->eraseFromParent();
165  }
166  return SrcReg;
167 }
168 
169 namespace {
170 // Pair {Register, FrameIndex}
171 using RegSlotPair = std::pair<Register, int>;
172 
173 // Keeps track of what reloads were inserted in MBB.
174 class RegReloadCache {
175  using ReloadSet = SmallSet<RegSlotPair, 8>;
177 
178 public:
179  RegReloadCache() = default;
180 
181  // Record reload of Reg from FI in block MBB
182  void recordReload(Register Reg, int FI, const MachineBasicBlock *MBB) {
183  RegSlotPair RSP(Reg, FI);
184  auto Res = Reloads[MBB].insert(RSP);
185  (void)Res;
186  assert(Res.second && "reload already exists");
187  }
188 
189  // Does basic block MBB contains reload of Reg from FI?
190  bool hasReload(Register Reg, int FI, const MachineBasicBlock *MBB) {
191  RegSlotPair RSP(Reg, FI);
192  return Reloads.count(MBB) && Reloads[MBB].count(RSP);
193  }
194 };
195 
196 // Cache used frame indexes during statepoint re-write to re-use them in
197 // processing next statepoint instruction.
198 // Two strategies. One is to preserve the size of spill slot while another one
199 // extends the size of spill slots to reduce the number of them, causing
200 // the less total frame size. But unspill will have "implicit" any extend.
201 class FrameIndexesCache {
202 private:
203  struct FrameIndexesPerSize {
204  // List of used frame indexes during processing previous statepoints.
205  SmallVector<int, 8> Slots;
206  // Current index of un-used yet frame index.
207  unsigned Index = 0;
208  };
209  MachineFrameInfo &MFI;
210  const TargetRegisterInfo &TRI;
211  // Map size to list of frame indexes of this size. If the mode is
212  // FixupSCSExtendSlotSize then the key 0 is used to keep all frame indexes.
213  // If the size of required spill slot is greater than in a cache then the
214  // size will be increased.
216 
217  // Keeps track of slots reserved for the shared landing pad processing.
218  // Initialized from GlobalIndices for the current EHPad.
219  SmallSet<int, 8> ReservedSlots;
220 
221  // Landing pad can be destination of several statepoints. Every register
222  // defined by such statepoints must be spilled to the same stack slot.
223  // This map keeps that information.
225  GlobalIndices;
226 
227  FrameIndexesPerSize &getCacheBucket(unsigned Size) {
228  // In FixupSCSExtendSlotSize mode the bucket with 0 index is used
229  // for all sizes.
230  return Cache[FixupSCSExtendSlotSize ? 0 : Size];
231  }
232 
233 public:
234  FrameIndexesCache(MachineFrameInfo &MFI, const TargetRegisterInfo &TRI)
235  : MFI(MFI), TRI(TRI) {}
236  // Reset the current state of used frame indexes. After invocation of
237  // this function all frame indexes are available for allocation with
238  // the exception of slots reserved for landing pad processing (if any).
239  void reset(const MachineBasicBlock *EHPad) {
240  for (auto &It : Cache)
241  It.second.Index = 0;
242 
243  ReservedSlots.clear();
244  if (EHPad && GlobalIndices.count(EHPad))
245  for (auto &RSP : GlobalIndices[EHPad])
246  ReservedSlots.insert(RSP.second);
247  }
248 
249  // Get frame index to spill the register.
250  int getFrameIndex(Register Reg, MachineBasicBlock *EHPad) {
251  // Check if slot for Reg is already reserved at EHPad.
252  auto It = GlobalIndices.find(EHPad);
253  if (It != GlobalIndices.end()) {
254  auto &Vec = It->second;
255  auto Idx = llvm::find_if(
256  Vec, [Reg](RegSlotPair &RSP) { return Reg == RSP.first; });
257  if (Idx != Vec.end()) {
258  int FI = Idx->second;
259  LLVM_DEBUG(dbgs() << "Found global FI " << FI << " for register "
260  << printReg(Reg, &TRI) << " at "
261  << printMBBReference(*EHPad) << "\n");
262  assert(ReservedSlots.count(FI) && "using unreserved slot");
263  return FI;
264  }
265  }
266 
267  unsigned Size = getRegisterSize(TRI, Reg);
268  FrameIndexesPerSize &Line = getCacheBucket(Size);
269  while (Line.Index < Line.Slots.size()) {
270  int FI = Line.Slots[Line.Index++];
271  if (ReservedSlots.count(FI))
272  continue;
273  // If all sizes are kept together we probably need to extend the
274  // spill slot size.
275  if (MFI.getObjectSize(FI) < Size) {
276  MFI.setObjectSize(FI, Size);
277  MFI.setObjectAlignment(FI, Align(Size));
278  NumSpillSlotsExtended++;
279  }
280  return FI;
281  }
282  int FI = MFI.CreateSpillStackObject(Size, Align(Size));
283  NumSpillSlotsAllocated++;
284  Line.Slots.push_back(FI);
285  ++Line.Index;
286 
287  // Remember assignment {Reg, FI} for EHPad
288  if (EHPad) {
289  GlobalIndices[EHPad].push_back(std::make_pair(Reg, FI));
290  LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling reg "
291  << printReg(Reg, &TRI) << " at landing pad "
292  << printMBBReference(*EHPad) << "\n");
293  }
294 
295  return FI;
296  }
297 
298  // Sort all registers to spill in descendent order. In the
299  // FixupSCSExtendSlotSize mode it will minimize the total frame size.
300  // In non FixupSCSExtendSlotSize mode we can skip this step.
301  void sortRegisters(SmallVectorImpl<Register> &Regs) {
303  return;
304  llvm::sort(Regs, [&](Register &A, Register &B) {
305  return getRegisterSize(TRI, A) > getRegisterSize(TRI, B);
306  });
307  }
308 };
309 
310 // Describes the state of the current processing statepoint instruction.
311 class StatepointState {
312 private:
313  // statepoint instruction.
314  MachineInstr &MI;
315  MachineFunction &MF;
316  // If non-null then statepoint is invoke, and this points to the landing pad.
317  MachineBasicBlock *EHPad;
318  const TargetRegisterInfo &TRI;
319  const TargetInstrInfo &TII;
320  MachineFrameInfo &MFI;
321  // Mask with callee saved registers.
322  const uint32_t *Mask;
323  // Cache of frame indexes used on previous instruction processing.
324  FrameIndexesCache &CacheFI;
325  bool AllowGCPtrInCSR;
326  // Operands with physical registers requiring spilling.
327  SmallVector<unsigned, 8> OpsToSpill;
328  // Set of register to spill.
329  SmallVector<Register, 8> RegsToSpill;
330  // Set of registers to reload after statepoint.
331  SmallVector<Register, 8> RegsToReload;
332  // Map Register to Frame Slot index.
333  DenseMap<Register, int> RegToSlotIdx;
334 
335 public:
336  StatepointState(MachineInstr &MI, const uint32_t *Mask,
337  FrameIndexesCache &CacheFI, bool AllowGCPtrInCSR)
338  : MI(MI), MF(*MI.getMF()), TRI(*MF.getSubtarget().getRegisterInfo()),
339  TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()),
340  Mask(Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {
341 
342  // Find statepoint's landing pad, if any.
343  EHPad = nullptr;
344  MachineBasicBlock *MBB = MI.getParent();
345  // Invoke statepoint must be last one in block.
346  bool Last = std::none_of(++MI.getIterator(), MBB->end().getInstrIterator(),
347  [](MachineInstr &I) {
348  return I.getOpcode() == TargetOpcode::STATEPOINT;
349  });
350 
351  if (!Last)
352  return;
353 
354  auto IsEHPad = [](MachineBasicBlock *B) { return B->isEHPad(); };
355 
356  assert(llvm::count_if(MBB->successors(), IsEHPad) < 2 && "multiple EHPads");
357 
358  auto It = llvm::find_if(MBB->successors(), IsEHPad);
359  if (It != MBB->succ_end())
360  EHPad = *It;
361  }
362 
363  MachineBasicBlock *getEHPad() const { return EHPad; }
364 
365  // Return true if register is callee saved.
366  bool isCalleeSaved(Register Reg) { return (Mask[Reg / 32] >> Reg % 32) & 1; }
367 
368  // Iterates over statepoint meta args to find caller saver registers.
369  // Also cache the size of found registers.
370  // Returns true if caller save registers found.
371  bool findRegistersToSpill() {
372  SmallSet<Register, 8> GCRegs;
373  // All GC pointer operands assigned to registers produce new value.
374  // Since they're tied to their defs, it is enough to collect def registers.
375  for (const auto &Def : MI.defs())
376  GCRegs.insert(Def.getReg());
377 
378  SmallSet<Register, 8> VisitedRegs;
379  for (unsigned Idx = StatepointOpers(&MI).getVarIdx(),
380  EndIdx = MI.getNumOperands();
381  Idx < EndIdx; ++Idx) {
382  MachineOperand &MO = MI.getOperand(Idx);
383  // Leave `undef` operands as is, StackMaps will rewrite them
384  // into a constant.
385  if (!MO.isReg() || MO.isImplicit() || MO.isUndef())
386  continue;
387  Register Reg = MO.getReg();
388  assert(Reg.isPhysical() && "Only physical regs are expected");
389 
390  if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !is_contained(GCRegs, Reg)))
391  continue;
392 
393  LLVM_DEBUG(dbgs() << "Will spill " << printReg(Reg, &TRI) << " at index "
394  << Idx << "\n");
395 
396  if (VisitedRegs.insert(Reg).second)
397  RegsToSpill.push_back(Reg);
398  OpsToSpill.push_back(Idx);
399  }
400  CacheFI.sortRegisters(RegsToSpill);
401  return !RegsToSpill.empty();
402  }
403 
404  // Spill all caller saved registers right before statepoint instruction.
405  // Remember frame index where register is spilled.
406  void spillRegisters() {
407  for (Register Reg : RegsToSpill) {
408  int FI = CacheFI.getFrameIndex(Reg, EHPad);
410 
411  NumSpilledRegisters++;
412  RegToSlotIdx[Reg] = FI;
413 
414  LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, &TRI) << " to FI " << FI
415  << "\n");
416 
417  // Perform trivial copy propagation
418  bool IsKill = true;
419  MachineBasicBlock::iterator InsertBefore(MI);
420  Reg = performCopyPropagation(Reg, InsertBefore, IsKill, TII, TRI);
421 
422  LLVM_DEBUG(dbgs() << "Insert spill before " << *InsertBefore);
423  TII.storeRegToStackSlot(*MI.getParent(), InsertBefore, Reg, IsKill, FI,
424  RC, &TRI);
425  }
426  }
427 
428  void insertReloadBefore(unsigned Reg, MachineBasicBlock::iterator It,
431  int FI = RegToSlotIdx[Reg];
432  if (It != MBB->end()) {
433  TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI);
434  return;
435  }
436 
437  // To insert reload at the end of MBB, insert it before last instruction
438  // and then swap them.
439  assert(!MBB->empty() && "Empty block");
440  --It;
441  TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI);
442  MachineInstr *Reload = It->getPrevNode();
443  int Dummy = 0;
444  (void)Dummy;
445  assert(TII.isLoadFromStackSlot(*Reload, Dummy) == Reg);
446  assert(Dummy == FI);
447  MBB->remove(Reload);
448  MBB->insertAfter(It, Reload);
449  }
450 
451  // Insert reloads of (relocated) registers spilled in statepoint.
452  void insertReloads(MachineInstr *NewStatepoint, RegReloadCache &RC) {
453  MachineBasicBlock *MBB = NewStatepoint->getParent();
454  auto InsertPoint = std::next(NewStatepoint->getIterator());
455 
456  for (auto Reg : RegsToReload) {
457  insertReloadBefore(Reg, InsertPoint, MBB);
458  LLVM_DEBUG(dbgs() << "Reloading " << printReg(Reg, &TRI) << " from FI "
459  << RegToSlotIdx[Reg] << " after statepoint\n");
460 
461  if (EHPad && !RC.hasReload(Reg, RegToSlotIdx[Reg], EHPad)) {
462  RC.recordReload(Reg, RegToSlotIdx[Reg], EHPad);
463  auto EHPadInsertPoint = EHPad->SkipPHIsLabelsAndDebug(EHPad->begin());
464  insertReloadBefore(Reg, EHPadInsertPoint, EHPad);
465  LLVM_DEBUG(dbgs() << "...also reload at EHPad "
466  << printMBBReference(*EHPad) << "\n");
467  }
468  }
469  }
470 
471  // Re-write statepoint machine instruction to replace caller saved operands
472  // with indirect memory location (frame index).
473  MachineInstr *rewriteStatepoint() {
474  MachineInstr *NewMI =
475  MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true);
476  MachineInstrBuilder MIB(MF, NewMI);
477 
478  unsigned NumOps = MI.getNumOperands();
479 
480  // New indices for the remaining defs.
481  SmallVector<unsigned, 8> NewIndices;
482  unsigned NumDefs = MI.getNumDefs();
483  for (unsigned I = 0; I < NumDefs; ++I) {
484  MachineOperand &DefMO = MI.getOperand(I);
485  assert(DefMO.isReg() && DefMO.isDef() && "Expected Reg Def operand");
486  Register Reg = DefMO.getReg();
487  assert(DefMO.isTied() && "Def is expected to be tied");
488  // We skipped undef uses and did not spill them, so we should not
489  // proceed with defs here.
490  if (MI.getOperand(MI.findTiedOperandIdx(I)).isUndef()) {
491  if (AllowGCPtrInCSR) {
492  NewIndices.push_back(NewMI->getNumOperands());
493  MIB.addReg(Reg, RegState::Define);
494  }
495  continue;
496  }
497  if (!AllowGCPtrInCSR) {
498  assert(is_contained(RegsToSpill, Reg));
499  RegsToReload.push_back(Reg);
500  } else {
501  if (isCalleeSaved(Reg)) {
502  NewIndices.push_back(NewMI->getNumOperands());
503  MIB.addReg(Reg, RegState::Define);
504  } else {
505  NewIndices.push_back(NumOps);
506  RegsToReload.push_back(Reg);
507  }
508  }
509  }
510 
511  // Add End marker.
512  OpsToSpill.push_back(MI.getNumOperands());
513  unsigned CurOpIdx = 0;
514 
515  for (unsigned I = NumDefs; I < MI.getNumOperands(); ++I) {
516  MachineOperand &MO = MI.getOperand(I);
517  if (I == OpsToSpill[CurOpIdx]) {
518  int FI = RegToSlotIdx[MO.getReg()];
519  MIB.addImm(StackMaps::IndirectMemRefOp);
520  MIB.addImm(getRegisterSize(TRI, MO.getReg()));
521  assert(MO.isReg() && "Should be register");
522  assert(MO.getReg().isPhysical() && "Should be physical register");
523  MIB.addFrameIndex(FI);
524  MIB.addImm(0);
525  ++CurOpIdx;
526  } else {
527  MIB.add(MO);
528  unsigned OldDef;
529  if (AllowGCPtrInCSR && MI.isRegTiedToDefOperand(I, &OldDef)) {
530  assert(OldDef < NumDefs);
531  assert(NewIndices[OldDef] < NumOps);
532  MIB->tieOperands(NewIndices[OldDef], MIB->getNumOperands() - 1);
533  }
534  }
535  }
536  assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed");
537  // Add mem operands.
538  NewMI->setMemRefs(MF, MI.memoperands());
539  for (auto It : RegToSlotIdx) {
540  Register R = It.first;
541  int FrameIndex = It.second;
542  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
544  if (is_contained(RegsToReload, R))
546  auto *MMO =
547  MF.getMachineMemOperand(PtrInfo, Flags, getRegisterSize(TRI, R),
549  NewMI->addMemOperand(MF, MMO);
550  }
551 
552  // Insert new statepoint and erase old one.
553  MI.getParent()->insert(MI, NewMI);
554 
555  LLVM_DEBUG(dbgs() << "rewritten statepoint to : " << *NewMI << "\n");
556  MI.eraseFromParent();
557  return NewMI;
558  }
559 };
560 
561 class StatepointProcessor {
562 private:
563  MachineFunction &MF;
564  const TargetRegisterInfo &TRI;
565  FrameIndexesCache CacheFI;
566  RegReloadCache ReloadCache;
567 
568 public:
569  StatepointProcessor(MachineFunction &MF)
570  : MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()),
571  CacheFI(MF.getFrameInfo(), TRI) {}
572 
573  bool process(MachineInstr &MI, bool AllowGCPtrInCSR) {
574  StatepointOpers SO(&MI);
575  uint64_t Flags = SO.getFlags();
576  // Do nothing for LiveIn, it supports all registers.
577  if (Flags & (uint64_t)StatepointFlags::DeoptLiveIn)
578  return false;
579  LLVM_DEBUG(dbgs() << "\nMBB " << MI.getParent()->getNumber() << " "
580  << MI.getParent()->getName() << " : process statepoint "
581  << MI);
582  CallingConv::ID CC = SO.getCallingConv();
583  const uint32_t *Mask = TRI.getCallPreservedMask(MF, CC);
584  StatepointState SS(MI, Mask, CacheFI, AllowGCPtrInCSR);
585  CacheFI.reset(SS.getEHPad());
586 
587  if (!SS.findRegistersToSpill())
588  return false;
589 
590  SS.spillRegisters();
591  auto *NewStatepoint = SS.rewriteStatepoint();
592  SS.insertReloads(NewStatepoint, ReloadCache);
593  return true;
594  }
595 };
596 } // namespace
597 
598 bool FixupStatepointCallerSaved::runOnMachineFunction(MachineFunction &MF) {
599  if (skipFunction(MF.getFunction()))
600  return false;
601 
602  const Function &F = MF.getFunction();
603  if (!F.hasGC())
604  return false;
605 
607  for (MachineBasicBlock &BB : MF)
608  for (MachineInstr &I : BB)
609  if (I.getOpcode() == TargetOpcode::STATEPOINT)
610  Statepoints.push_back(&I);
611 
612  if (Statepoints.empty())
613  return false;
614 
615  bool Changed = false;
616  StatepointProcessor SPP(MF);
617  unsigned NumStatepoints = 0;
618  bool AllowGCPtrInCSR = PassGCPtrInCSR;
619  for (MachineInstr *I : Statepoints) {
620  ++NumStatepoints;
621  if (MaxStatepointsWithRegs.getNumOccurrences() &&
622  NumStatepoints >= MaxStatepointsWithRegs)
623  AllowGCPtrInCSR = false;
624  Changed |= SPP.process(*I, AllowGCPtrInCSR);
625  }
626  return Changed;
627 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:100
llvm
Definition: AllocatorList.h:23
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1496
TargetFrameLowering.h
llvm::Function
Definition: Function.h:61
llvm::SmallVector< int, 8 >
Statistic.h
PassGCPtrInCSR
static cl::opt< bool > PassGCPtrInCSR("fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false), cl::desc("Allow passing GC Pointer arguments in callee saved registers"))
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:430
llvm::printMBBReference
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Definition: MachineBasicBlock.cpp:118
llvm::MachineOperand::isTied
bool isTied() const
Definition: MachineOperand.h:441
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
Saved
Fixup Statepoint Caller Saved
Definition: FixupStatepointCallerSaved.cpp:93
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:231
TargetInstrInfo.h
llvm::SmallSet
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:134
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::count
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:145
llvm::MachineFrameInfo::setObjectSize
void setObjectSize(int ObjectIdx, int64_t Size)
Change the size of the specified stack object.
Definition: MachineFrameInfo.h:458
llvm::BitmaskEnumDetail::Mask
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1567
llvm::count_if
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:1572
llvm::MachineFrameInfo::setObjectAlignment
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
Definition: MachineFrameInfo.h:472
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:102
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:122
llvm::MachineInstr::addMemOperand
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
Definition: MachineInstr.cpp:382
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:180
MachineRegisterInfo.h
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(FixupStatepointCallerSaved, DEBUG_TYPE, "Fixup Statepoint Caller Saved", false, false) INITIALIZE_PASS_END(FixupStatepointCallerSaved
llvm::Register::isPhysical
bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:97
llvm::X86AS::SS
@ SS
Definition: X86.h:189
llvm::MachineOperand::isImplicit
bool isImplicit() const
Definition: MachineOperand.h:380
llvm::MachineBasicBlock::insertAfter
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
Definition: MachineBasicBlock.h:784
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
llvm::MachineBasicBlock::remove
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
Definition: MachineBasicBlock.h:843
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineInstr::setMemRefs
void setMemRefs(MachineFunction &MF, ArrayRef< MachineMemOperand * > MemRefs)
Assign this MachineInstr's memory reference descriptor list.
Definition: MachineInstr.cpp:371
llvm::MachineBasicBlock::succ_end
succ_iterator succ_end()
Definition: MachineBasicBlock.h:334
performCopyPropagation
static Register performCopyPropagation(Register Reg, MachineBasicBlock::iterator &RI, bool &IsKill, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI)
Definition: FixupStatepointCallerSaved.cpp:114
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
false
Definition: StackSlotColoring.cpp:142
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:129
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:49
llvm::MachineBasicBlock::rend
reverse_iterator rend()
Definition: MachineBasicBlock.h:278
FixupSCSExtendSlotSize
static cl::opt< bool > FixupSCSExtendSlotSize("fixup-scs-extend-slot-size", cl::Hidden, cl::init(false), cl::desc("Allow spill in spill slot of greater size than register size"), cl::Hidden)
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
EnableCopyProp
static cl::opt< bool > EnableCopyProp("fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true), cl::desc("Enable simple copy propagation during register reloading"))
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
Statepoint.h
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
llvm::MachineFrameInfo::CreateSpillStackObject
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
Definition: MachineFrameInfo.cpp:66
Passes.h
llvm::MachineInstrBundleIterator::getInstrIterator
instr_iterator getInstrIterator() const
Definition: MachineInstrBundleIterator.h:274
llvm::cl::opt< bool >
llvm::TargetRegisterInfo::getSpillSize
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
Definition: TargetRegisterInfo.h:280
llvm::MachineOperand::isUndef
bool isUndef() const
Definition: MachineOperand.h:395
llvm::SmallSet::count
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164
llvm::MachineInstrBundleIterator::getReverse
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Definition: MachineInstrBundleIterator.h:283
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:321
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
llvm::MachineFrameInfo::getObjectSize
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Definition: MachineFrameInfo.h:451
getRegisterSize
Fixup Statepoint Caller static false unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg)
Definition: FixupStatepointCallerSaved.cpp:96
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
llvm::HexagonInstrInfo::storeRegToStackSlot
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const override
Store the specified register of the given register class to the specified stack frame index.
Definition: HexagonInstrInfo.cpp:913
llvm::DenseMap
Definition: DenseMap.h:714
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::MachineFrameInfo::getObjectAlign
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:465
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1547
MachineFunctionPass.h
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::find
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
llvm::MachineMemOperand::Flags
Flags
Flags values. These may be or'd together.
Definition: MachineMemOperand.h:130
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::IndexedInstrProf::HashT::Last
@ Last
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:360
MaxStatepointsWithRegs
static cl::opt< unsigned > MaxStatepointsWithRegs("fixup-max-csr-statepoints", cl::Hidden, cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"))
llvm::MachineFunction
Definition: MachineFunction.h:230
llvm::NVPTXISD::Dummy
@ Dummy
Definition: NVPTXISelLowering.h:60
llvm::MachineBasicBlock::iterator
MachineInstrBundleIterator< MachineInstr > iterator
Definition: MachineBasicBlock.h:233
llvm::MachineBasicBlock::successors
iterator_range< succ_iterator > successors()
Definition: MachineBasicBlock.h:355
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:253
llvm::HexagonInstrInfo::loadRegFromStackSlot
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const override
Load the specified register of the given register class from the specified stack frame index.
Definition: HexagonInstrInfo.cpp:958
uint32_t
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
llvm::MachineOperand::isDef
bool isDef() const
Definition: MachineOperand.h:375
llvm::HexagonInstrInfo::isLoadFromStackSlot
unsigned isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
Definition: HexagonInstrInfo.cpp:245
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:286
llvm::SmallSet::insert
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:180
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:207
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:134
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1509
llvm::FixupStatepointCallerSavedID
char & FixupStatepointCallerSavedID
The pass fixups statepoint machine instruction to replace usage of caller saved registers with stack ...
Definition: FixupStatepointCallerSaved.cpp:88
llvm::ISD::FrameIndex
@ FrameIndex
Definition: ISDOpcodes.h:73
DEBUG_TYPE
#define DEBUG_TYPE
Definition: FixupStatepointCallerSaved.cpp:39
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::StatepointOpers
MI-level Statepoint operands.
Definition: StackMaps.h:158
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:524
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::end
iterator end()
Definition: DenseMap.h:83
MachineFrameInfo.h
llvm::sort
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1423
StackMaps.h
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:136
llvm::MachineFrameInfo
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Definition: MachineFrameInfo.h:107
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:995
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:268
llvm::MachineInstr::getNumOperands
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:481
llvm::MachineBasicBlock::empty
bool empty() const
Definition: MachineBasicBlock.h:240
llvm::SmallSet::clear
void clear()
Definition: SmallSet.h:218
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
llvm::StatepointFlags::DeoptLiveIn
@ DeoptLiveIn
Mark the deopt arguments associated with the statepoint as only being "live-in".
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::MachineFunction::CreateMachineInstr
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, const DebugLoc &DL, bool NoImplicit=false)
CreateMachineInstr - Allocate a new MachineInstr.
Definition: MachineFunction.cpp:348
llvm::MachineBasicBlock::SkipPHIsLabelsAndDebug
iterator SkipPHIsLabelsAndDebug(iterator I, bool SkipPseudoOp=true)
Return the first instruction in MBB after I that is not a PHI, label or debug.
Definition: MachineBasicBlock.cpp:224
llvm::cl::desc
Definition: CommandLine.h:414
llvm::TargetRegisterInfo::getMinimalPhysRegClass
const TargetRegisterClass * getMinimalPhysRegClass(MCRegister Reg, MVT VT=MVT::Other) const
Returns the Register Class of a physical register of the given type, picking the most sub register cl...
Definition: TargetRegisterInfo.cpp:211
llvm::printReg
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Definition: TargetRegisterInfo.cpp:110
llvm::MachineInstrBundleIterator< MachineInstr >
InitializePasses.h
llvm::initializeFixupStatepointCallerSavedPass
void initializeFixupStatepointCallerSavedPass(PassRegistry &)
Debug.h
llvm::TargetRegisterInfo::getCallPreservedMask
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function.
Definition: TargetRegisterInfo.h:485
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:270
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:44
SmallSet.h
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38