LLVM  14.0.0git
SIFrameLowering.cpp
Go to the documentation of this file.
1 //===----------------------- SIFrameLowering.cpp --------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 
9 #include "SIFrameLowering.h"
10 #include "AMDGPU.h"
11 #include "GCNSubtarget.h"
13 #include "SIMachineFunctionInfo.h"
18 
19 using namespace llvm;
20 
21 #define DEBUG_TYPE "frame-info"
22 
24  "amdgpu-spill-vgpr-to-agpr",
25  cl::desc("Enable spilling VGPRs to AGPRs"),
27  cl::init(true));
28 
29 // Find a scratch register that we can use in the prologue. We avoid using
30 // callee-save registers since they may appear to be free when this is called
31 // from canUseAsPrologue (during shrink wrapping), but then no longer be free
32 // when this is called from emitPrologue.
34  LivePhysRegs &LiveRegs,
35  const TargetRegisterClass &RC,
36  bool Unused = false) {
37  // Mark callee saved registers as used so we will not choose them.
38  const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
39  for (unsigned i = 0; CSRegs[i]; ++i)
40  LiveRegs.addReg(CSRegs[i]);
41 
42  if (Unused) {
43  // We are looking for a register that can be used throughout the entire
44  // function, so any use is unacceptable.
45  for (MCRegister Reg : RC) {
46  if (!MRI.isPhysRegUsed(Reg) && LiveRegs.available(MRI, Reg))
47  return Reg;
48  }
49  } else {
50  for (MCRegister Reg : RC) {
51  if (LiveRegs.available(MRI, Reg))
52  return Reg;
53  }
54  }
55 
56  return MCRegister();
57 }
58 
60  LivePhysRegs &LiveRegs,
61  Register &TempSGPR,
63  bool IsFP) {
65  MachineFrameInfo &FrameInfo = MF.getFrameInfo();
66 
68  const SIRegisterInfo *TRI = ST.getRegisterInfo();
69 
70  // We need to save and restore the current FP/BP.
71 
72  // 1: If there is already a VGPR with free lanes, use it. We
73  // may already have to pay the penalty for spilling a CSR VGPR.
74  if (MFI->haveFreeLanesForSGPRSpill(MF, 1)) {
75  int NewFI = FrameInfo.CreateStackObject(4, Align(4), true, nullptr,
77 
78  if (!MFI->allocateSGPRSpillToVGPR(MF, NewFI))
79  llvm_unreachable("allocate SGPR spill should have worked");
80 
81  FrameIndex = NewFI;
82 
83  LLVM_DEBUG(auto Spill = MFI->getSGPRToVGPRSpills(NewFI).front();
84  dbgs() << "Spilling " << (IsFP ? "FP" : "BP") << " to "
85  << printReg(Spill.VGPR, TRI) << ':' << Spill.Lane
86  << '\n');
87  return;
88  }
89 
90  // 2: Next, try to save the FP/BP in an unused SGPR.
92  MF.getRegInfo(), LiveRegs, AMDGPU::SReg_32_XM0_XEXECRegClass, true);
93 
94  if (!TempSGPR) {
95  int NewFI = FrameInfo.CreateStackObject(4, Align(4), true, nullptr,
97 
98  if (TRI->spillSGPRToVGPR() && MFI->allocateSGPRSpillToVGPR(MF, NewFI)) {
99  // 3: There's no free lane to spill, and no free register to save FP/BP,
100  // so we're forced to spill another VGPR to use for the spill.
101  FrameIndex = NewFI;
102 
103  LLVM_DEBUG(
104  auto Spill = MFI->getSGPRToVGPRSpills(NewFI).front();
105  dbgs() << (IsFP ? "FP" : "BP") << " requires fallback spill to "
106  << printReg(Spill.VGPR, TRI) << ':' << Spill.Lane << '\n';);
107  } else {
108  // Remove dead <NewFI> index
109  MF.getFrameInfo().RemoveStackObject(NewFI);
110  // 4: If all else fails, spill the FP/BP to memory.
111  FrameIndex = FrameInfo.CreateSpillStackObject(4, Align(4));
112  LLVM_DEBUG(dbgs() << "Reserved FI " << FrameIndex << " for spilling "
113  << (IsFP ? "FP" : "BP") << '\n');
114  }
115  } else {
116  LLVM_DEBUG(dbgs() << "Saving " << (IsFP ? "FP" : "BP") << " with copy to "
117  << printReg(TempSGPR, TRI) << '\n');
118  }
119 }
120 
121 // We need to specially emit stack operations here because a different frame
122 // register is used than in the rest of the function, as getFrameRegister would
123 // use.
124 static void buildPrologSpill(const GCNSubtarget &ST, const SIRegisterInfo &TRI,
125  const SIMachineFunctionInfo &FuncInfo,
126  LivePhysRegs &LiveRegs, MachineFunction &MF,
129  int FI) {
130  unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
131  : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
132 
133  MachineFrameInfo &FrameInfo = MF.getFrameInfo();
136  PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FI),
137  FrameInfo.getObjectAlign(FI));
138  LiveRegs.addReg(SpillReg);
139  TRI.buildSpillLoadStore(MBB, I, Opc, FI, SpillReg, true,
140  FuncInfo.getStackPtrOffsetReg(), 0, MMO, nullptr,
141  &LiveRegs);
142  LiveRegs.removeReg(SpillReg);
143 }
144 
145 static void buildEpilogRestore(const GCNSubtarget &ST,
146  const SIRegisterInfo &TRI,
147  const SIMachineFunctionInfo &FuncInfo,
148  LivePhysRegs &LiveRegs, MachineFunction &MF,
151  int FI) {
152  unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
153  : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
154 
155  MachineFrameInfo &FrameInfo = MF.getFrameInfo();
158  PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FI),
159  FrameInfo.getObjectAlign(FI));
160  TRI.buildSpillLoadStore(MBB, I, Opc, FI, SpillReg, false,
161  FuncInfo.getStackPtrOffsetReg(), 0, MMO, nullptr,
162  &LiveRegs);
163 }
164 
166  const DebugLoc &DL, const SIInstrInfo *TII,
167  Register TargetReg) {
168  MachineFunction *MF = MBB.getParent();
170  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
171  const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
172  Register TargetLo = TRI->getSubReg(TargetReg, AMDGPU::sub0);
173  Register TargetHi = TRI->getSubReg(TargetReg, AMDGPU::sub1);
174 
175  if (MFI->getGITPtrHigh() != 0xffffffff) {
176  BuildMI(MBB, I, DL, SMovB32, TargetHi)
177  .addImm(MFI->getGITPtrHigh())
178  .addReg(TargetReg, RegState::ImplicitDefine);
179  } else {
180  const MCInstrDesc &GetPC64 = TII->get(AMDGPU::S_GETPC_B64);
181  BuildMI(MBB, I, DL, GetPC64, TargetReg);
182  }
183  Register GitPtrLo = MFI->getGITPtrLoReg(*MF);
184  MF->getRegInfo().addLiveIn(GitPtrLo);
185  MBB.addLiveIn(GitPtrLo);
186  BuildMI(MBB, I, DL, SMovB32, TargetLo)
187  .addReg(GitPtrLo);
188 }
189 
190 // Emit flat scratch setup code, assuming `MFI->hasFlatScratchInit()`
191 void SIFrameLowering::emitEntryFunctionFlatScratchInit(
193  const DebugLoc &DL, Register ScratchWaveOffsetReg) const {
194  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
195  const SIInstrInfo *TII = ST.getInstrInfo();
196  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
198 
199  // We don't need this if we only have spills since there is no user facing
200  // scratch.
201 
202  // TODO: If we know we don't have flat instructions earlier, we can omit
203  // this from the input registers.
204  //
205  // TODO: We only need to know if we access scratch space through a flat
206  // pointer. Because we only detect if flat instructions are used at all,
207  // this will be used more often than necessary on VI.
208 
209  Register FlatScrInitLo;
210  Register FlatScrInitHi;
211 
212  if (ST.isAmdPalOS()) {
213  // Extract the scratch offset from the descriptor in the GIT
214  LivePhysRegs LiveRegs;
215  LiveRegs.init(*TRI);
216  LiveRegs.addLiveIns(MBB);
217 
218  // Find unused reg to load flat scratch init into
220  Register FlatScrInit = AMDGPU::NoRegister;
221  ArrayRef<MCPhysReg> AllSGPR64s = TRI->getAllSGPR64(MF);
222  unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 1) / 2;
223  AllSGPR64s = AllSGPR64s.slice(
224  std::min(static_cast<unsigned>(AllSGPR64s.size()), NumPreloaded));
225  Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
226  for (MCPhysReg Reg : AllSGPR64s) {
227  if (LiveRegs.available(MRI, Reg) && MRI.isAllocatable(Reg) &&
228  !TRI->isSubRegisterEq(Reg, GITPtrLoReg)) {
229  FlatScrInit = Reg;
230  break;
231  }
232  }
233  assert(FlatScrInit && "Failed to find free register for scratch init");
234 
235  FlatScrInitLo = TRI->getSubReg(FlatScrInit, AMDGPU::sub0);
236  FlatScrInitHi = TRI->getSubReg(FlatScrInit, AMDGPU::sub1);
237 
238  buildGitPtr(MBB, I, DL, TII, FlatScrInit);
239 
240  // We now have the GIT ptr - now get the scratch descriptor from the entry
241  // at offset 0 (or offset 16 for a compute shader).
243  const MCInstrDesc &LoadDwordX2 = TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
244  auto *MMO = MF.getMachineMemOperand(
245  PtrInfo,
248  8, Align(4));
249  unsigned Offset =
251  const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
252  unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(Subtarget, Offset);
253  BuildMI(MBB, I, DL, LoadDwordX2, FlatScrInit)
254  .addReg(FlatScrInit)
255  .addImm(EncodedOffset) // offset
256  .addImm(0) // cpol
257  .addMemOperand(MMO);
258 
259  // Mask the offset in [47:0] of the descriptor
260  const MCInstrDesc &SAndB32 = TII->get(AMDGPU::S_AND_B32);
261  BuildMI(MBB, I, DL, SAndB32, FlatScrInitHi)
262  .addReg(FlatScrInitHi)
263  .addImm(0xffff);
264  } else {
265  Register FlatScratchInitReg =
267  assert(FlatScratchInitReg);
268 
270  MRI.addLiveIn(FlatScratchInitReg);
271  MBB.addLiveIn(FlatScratchInitReg);
272 
273  FlatScrInitLo = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub0);
274  FlatScrInitHi = TRI->getSubReg(FlatScratchInitReg, AMDGPU::sub1);
275  }
276 
277  // Do a 64-bit pointer add.
278  if (ST.flatScratchIsPointer()) {
279  if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
280  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), FlatScrInitLo)
281  .addReg(FlatScrInitLo)
282  .addReg(ScratchWaveOffsetReg);
283  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32), FlatScrInitHi)
284  .addReg(FlatScrInitHi)
285  .addImm(0);
286  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SETREG_B32)).
287  addReg(FlatScrInitLo).
288  addImm(int16_t(AMDGPU::Hwreg::ID_FLAT_SCR_LO |
290  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SETREG_B32)).
291  addReg(FlatScrInitHi).
292  addImm(int16_t(AMDGPU::Hwreg::ID_FLAT_SCR_HI |
294  return;
295  }
296 
297  // For GFX9.
298  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), AMDGPU::FLAT_SCR_LO)
299  .addReg(FlatScrInitLo)
300  .addReg(ScratchWaveOffsetReg);
301  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32), AMDGPU::FLAT_SCR_HI)
302  .addReg(FlatScrInitHi)
303  .addImm(0);
304 
305  return;
306  }
307 
308  assert(ST.getGeneration() < AMDGPUSubtarget::GFX9);
309 
310  // Copy the size in bytes.
311  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::FLAT_SCR_LO)
312  .addReg(FlatScrInitHi, RegState::Kill);
313 
314  // Add wave offset in bytes to private base offset.
315  // See comment in AMDKernelCodeT.h for enable_sgpr_flat_scratch_init.
316  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), FlatScrInitLo)
317  .addReg(FlatScrInitLo)
318  .addReg(ScratchWaveOffsetReg);
319 
320  // Convert offset to 256-byte units.
321  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_LSHR_B32), AMDGPU::FLAT_SCR_HI)
322  .addReg(FlatScrInitLo, RegState::Kill)
323  .addImm(8);
324 }
325 
326 // Note SGPRSpill stack IDs should only be used for SGPR spilling to VGPRs, not
327 // memory. They should have been removed by now.
328 static bool allStackObjectsAreDead(const MachineFrameInfo &MFI) {
329  for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
330  I != E; ++I) {
331  if (!MFI.isDeadObjectIndex(I))
332  return false;
333  }
334 
335  return true;
336 }
337 
338 // Shift down registers reserved for the scratch RSRC.
339 Register SIFrameLowering::getEntryFunctionReservedScratchRsrcReg(
340  MachineFunction &MF) const {
341 
342  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
343  const SIInstrInfo *TII = ST.getInstrInfo();
344  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
347 
348  assert(MFI->isEntryFunction());
349 
350  Register ScratchRsrcReg = MFI->getScratchRSrcReg();
351 
352  if (!ScratchRsrcReg || (!MRI.isPhysRegUsed(ScratchRsrcReg) &&
354  return Register();
355 
356  if (ST.hasSGPRInitBug() ||
357  ScratchRsrcReg != TRI->reservedPrivateSegmentBufferReg(MF))
358  return ScratchRsrcReg;
359 
360  // We reserved the last registers for this. Shift it down to the end of those
361  // which were actually used.
362  //
363  // FIXME: It might be safer to use a pseudoregister before replacement.
364 
365  // FIXME: We should be able to eliminate unused input registers. We only
366  // cannot do this for the resources required for scratch access. For now we
367  // skip over user SGPRs and may leave unused holes.
368 
369  unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 3) / 4;
370  ArrayRef<MCPhysReg> AllSGPR128s = TRI->getAllSGPR128(MF);
371  AllSGPR128s = AllSGPR128s.slice(std::min(static_cast<unsigned>(AllSGPR128s.size()), NumPreloaded));
372 
373  // Skip the last N reserved elements because they should have already been
374  // reserved for VCC etc.
375  Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
376  for (MCPhysReg Reg : AllSGPR128s) {
377  // Pick the first unallocated one. Make sure we don't clobber the other
378  // reserved input we needed. Also for PAL, make sure we don't clobber
379  // the GIT pointer passed in SGPR0 or SGPR8.
381  !TRI->isSubRegisterEq(Reg, GITPtrLoReg)) {
382  MRI.replaceRegWith(ScratchRsrcReg, Reg);
383  MFI->setScratchRSrcReg(Reg);
384  return Reg;
385  }
386  }
387 
388  return ScratchRsrcReg;
389 }
390 
391 static unsigned getScratchScaleFactor(const GCNSubtarget &ST) {
392  return ST.enableFlatScratch() ? 1 : ST.getWavefrontSize();
393 }
394 
396  MachineBasicBlock &MBB) const {
397  assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
398 
399  // FIXME: If we only have SGPR spills, we won't actually be using scratch
400  // memory since these spill to VGPRs. We should be cleaning up these unused
401  // SGPR spill frame indices somewhere.
402 
403  // FIXME: We still have implicit uses on SGPR spill instructions in case they
404  // need to spill to vector memory. It's likely that will not happen, but at
405  // this point it appears we need the setup. This part of the prolog should be
406  // emitted after frame indices are eliminated.
407 
408  // FIXME: Remove all of the isPhysRegUsed checks
409 
411  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
412  const SIInstrInfo *TII = ST.getInstrInfo();
413  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
415  const Function &F = MF.getFunction();
416  MachineFrameInfo &FrameInfo = MF.getFrameInfo();
417 
418  assert(MFI->isEntryFunction());
419 
420  Register PreloadedScratchWaveOffsetReg = MFI->getPreloadedReg(
422  // FIXME: Hack to not crash in situations which emitted an error.
423  if (!PreloadedScratchWaveOffsetReg)
424  return;
425 
426  // We need to do the replacement of the private segment buffer register even
427  // if there are no stack objects. There could be stores to undef or a
428  // constant without an associated object.
429  //
430  // This will return `Register()` in cases where there are no actual
431  // uses of the SRSRC.
432  Register ScratchRsrcReg;
433  if (!ST.enableFlatScratch())
434  ScratchRsrcReg = getEntryFunctionReservedScratchRsrcReg(MF);
435 
436  // Make the selected register live throughout the function.
437  if (ScratchRsrcReg) {
438  for (MachineBasicBlock &OtherBB : MF) {
439  if (&OtherBB != &MBB) {
440  OtherBB.addLiveIn(ScratchRsrcReg);
441  }
442  }
443  }
444 
445  // Now that we have fixed the reserved SRSRC we need to locate the
446  // (potentially) preloaded SRSRC.
447  Register PreloadedScratchRsrcReg;
448  if (ST.isAmdHsaOrMesa(F)) {
449  PreloadedScratchRsrcReg =
451  if (ScratchRsrcReg && PreloadedScratchRsrcReg) {
452  // We added live-ins during argument lowering, but since they were not
453  // used they were deleted. We're adding the uses now, so add them back.
454  MRI.addLiveIn(PreloadedScratchRsrcReg);
455  MBB.addLiveIn(PreloadedScratchRsrcReg);
456  }
457  }
458 
459  // Debug location must be unknown since the first debug location is used to
460  // determine the end of the prologue.
461  DebugLoc DL;
463 
464  // We found the SRSRC first because it needs four registers and has an
465  // alignment requirement. If the SRSRC that we found is clobbering with
466  // the scratch wave offset, which may be in a fixed SGPR or a free SGPR
467  // chosen by SITargetLowering::allocateSystemSGPRs, COPY the scratch
468  // wave offset to a free SGPR.
469  Register ScratchWaveOffsetReg;
470  if (TRI->isSubRegisterEq(ScratchRsrcReg, PreloadedScratchWaveOffsetReg)) {
471  ArrayRef<MCPhysReg> AllSGPRs = TRI->getAllSGPR32(MF);
472  unsigned NumPreloaded = MFI->getNumPreloadedSGPRs();
473  AllSGPRs = AllSGPRs.slice(
474  std::min(static_cast<unsigned>(AllSGPRs.size()), NumPreloaded));
475  Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
476  for (MCPhysReg Reg : AllSGPRs) {
478  !TRI->isSubRegisterEq(ScratchRsrcReg, Reg) && GITPtrLoReg != Reg) {
479  ScratchWaveOffsetReg = Reg;
480  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchWaveOffsetReg)
481  .addReg(PreloadedScratchWaveOffsetReg, RegState::Kill);
482  break;
483  }
484  }
485  } else {
486  ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg;
487  }
488  assert(ScratchWaveOffsetReg);
489 
491  Register SPReg = MFI->getStackPtrOffsetReg();
492  assert(SPReg != AMDGPU::SP_REG);
493  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), SPReg)
494  .addImm(FrameInfo.getStackSize() * getScratchScaleFactor(ST));
495  }
496 
497  if (hasFP(MF)) {
498  Register FPReg = MFI->getFrameOffsetReg();
499  assert(FPReg != AMDGPU::FP_REG);
500  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), FPReg).addImm(0);
501  }
502 
503  bool NeedsFlatScratchInit =
504  MFI->hasFlatScratchInit() &&
505  (MRI.isPhysRegUsed(AMDGPU::FLAT_SCR) || FrameInfo.hasCalls() ||
506  (!allStackObjectsAreDead(FrameInfo) && ST.enableFlatScratch()));
507 
508  if ((NeedsFlatScratchInit || ScratchRsrcReg) &&
509  !ST.flatScratchIsArchitected()) {
510  MRI.addLiveIn(PreloadedScratchWaveOffsetReg);
511  MBB.addLiveIn(PreloadedScratchWaveOffsetReg);
512  }
513 
514  if (NeedsFlatScratchInit) {
515  emitEntryFunctionFlatScratchInit(MF, MBB, I, DL, ScratchWaveOffsetReg);
516  }
517 
518  if (ScratchRsrcReg) {
519  emitEntryFunctionScratchRsrcRegSetup(MF, MBB, I, DL,
520  PreloadedScratchRsrcReg,
521  ScratchRsrcReg, ScratchWaveOffsetReg);
522  }
523 }
524 
525 // Emit scratch RSRC setup code, assuming `ScratchRsrcReg != AMDGPU::NoReg`
526 void SIFrameLowering::emitEntryFunctionScratchRsrcRegSetup(
528  const DebugLoc &DL, Register PreloadedScratchRsrcReg,
529  Register ScratchRsrcReg, Register ScratchWaveOffsetReg) const {
530 
531  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
532  const SIInstrInfo *TII = ST.getInstrInfo();
533  const SIRegisterInfo *TRI = &TII->getRegisterInfo();
535  const Function &Fn = MF.getFunction();
536 
537  if (ST.isAmdPalOS()) {
538  // The pointer to the GIT is formed from the offset passed in and either
539  // the amdgpu-git-ptr-high function attribute or the top part of the PC
540  Register Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
541  Register Rsrc03 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
542 
543  buildGitPtr(MBB, I, DL, TII, Rsrc01);
544 
545  // We now have the GIT ptr - now get the scratch descriptor from the entry
546  // at offset 0 (or offset 16 for a compute shader).
548  const MCInstrDesc &LoadDwordX4 = TII->get(AMDGPU::S_LOAD_DWORDX4_IMM);
549  auto MMO = MF.getMachineMemOperand(PtrInfo,
553  16, Align(4));
554  unsigned Offset = Fn.getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
555  const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
556  unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(Subtarget, Offset);
557  BuildMI(MBB, I, DL, LoadDwordX4, ScratchRsrcReg)
558  .addReg(Rsrc01)
559  .addImm(EncodedOffset) // offset
560  .addImm(0) // cpol
561  .addReg(ScratchRsrcReg, RegState::ImplicitDefine)
562  .addMemOperand(MMO);
563 
564  // The driver will always set the SRD for wave 64 (bits 118:117 of
565  // descriptor / bits 22:21 of third sub-reg will be 0b11)
566  // If the shader is actually wave32 we have to modify the const_index_stride
567  // field of the descriptor 3rd sub-reg (bits 22:21) to 0b10 (stride=32). The
568  // reason the driver does this is that there can be cases where it presents
569  // 2 shaders with different wave size (e.g. VsFs).
570  // TODO: convert to using SCRATCH instructions or multiple SRD buffers
571  if (ST.isWave32()) {
572  const MCInstrDesc &SBitsetB32 = TII->get(AMDGPU::S_BITSET0_B32);
573  BuildMI(MBB, I, DL, SBitsetB32, Rsrc03)
574  .addImm(21)
575  .addReg(Rsrc03);
576  }
577  } else if (ST.isMesaGfxShader(Fn) || !PreloadedScratchRsrcReg) {
578  assert(!ST.isAmdHsaOrMesa(Fn));
579  const MCInstrDesc &SMovB32 = TII->get(AMDGPU::S_MOV_B32);
580 
581  Register Rsrc2 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub2);
582  Register Rsrc3 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub3);
583 
584  // Use relocations to get the pointer, and setup the other bits manually.
585  uint64_t Rsrc23 = TII->getScratchRsrcWords23();
586 
587  if (MFI->hasImplicitBufferPtr()) {
588  Register Rsrc01 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0_sub1);
589 
591  const MCInstrDesc &Mov64 = TII->get(AMDGPU::S_MOV_B64);
592 
593  BuildMI(MBB, I, DL, Mov64, Rsrc01)
595  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
596  } else {
597  const MCInstrDesc &LoadDwordX2 = TII->get(AMDGPU::S_LOAD_DWORDX2_IMM);
598 
600  auto MMO = MF.getMachineMemOperand(
601  PtrInfo,
604  8, Align(4));
605  BuildMI(MBB, I, DL, LoadDwordX2, Rsrc01)
607  .addImm(0) // offset
608  .addImm(0) // cpol
609  .addMemOperand(MMO)
610  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
611 
614  }
615  } else {
616  Register Rsrc0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
617  Register Rsrc1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
618 
619  BuildMI(MBB, I, DL, SMovB32, Rsrc0)
620  .addExternalSymbol("SCRATCH_RSRC_DWORD0")
621  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
622 
623  BuildMI(MBB, I, DL, SMovB32, Rsrc1)
624  .addExternalSymbol("SCRATCH_RSRC_DWORD1")
625  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
626 
627  }
628 
629  BuildMI(MBB, I, DL, SMovB32, Rsrc2)
630  .addImm(Rsrc23 & 0xffffffff)
631  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
632 
633  BuildMI(MBB, I, DL, SMovB32, Rsrc3)
634  .addImm(Rsrc23 >> 32)
635  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
636  } else if (ST.isAmdHsaOrMesa(Fn)) {
637  assert(PreloadedScratchRsrcReg);
638 
639  if (ScratchRsrcReg != PreloadedScratchRsrcReg) {
640  BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), ScratchRsrcReg)
641  .addReg(PreloadedScratchRsrcReg, RegState::Kill);
642  }
643  }
644 
645  // Add the scratch wave offset into the scratch RSRC.
646  //
647  // We only want to update the first 48 bits, which is the base address
648  // pointer, without touching the adjacent 16 bits of flags. We know this add
649  // cannot carry-out from bit 47, otherwise the scratch allocation would be
650  // impossible to fit in the 48-bit global address space.
651  //
652  // TODO: Evaluate if it is better to just construct an SRD using the flat
653  // scratch init and some constants rather than update the one we are passed.
654  Register ScratchRsrcSub0 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub0);
655  Register ScratchRsrcSub1 = TRI->getSubReg(ScratchRsrcReg, AMDGPU::sub1);
656 
657  // We cannot Kill ScratchWaveOffsetReg here because we allow it to be used in
658  // the kernel body via inreg arguments.
659  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_U32), ScratchRsrcSub0)
660  .addReg(ScratchRsrcSub0)
661  .addReg(ScratchWaveOffsetReg)
662  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
663  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADDC_U32), ScratchRsrcSub1)
664  .addReg(ScratchRsrcSub1)
665  .addImm(0)
666  .addReg(ScratchRsrcReg, RegState::ImplicitDefine);
667 }
668 
670  switch (ID) {
674  return true;
677  return false;
678  }
679  llvm_unreachable("Invalid TargetStackID::Value");
680 }
681 
682 static void initLiveRegs(LivePhysRegs &LiveRegs, const SIRegisterInfo &TRI,
683  const SIMachineFunctionInfo *FuncInfo,
685  MachineBasicBlock::iterator MBBI, bool IsProlog) {
686  if (LiveRegs.empty()) {
687  LiveRegs.init(TRI);
688  if (IsProlog) {
689  LiveRegs.addLiveIns(MBB);
690  } else {
691  // In epilog.
692  LiveRegs.addLiveOuts(MBB);
693  LiveRegs.stepBackward(*MBBI);
694  }
695  }
696 }
697 
698 // Activate all lanes, returns saved exec.
700  MachineFunction &MF,
703  bool IsProlog) {
704  Register ScratchExecCopy;
706  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
707  const SIInstrInfo *TII = ST.getInstrInfo();
708  const SIRegisterInfo &TRI = TII->getRegisterInfo();
710  DebugLoc DL;
711 
712  initLiveRegs(LiveRegs, TRI, FuncInfo, MF, MBB, MBBI, IsProlog);
713 
714  ScratchExecCopy = findScratchNonCalleeSaveRegister(
715  MRI, LiveRegs, *TRI.getWaveMaskRegClass());
716  if (!ScratchExecCopy)
717  report_fatal_error("failed to find free scratch register");
718 
719  LiveRegs.addReg(ScratchExecCopy);
720 
721  const unsigned OrSaveExec =
722  ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 : AMDGPU::S_OR_SAVEEXEC_B64;
723  BuildMI(MBB, MBBI, DL, TII->get(OrSaveExec), ScratchExecCopy).addImm(-1);
724 
725  return ScratchExecCopy;
726 }
727 
728 // A StackID of SGPRSpill implies that this is a spill from SGPR to VGPR.
729 // Otherwise we are spilling to memory.
730 static bool spilledToMemory(const MachineFunction &MF, int SaveIndex) {
731  const MachineFrameInfo &MFI = MF.getFrameInfo();
732  return MFI.getStackID(SaveIndex) != TargetStackID::SGPRSpill;
733 }
734 
736  MachineBasicBlock &MBB) const {
738  if (FuncInfo->isEntryFunction()) {
740  return;
741  }
742 
743  const MachineFrameInfo &MFI = MF.getFrameInfo();
745  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
746  const SIInstrInfo *TII = ST.getInstrInfo();
747  const SIRegisterInfo &TRI = TII->getRegisterInfo();
748 
749  Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
750  Register FramePtrReg = FuncInfo->getFrameOffsetReg();
751  Register BasePtrReg =
752  TRI.hasBasePointer(MF) ? TRI.getBaseRegister() : Register();
753  LivePhysRegs LiveRegs;
754 
756  DebugLoc DL;
757 
758  bool HasFP = false;
759  bool HasBP = false;
760  uint32_t NumBytes = MFI.getStackSize();
761  uint32_t RoundedSize = NumBytes;
762  // To avoid clobbering VGPRs in lanes that weren't active on function entry,
763  // turn on all lanes before doing the spill to memory.
764  Register ScratchExecCopy;
765 
766  Optional<int> FPSaveIndex = FuncInfo->FramePointerSaveIndex;
767  Optional<int> BPSaveIndex = FuncInfo->BasePointerSaveIndex;
768 
769  // VGPRs used for SGPR->VGPR spills
771  FuncInfo->getSGPRSpillVGPRs()) {
772  if (!Reg.FI)
773  continue;
774 
775  if (!ScratchExecCopy)
776  ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI,
777  /*IsProlog*/ true);
778 
779  buildPrologSpill(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, Reg.VGPR,
780  *Reg.FI);
781  }
782 
783  // VGPRs used for Whole Wave Mode
784  for (const auto &Reg : FuncInfo->WWMReservedRegs) {
785  auto VGPR = Reg.first;
786  auto FI = Reg.second;
787  if (!FI)
788  continue;
789 
790  if (!ScratchExecCopy)
791  ScratchExecCopy =
792  buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, /*IsProlog*/ true);
793 
794  buildPrologSpill(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, VGPR, *FI);
795  }
796 
797  if (ScratchExecCopy) {
798  // FIXME: Split block and make terminator.
799  unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
800  MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
801  BuildMI(MBB, MBBI, DL, TII->get(ExecMov), Exec)
802  .addReg(ScratchExecCopy, RegState::Kill);
803  LiveRegs.addReg(ScratchExecCopy);
804  }
805 
806  if (FPSaveIndex && spilledToMemory(MF, *FPSaveIndex)) {
807  const int FramePtrFI = *FPSaveIndex;
808  assert(!MFI.isDeadObjectIndex(FramePtrFI));
809 
810  initLiveRegs(LiveRegs, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ true);
811 
813  MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
814  if (!TmpVGPR)
815  report_fatal_error("failed to find free scratch register");
816 
817  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
818  .addReg(FramePtrReg);
819 
820  buildPrologSpill(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, TmpVGPR,
821  FramePtrFI);
822  }
823 
824  if (BPSaveIndex && spilledToMemory(MF, *BPSaveIndex)) {
825  const int BasePtrFI = *BPSaveIndex;
826  assert(!MFI.isDeadObjectIndex(BasePtrFI));
827 
828  initLiveRegs(LiveRegs, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ true);
829 
831  MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
832  if (!TmpVGPR)
833  report_fatal_error("failed to find free scratch register");
834 
835  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_MOV_B32_e32), TmpVGPR)
836  .addReg(BasePtrReg);
837 
838  buildPrologSpill(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, TmpVGPR,
839  BasePtrFI);
840  }
841 
842  // In this case, spill the FP to a reserved VGPR.
843  if (FPSaveIndex && !spilledToMemory(MF, *FPSaveIndex)) {
844  const int FramePtrFI = *FPSaveIndex;
845  assert(!MFI.isDeadObjectIndex(FramePtrFI));
846 
847  assert(MFI.getStackID(FramePtrFI) == TargetStackID::SGPRSpill);
849  FuncInfo->getSGPRToVGPRSpills(FramePtrFI);
850  assert(Spill.size() == 1);
851 
852  // Save FP before setting it up.
853  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_WRITELANE_B32), Spill[0].VGPR)
854  .addReg(FramePtrReg)
855  .addImm(Spill[0].Lane)
856  .addReg(Spill[0].VGPR, RegState::Undef);
857  }
858 
859  // In this case, spill the BP to a reserved VGPR.
860  if (BPSaveIndex && !spilledToMemory(MF, *BPSaveIndex)) {
861  const int BasePtrFI = *BPSaveIndex;
862  assert(!MFI.isDeadObjectIndex(BasePtrFI));
863 
864  assert(MFI.getStackID(BasePtrFI) == TargetStackID::SGPRSpill);
866  FuncInfo->getSGPRToVGPRSpills(BasePtrFI);
867  assert(Spill.size() == 1);
868 
869  // Save BP before setting it up.
870  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_WRITELANE_B32), Spill[0].VGPR)
871  .addReg(BasePtrReg)
872  .addImm(Spill[0].Lane)
873  .addReg(Spill[0].VGPR, RegState::Undef);
874  }
875 
876  // Emit the copy if we need an FP, and are using a free SGPR to save it.
877  if (FuncInfo->SGPRForFPSaveRestoreCopy) {
878  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY),
879  FuncInfo->SGPRForFPSaveRestoreCopy)
880  .addReg(FramePtrReg)
882  }
883 
884  // Emit the copy if we need a BP, and are using a free SGPR to save it.
885  if (FuncInfo->SGPRForBPSaveRestoreCopy) {
886  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY),
887  FuncInfo->SGPRForBPSaveRestoreCopy)
888  .addReg(BasePtrReg)
890  }
891 
892  // If a copy has been emitted for FP and/or BP, Make the SGPRs
893  // used in the copy instructions live throughout the function.
894  SmallVector<MCPhysReg, 2> TempSGPRs;
895  if (FuncInfo->SGPRForFPSaveRestoreCopy)
896  TempSGPRs.push_back(FuncInfo->SGPRForFPSaveRestoreCopy);
897 
898  if (FuncInfo->SGPRForBPSaveRestoreCopy)
899  TempSGPRs.push_back(FuncInfo->SGPRForBPSaveRestoreCopy);
900 
901  if (!TempSGPRs.empty()) {
902  for (MachineBasicBlock &MBB : MF) {
903  for (MCPhysReg Reg : TempSGPRs)
904  MBB.addLiveIn(Reg);
905 
907  }
908  if (!LiveRegs.empty()) {
909  LiveRegs.addReg(FuncInfo->SGPRForFPSaveRestoreCopy);
910  LiveRegs.addReg(FuncInfo->SGPRForBPSaveRestoreCopy);
911  }
912  }
913 
914  if (TRI.hasStackRealignment(MF)) {
915  HasFP = true;
916  const unsigned Alignment = MFI.getMaxAlign().value();
917 
918  RoundedSize += Alignment;
919  if (LiveRegs.empty()) {
920  LiveRegs.init(TRI);
921  LiveRegs.addLiveIns(MBB);
922  }
923 
924  // s_add_i32 s33, s32, NumBytes
925  // s_and_b32 s33, s33, 0b111...0000
926  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_I32), FramePtrReg)
927  .addReg(StackPtrReg)
928  .addImm((Alignment - 1) * getScratchScaleFactor(ST))
930  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_AND_B32), FramePtrReg)
931  .addReg(FramePtrReg, RegState::Kill)
932  .addImm(-Alignment * getScratchScaleFactor(ST))
934  FuncInfo->setIsStackRealigned(true);
935  } else if ((HasFP = hasFP(MF))) {
936  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrReg)
937  .addReg(StackPtrReg)
939  }
940 
941  // If we need a base pointer, set it up here. It's whatever the value of
942  // the stack pointer is at this point. Any variable size objects will be
943  // allocated after this, so we can still use the base pointer to reference
944  // the incoming arguments.
945  if ((HasBP = TRI.hasBasePointer(MF))) {
946  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), BasePtrReg)
947  .addReg(StackPtrReg)
949  }
950 
951  if (HasFP && RoundedSize != 0) {
952  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_I32), StackPtrReg)
953  .addReg(StackPtrReg)
954  .addImm(RoundedSize * getScratchScaleFactor(ST))
956  }
957 
958  assert((!HasFP || (FuncInfo->SGPRForFPSaveRestoreCopy ||
959  FuncInfo->FramePointerSaveIndex)) &&
960  "Needed to save FP but didn't save it anywhere");
961 
962  assert((HasFP || (!FuncInfo->SGPRForFPSaveRestoreCopy &&
963  !FuncInfo->FramePointerSaveIndex)) &&
964  "Saved FP but didn't need it");
965 
966  assert((!HasBP || (FuncInfo->SGPRForBPSaveRestoreCopy ||
967  FuncInfo->BasePointerSaveIndex)) &&
968  "Needed to save BP but didn't save it anywhere");
969 
970  assert((HasBP || (!FuncInfo->SGPRForBPSaveRestoreCopy &&
971  !FuncInfo->BasePointerSaveIndex)) &&
972  "Saved BP but didn't need it");
973 }
974 
976  MachineBasicBlock &MBB) const {
977  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
978  if (FuncInfo->isEntryFunction())
979  return;
980 
981  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
982  const SIInstrInfo *TII = ST.getInstrInfo();
984  const SIRegisterInfo &TRI = TII->getRegisterInfo();
986  LivePhysRegs LiveRegs;
987  DebugLoc DL;
988 
989  const MachineFrameInfo &MFI = MF.getFrameInfo();
990  uint32_t NumBytes = MFI.getStackSize();
991  uint32_t RoundedSize = FuncInfo->isStackRealigned()
992  ? NumBytes + MFI.getMaxAlign().value()
993  : NumBytes;
994  const Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
995  const Register FramePtrReg = FuncInfo->getFrameOffsetReg();
996  const Register BasePtrReg =
997  TRI.hasBasePointer(MF) ? TRI.getBaseRegister() : Register();
998 
999  Optional<int> FPSaveIndex = FuncInfo->FramePointerSaveIndex;
1000  Optional<int> BPSaveIndex = FuncInfo->BasePointerSaveIndex;
1001 
1002  if (RoundedSize != 0 && hasFP(MF)) {
1003  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_ADD_I32), StackPtrReg)
1004  .addReg(StackPtrReg)
1005  .addImm(-static_cast<int64_t>(RoundedSize * getScratchScaleFactor(ST)))
1007  }
1008 
1009  if (FuncInfo->SGPRForFPSaveRestoreCopy) {
1010  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), FramePtrReg)
1011  .addReg(FuncInfo->SGPRForFPSaveRestoreCopy)
1013  }
1014 
1015  if (FuncInfo->SGPRForBPSaveRestoreCopy) {
1016  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::COPY), BasePtrReg)
1017  .addReg(FuncInfo->SGPRForBPSaveRestoreCopy)
1019  }
1020 
1021  if (FPSaveIndex) {
1022  const int FramePtrFI = *FPSaveIndex;
1023  assert(!MFI.isDeadObjectIndex(FramePtrFI));
1024  if (spilledToMemory(MF, FramePtrFI)) {
1025  initLiveRegs(LiveRegs, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ false);
1026 
1028  MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
1029  if (!TmpVGPR)
1030  report_fatal_error("failed to find free scratch register");
1031  buildEpilogRestore(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, TmpVGPR,
1032  FramePtrFI);
1033  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), FramePtrReg)
1034  .addReg(TmpVGPR, RegState::Kill);
1035  } else {
1036  // Reload from VGPR spill.
1037  assert(MFI.getStackID(FramePtrFI) == TargetStackID::SGPRSpill);
1039  FuncInfo->getSGPRToVGPRSpills(FramePtrFI);
1040  assert(Spill.size() == 1);
1041  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READLANE_B32), FramePtrReg)
1042  .addReg(Spill[0].VGPR)
1043  .addImm(Spill[0].Lane);
1044  }
1045  }
1046 
1047  if (BPSaveIndex) {
1048  const int BasePtrFI = *BPSaveIndex;
1049  assert(!MFI.isDeadObjectIndex(BasePtrFI));
1050  if (spilledToMemory(MF, BasePtrFI)) {
1051  initLiveRegs(LiveRegs, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ false);
1052 
1054  MRI, LiveRegs, AMDGPU::VGPR_32RegClass);
1055  if (!TmpVGPR)
1056  report_fatal_error("failed to find free scratch register");
1057  buildEpilogRestore(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, TmpVGPR,
1058  BasePtrFI);
1059  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), BasePtrReg)
1060  .addReg(TmpVGPR, RegState::Kill);
1061  } else {
1062  // Reload from VGPR spill.
1063  assert(MFI.getStackID(BasePtrFI) == TargetStackID::SGPRSpill);
1065  FuncInfo->getSGPRToVGPRSpills(BasePtrFI);
1066  assert(Spill.size() == 1);
1067  BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READLANE_B32), BasePtrReg)
1068  .addReg(Spill[0].VGPR)
1069  .addImm(Spill[0].Lane);
1070  }
1071  }
1072 
1073  Register ScratchExecCopy;
1075  FuncInfo->getSGPRSpillVGPRs()) {
1076  if (!Reg.FI)
1077  continue;
1078 
1079  if (!ScratchExecCopy)
1080  ScratchExecCopy =
1081  buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, /*IsProlog*/ false);
1082 
1083  buildEpilogRestore(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, Reg.VGPR,
1084  *Reg.FI);
1085  }
1086 
1087  for (const auto &Reg : FuncInfo->WWMReservedRegs) {
1088  auto VGPR = Reg.first;
1089  auto FI = Reg.second;
1090  if (!FI)
1091  continue;
1092 
1093  if (!ScratchExecCopy)
1094  ScratchExecCopy =
1095  buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, /*IsProlog*/ false);
1096 
1097  buildEpilogRestore(ST, TRI, *FuncInfo, LiveRegs, MF, MBB, MBBI, VGPR, *FI);
1098  }
1099 
1100  if (ScratchExecCopy) {
1101  // FIXME: Split block and make terminator.
1102  unsigned ExecMov = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
1103  MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1104  BuildMI(MBB, MBBI, DL, TII->get(ExecMov), Exec)
1105  .addReg(ScratchExecCopy, RegState::Kill);
1106  }
1107 }
1108 
1109 #ifndef NDEBUG
1110 static bool allSGPRSpillsAreDead(const MachineFunction &MF) {
1111  const MachineFrameInfo &MFI = MF.getFrameInfo();
1112  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1113  for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
1114  I != E; ++I) {
1115  if (!MFI.isDeadObjectIndex(I) &&
1117  (I != FuncInfo->FramePointerSaveIndex &&
1118  I != FuncInfo->BasePointerSaveIndex)) {
1119  return false;
1120  }
1121  }
1122 
1123  return true;
1124 }
1125 #endif
1126 
1128  int FI,
1129  Register &FrameReg) const {
1130  const SIRegisterInfo *RI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
1131 
1132  FrameReg = RI->getFrameRegister(MF);
1134 }
1135 
1137  MachineFunction &MF,
1138  RegScavenger *RS) const {
1139  MachineFrameInfo &MFI = MF.getFrameInfo();
1140 
1141  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1142  const SIInstrInfo *TII = ST.getInstrInfo();
1143  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1146 
1147  const bool SpillVGPRToAGPR = ST.hasMAIInsts() && FuncInfo->hasSpilledVGPRs()
1149 
1150  if (SpillVGPRToAGPR) {
1151  // To track the spill frame indices handled in this pass.
1152  BitVector SpillFIs(MFI.getObjectIndexEnd(), false);
1153 
1154  bool SeenDbgInstr = false;
1155 
1156  for (MachineBasicBlock &MBB : MF) {
1158  for (auto I = MBB.begin(), E = MBB.end(); I != E; I = Next) {
1159  MachineInstr &MI = *I;
1160  Next = std::next(I);
1161 
1162  if (MI.isDebugInstr())
1163  SeenDbgInstr = true;
1164 
1165  if (TII->isVGPRSpill(MI)) {
1166  // Try to eliminate stack used by VGPR spills before frame
1167  // finalization.
1168  unsigned FIOp = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
1169  AMDGPU::OpName::vaddr);
1170  int FI = MI.getOperand(FIOp).getIndex();
1171  Register VReg =
1172  TII->getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
1173  if (FuncInfo->allocateVGPRSpillToAGPR(MF, FI,
1174  TRI->isAGPR(MRI, VReg))) {
1175  // FIXME: change to enterBasicBlockEnd()
1176  RS->enterBasicBlock(MBB);
1177  TRI->eliminateFrameIndex(MI, 0, FIOp, RS);
1178  SpillFIs.set(FI);
1179  continue;
1180  }
1181  }
1182  }
1183  }
1184 
1185  for (MachineBasicBlock &MBB : MF) {
1186  for (MCPhysReg Reg : FuncInfo->getVGPRSpillAGPRs())
1187  MBB.addLiveIn(Reg);
1188 
1189  for (MCPhysReg Reg : FuncInfo->getAGPRSpillVGPRs())
1190  MBB.addLiveIn(Reg);
1191 
1193 
1194  if (!SpillFIs.empty() && SeenDbgInstr) {
1195  // FIXME: The dead frame indices are replaced with a null register from
1196  // the debug value instructions. We should instead, update it with the
1197  // correct register value. But not sure the register value alone is
1198  for (MachineInstr &MI : MBB) {
1199  if (MI.isDebugValue() && MI.getOperand(0).isFI() &&
1200  SpillFIs[MI.getOperand(0).getIndex()]) {
1201  MI.getOperand(0).ChangeToRegister(Register(), false /*isDef*/);
1202  MI.getOperand(0).setIsDebug();
1203  }
1204  }
1205  }
1206  }
1207  }
1208 
1209  FuncInfo->removeDeadFrameIndices(MFI);
1211  "SGPR spill should have been removed in SILowerSGPRSpills");
1212 
1213  // FIXME: The other checks should be redundant with allStackObjectsAreDead,
1214  // but currently hasNonSpillStackObjects is set only from source
1215  // allocas. Stack temps produced from legalization are not counted currently.
1216  if (!allStackObjectsAreDead(MFI)) {
1217  assert(RS && "RegScavenger required if spilling");
1218 
1219  // Add an emergency spill slot
1220  RS->addScavengingFrameIndex(FuncInfo->getScavengeFI(MFI, *TRI));
1221  }
1222 }
1223 
1224 // Only report VGPRs to generic code.
1226  BitVector &SavedVGPRs,
1227  RegScavenger *RS) const {
1228  TargetFrameLowering::determineCalleeSaves(MF, SavedVGPRs, RS);
1230  if (MFI->isEntryFunction())
1231  return;
1232 
1233  MachineFrameInfo &FrameInfo = MF.getFrameInfo();
1234  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1235  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1236 
1237  // Ignore the SGPRs the default implementation found.
1238  SavedVGPRs.clearBitsNotInMask(TRI->getAllVectorRegMask());
1239 
1240  // Do not save AGPRs prior to GFX90A because there was no easy way to do so.
1241  // In gfx908 there was do AGPR loads and stores and thus spilling also
1242  // require a temporary VGPR.
1243  if (!ST.hasGFX90AInsts())
1244  SavedVGPRs.clearBitsInMask(TRI->getAllAGPRRegMask());
1245 
1246  // hasFP only knows about stack objects that already exist. We're now
1247  // determining the stack slots that will be created, so we have to predict
1248  // them. Stack objects force FP usage with calls.
1249  //
1250  // Note a new VGPR CSR may be introduced if one is used for the spill, but we
1251  // don't want to report it here.
1252  //
1253  // FIXME: Is this really hasReservedCallFrame?
1254  const bool WillHaveFP =
1255  FrameInfo.hasCalls() &&
1256  (SavedVGPRs.any() || !allStackObjectsAreDead(FrameInfo));
1257 
1258  // VGPRs used for SGPR spilling need to be specially inserted in the prolog,
1259  // so don't allow the default insertion to handle them.
1260  for (auto SSpill : MFI->getSGPRSpillVGPRs())
1261  SavedVGPRs.reset(SSpill.VGPR);
1262 
1263  LivePhysRegs LiveRegs;
1264  LiveRegs.init(*TRI);
1265 
1266  if (WillHaveFP || hasFP(MF)) {
1268  "Re-reserving spill slot for FP");
1270  MFI->FramePointerSaveIndex, true);
1271  }
1272 
1273  if (TRI->hasBasePointer(MF)) {
1274  if (MFI->SGPRForFPSaveRestoreCopy)
1275  LiveRegs.addReg(MFI->SGPRForFPSaveRestoreCopy);
1276 
1278  !MFI->BasePointerSaveIndex && "Re-reserving spill slot for BP");
1280  MFI->BasePointerSaveIndex, false);
1281  }
1282 }
1283 
1285  BitVector &SavedRegs,
1286  RegScavenger *RS) const {
1287  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1289  if (MFI->isEntryFunction())
1290  return;
1291 
1292  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1293  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1294 
1295  // The SP is specifically managed and we don't want extra spills of it.
1296  SavedRegs.reset(MFI->getStackPtrOffsetReg());
1297 
1298  const BitVector AllSavedRegs = SavedRegs;
1299  SavedRegs.clearBitsInMask(TRI->getAllVectorRegMask());
1300 
1301  // If clearing VGPRs changed the mask, we will have some CSR VGPR spills.
1302  const bool HaveAnyCSRVGPR = SavedRegs != AllSavedRegs;
1303 
1304  // We have to anticipate introducing CSR VGPR spills or spill of caller
1305  // save VGPR reserved for SGPR spills as we now always create stack entry
1306  // for it, if we don't have any stack objects already, since we require
1307  // an FP if there is a call and stack.
1308  MachineFrameInfo &FrameInfo = MF.getFrameInfo();
1309  const bool WillHaveFP =
1310  FrameInfo.hasCalls() && (HaveAnyCSRVGPR || MFI->VGPRReservedForSGPRSpill);
1311 
1312  // FP will be specially managed like SP.
1313  if (WillHaveFP || hasFP(MF))
1314  SavedRegs.reset(MFI->getFrameOffsetReg());
1315 }
1316 
1319  std::vector<CalleeSavedInfo> &CSI) const {
1320  if (CSI.empty())
1321  return true; // Early exit if no callee saved registers are modified!
1322 
1323  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1324  if (!FuncInfo->SGPRForFPSaveRestoreCopy &&
1325  !FuncInfo->SGPRForBPSaveRestoreCopy)
1326  return false;
1327 
1328  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1329  const SIRegisterInfo *RI = ST.getRegisterInfo();
1330  Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1331  Register BasePtrReg = RI->getBaseRegister();
1332  unsigned NumModifiedRegs = 0;
1333 
1334  if (FuncInfo->SGPRForFPSaveRestoreCopy)
1335  NumModifiedRegs++;
1336  if (FuncInfo->SGPRForBPSaveRestoreCopy)
1337  NumModifiedRegs++;
1338 
1339  for (auto &CS : CSI) {
1340  if (CS.getReg() == FramePtrReg && FuncInfo->SGPRForFPSaveRestoreCopy) {
1341  CS.setDstReg(FuncInfo->SGPRForFPSaveRestoreCopy);
1342  if (--NumModifiedRegs)
1343  break;
1344  } else if (CS.getReg() == BasePtrReg &&
1345  FuncInfo->SGPRForBPSaveRestoreCopy) {
1346  CS.setDstReg(FuncInfo->SGPRForBPSaveRestoreCopy);
1347  if (--NumModifiedRegs)
1348  break;
1349  }
1350  }
1351 
1352  return false;
1353 }
1354 
1356  MachineFunction &MF,
1359  int64_t Amount = I->getOperand(0).getImm();
1360  if (Amount == 0)
1361  return MBB.erase(I);
1362 
1363  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1364  const SIInstrInfo *TII = ST.getInstrInfo();
1365  const DebugLoc &DL = I->getDebugLoc();
1366  unsigned Opc = I->getOpcode();
1367  bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
1368  uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
1369 
1370  if (!hasReservedCallFrame(MF)) {
1371  Amount = alignTo(Amount, getStackAlign());
1372  assert(isUInt<32>(Amount) && "exceeded stack address space size");
1374  Register SPReg = MFI->getStackPtrOffsetReg();
1375 
1376  Amount *= getScratchScaleFactor(ST);
1377  if (IsDestroy)
1378  Amount = -Amount;
1379  BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), SPReg)
1380  .addReg(SPReg)
1381  .addImm(Amount);
1382  } else if (CalleePopAmount != 0) {
1383  llvm_unreachable("is this used?");
1384  }
1385 
1386  return MBB.erase(I);
1387 }
1388 
1389 /// Returns true if the frame will require a reference to the stack pointer.
1390 ///
1391 /// This is the set of conditions common to setting up the stack pointer in a
1392 /// kernel, and for using a frame pointer in a callable function.
1393 ///
1394 /// FIXME: Should also check hasOpaqueSPAdjustment and if any inline asm
1395 /// references SP.
1397  return MFI.hasVarSizedObjects() || MFI.hasStackMap() || MFI.hasPatchPoint();
1398 }
1399 
1400 // The FP for kernels is always known 0, so we never really need to setup an
1401 // explicit register for it. However, DisableFramePointerElim will force us to
1402 // use a register for it.
1404  const MachineFrameInfo &MFI = MF.getFrameInfo();
1405 
1406  // For entry functions we can use an immediate offset in most cases, so the
1407  // presence of calls doesn't imply we need a distinct frame pointer.
1408  if (MFI.hasCalls() &&
1410  // All offsets are unsigned, so need to be addressed in the same direction
1411  // as stack growth.
1412 
1413  // FIXME: This function is pretty broken, since it can be called before the
1414  // frame layout is determined or CSR spills are inserted.
1415  return MFI.getStackSize() != 0;
1416  }
1417 
1418  return frameTriviallyRequiresSP(MFI) || MFI.isFrameAddressTaken() ||
1419  MF.getSubtarget<GCNSubtarget>().getRegisterInfo()->hasStackRealignment(
1420  MF) ||
1422 }
1423 
1424 // This is essentially a reduced version of hasFP for entry functions. Since the
1425 // stack pointer is known 0 on entry to kernels, we never really need an FP
1426 // register. We may need to initialize the stack pointer depending on the frame
1427 // properties, which logically overlaps many of the cases where an ordinary
1428 // function would require an FP.
1430  const MachineFunction &MF) const {
1431  // Callable functions always require a stack pointer reference.
1433  "only expected to call this for entry points");
1434 
1435  const MachineFrameInfo &MFI = MF.getFrameInfo();
1436 
1437  // Entry points ordinarily don't need to initialize SP. We have to set it up
1438  // for callees if there are any. Also note tail calls are impossible/don't
1439  // make any sense for kernels.
1440  if (MFI.hasCalls())
1441  return true;
1442 
1443  // We still need to initialize the SP if we're doing anything weird that
1444  // references the SP, like variable sized stack objects.
1445  return frameTriviallyRequiresSP(MFI);
1446 }
llvm::MachineRegisterInfo::addLiveIn
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
Definition: MachineRegisterInfo.h:944
i
i
Definition: README.txt:29
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:148
llvm::MachineFrameInfo::hasVarSizedObjects
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
Definition: MachineFrameInfo.h:351
llvm::SIMachineFunctionInfo::setIsStackRealigned
void setIsStackRealigned(bool Realigned=true)
Definition: SIMachineFunctionInfo.h:804
llvm::SIMachineFunctionInfo::getSGPRSpillVGPRs
ArrayRef< SGPRSpillVGPR > getSGPRSpillVGPRs() const
Definition: SIMachineFunctionInfo.h:523
llvm::AMDGPUFunctionArgInfo::FLAT_SCRATCH_INIT
@ FLAT_SCRATCH_INIT
Definition: AMDGPUArgumentUsageInfo.h:105
llvm::SIMachineFunctionInfo::hasSpilledVGPRs
bool hasSpilledVGPRs() const
Definition: SIMachineFunctionInfo.h:784
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:102
llvm::RegState::ImplicitDefine
@ ImplicitDefine
Definition: MachineInstrBuilder.h:63
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
llvm::TargetStackID::WasmLocal
@ WasmLocal
Definition: TargetFrameLowering.h:31
llvm::LivePhysRegs::removeReg
void removeReg(MCPhysReg Reg)
Removes a physical register, all its sub-registers, and all its super-registers from the set.
Definition: LivePhysRegs.h:89
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
llvm::MachineRegisterInfo::isPhysRegUsed
bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
Definition: MachineRegisterInfo.cpp:585
llvm::LivePhysRegs::addReg
void addReg(MCPhysReg Reg)
Adds a physical register and all its sub-registers to the set.
Definition: LivePhysRegs.h:79
llvm::SIMachineFunctionInfo::getPreloadedReg
MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const
Definition: SIMachineFunctionInfo.h:704
llvm::SIFrameLowering::emitEpilogue
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
Definition: SIFrameLowering.cpp:975
SIMachineFunctionInfo.h
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
llvm::Function
Definition: Function.h:61
llvm::BitVector::set
BitVector & set()
Definition: BitVector.h:343
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:52
llvm::AMDGPUSubtarget::GFX9
@ GFX9
Definition: AMDGPUSubtarget.h:40
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:430
llvm::MachineMemOperand::MOInvariant
@ MOInvariant
The memory access always returns the same value (or traps).
Definition: MachineMemOperand.h:145
llvm::RegState::Kill
@ Kill
The last use of a register.
Definition: MachineInstrBuilder.h:48
llvm::SIMachineFunctionInfo::allocateSGPRSpillToVGPR
bool allocateSGPRSpillToVGPR(MachineFunction &MF, int FI)
Reserve a slice of a VGPR to support spilling for FrameIndex FI.
Definition: SIMachineFunctionInfo.cpp:287
llvm::LivePhysRegs
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:48
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:231
llvm::SIMachineFunctionInfo::getGITPtrHigh
unsigned getGITPtrHigh() const
Definition: SIMachineFunctionInfo.h:709
llvm::StackOffset::getFixed
ScalarTy getFixed() const
Definition: TypeSize.h:149
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:128
llvm::MachineMemOperand::MODereferenceable
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
Definition: MachineMemOperand.h:143
llvm::Optional< int >
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
llvm::GCNSubtarget
Definition: GCNSubtarget.h:38
llvm::MachineFrameInfo::RemoveStackObject
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
Definition: MachineFrameInfo.h:742
llvm::LivePhysRegs::empty
bool empty() const
Returns true if the set is empty.
Definition: LivePhysRegs.h:76
llvm::MachineFrameInfo::getObjectIndexEnd
int getObjectIndexEnd() const
Return one past the maximum frame object index.
Definition: MachineFrameInfo.h:391
llvm::SIMachineFunctionInfo::SGPRForFPSaveRestoreCopy
Register SGPRForFPSaveRestoreCopy
If this is set, an SGPR used for save/restore of the register used for the frame pointer.
Definition: SIMachineFunctionInfo.h:494
llvm::AMDGPU::getNamedOperandIdx
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1567
llvm::SIMachineFunctionInfo::getNumPreloadedSGPRs
unsigned getNumPreloadedSGPRs() const
Definition: SIMachineFunctionInfo.h:727
llvm::cl::ReallyHidden
@ ReallyHidden
Definition: CommandLine.h:144
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:122
llvm::SIMachineFunctionInfo::VGPRReservedForSGPRSpill
Register VGPRReservedForSGPRSpill
Definition: SIMachineFunctionInfo.h:502
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::MachineInstr::FrameDestroy
@ FrameDestroy
Definition: MachineInstr.h:84
buildGitPtr
static void buildGitPtr(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, const SIInstrInfo *TII, Register TargetReg)
Definition: SIFrameLowering.cpp:165
llvm::MachineBasicBlock::erase
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
Definition: MachineBasicBlock.cpp:1299
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::SIFrameLowering::hasFP
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register.
Definition: SIFrameLowering.cpp:1403
llvm::SIRegisterInfo::getFrameRegister
Register getFrameRegister(const MachineFunction &MF) const override
Definition: SIRegisterInfo.cpp:387
frameTriviallyRequiresSP
static bool frameTriviallyRequiresSP(const MachineFrameInfo &MFI)
Returns true if the frame will require a reference to the stack pointer.
Definition: SIFrameLowering.cpp:1396
llvm::MachineFunction::front
const MachineBasicBlock & front() const
Definition: MachineFunction.h:816
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:632
TargetMachine.h
GCNSubtarget.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:720
llvm::LivePhysRegs::addLiveIns
void addLiveIns(const MachineBasicBlock &MBB)
Adds all live-in registers of basic block MBB.
Definition: LivePhysRegs.cpp:236
llvm::TargetFrameLowering::getStackAlign
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Definition: TargetFrameLowering.h:100
llvm::TargetFrameLowering::hasReservedCallFrame
virtual bool hasReservedCallFrame(const MachineFunction &MF) const
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
Definition: TargetFrameLowering.h:274
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
llvm::LivePhysRegs::addLiveOuts
void addLiveOuts(const MachineBasicBlock &MBB)
Adds all live-out registers of basic block MBB.
Definition: LivePhysRegs.cpp:230
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:129
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:195
llvm::SIFrameLowering::requiresStackPointerReference
bool requiresStackPointerReference(const MachineFunction &MF) const
Definition: SIFrameLowering.cpp:1429
initLiveRegs
static void initLiveRegs(LivePhysRegs &LiveRegs, const SIRegisterInfo &TRI, const SIMachineFunctionInfo *FuncInfo, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsProlog)
Definition: SIFrameLowering.cpp:682
llvm::MachineInstr::FrameSetup
@ FrameSetup
Definition: MachineInstr.h:82
buildScratchExecCopy
static Register buildScratchExecCopy(LivePhysRegs &LiveRegs, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsProlog)
Definition: SIFrameLowering.cpp:699
llvm::SIMachineFunctionInfo::getStackPtrOffsetReg
Register getStackPtrOffsetReg() const
Definition: SIMachineFunctionInfo.h:764
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::MachineFrameInfo::getStackID
uint8_t getStackID(int ObjectIdx) const
Definition: MachineFrameInfo.h:697
llvm::MachineFrameInfo::getStackSize
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
Definition: MachineFrameInfo.h:551
buildPrologSpill
static void buildPrologSpill(const GCNSubtarget &ST, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &FuncInfo, LivePhysRegs &LiveRegs, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I, Register SpillReg, int FI)
Definition: SIFrameLowering.cpp:124
getVGPRSpillLaneOrTempRegister
static void getVGPRSpillLaneOrTempRegister(MachineFunction &MF, LivePhysRegs &LiveRegs, Register &TempSGPR, Optional< int > &FrameIndex, bool IsFP)
Definition: SIFrameLowering.cpp:59
llvm::MachineFrameInfo::getObjectOffset
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
Definition: MachineFrameInfo.h:492
llvm::MCRegisterInfo::isSubRegisterEq
bool isSubRegisterEq(MCRegister RegA, MCRegister RegB) const
Returns true if RegB is a sub-register of RegA or if RegB == RegA.
Definition: MCRegisterInfo.h:568
llvm::AMDGPU::convertSMRDOffsetUnits
uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, uint64_t ByteOffset)
Convert ByteOffset to dwords if the subtarget uses dword SMRD immediate offsets.
Definition: AMDGPUBaseInfo.cpp:1862
llvm::SIRegisterInfo
Definition: SIRegisterInfo.h:29
llvm::SIMachineFunctionInfo::SGPRForBPSaveRestoreCopy
Register SGPRForBPSaveRestoreCopy
If this is set, an SGPR used for save/restore of the register used for the base pointer.
Definition: SIMachineFunctionInfo.h:499
llvm::BitVector::clearBitsNotInMask
void clearBitsNotInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
clearBitsNotInMask - Clear a bit in this vector for every '0' bit in Mask.
Definition: BitVector.h:705
llvm::BitVector
Definition: BitVector.h:74
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
llvm::MachineFrameInfo::getObjectIndexBegin
int getObjectIndexBegin() const
Return the minimum frame object index.
Definition: MachineFrameInfo.h:388
llvm::MachineInstrBuilder::addExternalSymbol
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:184
llvm::ArrayRef::slice
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:195
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
llvm::MachineFrameInfo::isDeadObjectIndex
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
Definition: MachineFrameInfo.h:711
llvm::SIFrameLowering::assignCalleeSavedSpillSlots
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI) const override
Definition: SIFrameLowering.cpp:1317
llvm::SIFrameLowering::emitPrologue
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
Definition: SIFrameLowering.cpp:735
llvm::BitVector::empty
bool empty() const
empty - Tests whether there are no bits in this bitvector.
Definition: BitVector.h:148
llvm::AMDGPU::Hwreg::WIDTH_M1_SHIFT_
@ WIDTH_M1_SHIFT_
Definition: SIDefines.h:394
llvm::AMDGPUSubtarget::GFX10
@ GFX10
Definition: AMDGPUSubtarget.h:41
llvm::SIMachineFunctionInfo::haveFreeLanesForSGPRSpill
bool haveFreeLanesForSGPRSpill(const MachineFunction &MF, unsigned NumLane) const
returns true if NumLanes slots are available in VGPRs already used for SGPR spilling.
Definition: SIMachineFunctionInfo.cpp:279
llvm::TargetOptions::DisableFramePointerElim
bool DisableFramePointerElim(const MachineFunction &MF) const
DisableFramePointerElim - This returns true if frame pointer elimination optimization should be disab...
Definition: TargetOptionsImpl.cpp:24
llvm::SIFrameLowering::determineCalleeSaves
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
Definition: SIFrameLowering.cpp:1225
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:622
llvm::MachineInstrBuilder::setMIFlag
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
Definition: MachineInstrBuilder.h:278
llvm::cl::opt< bool >
llvm::SIMachineFunctionInfo::WWMReservedRegs
MapVector< Register, Optional< int > > WWMReservedRegs
Definition: SIMachineFunctionInfo.h:470
llvm::LivePhysRegs::stepBackward
void stepBackward(const MachineInstr &MI)
Simulates liveness when stepping backwards over an instruction(bundle).
Definition: LivePhysRegs.cpp:68
llvm::SIMachineFunctionInfo::getVGPRSpillAGPRs
ArrayRef< MCPhysReg > getVGPRSpillAGPRs() const
Definition: SIMachineFunctionInfo.h:537
llvm::AMDGPU::Hwreg::ID_FLAT_SCR_HI
@ ID_FLAT_SCR_HI
Definition: SIDefines.h:369
AMDGPUMCTargetDesc.h
llvm::SIMachineFunctionInfo::isStackRealigned
bool isStackRealigned() const
Definition: SIMachineFunctionInfo.h:800
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
llvm::MachineFrameInfo::getObjectSize
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Definition: MachineFrameInfo.h:451
llvm::Function::getCallingConv
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:238
llvm::BitVector::any
bool any() const
any - Returns true if any bit is set.
Definition: BitVector.h:162
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET
@ PRIVATE_SEGMENT_WAVE_BYTE_OFFSET
Definition: AMDGPUArgumentUsageInfo.h:109
llvm::LivePhysRegs::available
bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const
Returns true if register Reg and no aliasing register is in the set.
Definition: LivePhysRegs.cpp:139
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:38
llvm::MachineRegisterInfo::getCalleeSavedRegs
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
Definition: MachineRegisterInfo.cpp:621
llvm::SIMachineFunctionInfo::FramePointerSaveIndex
Optional< int > FramePointerSaveIndex
Definition: SIMachineFunctionInfo.h:495
allStackObjectsAreDead
static bool allStackObjectsAreDead(const MachineFrameInfo &MFI)
Definition: SIFrameLowering.cpp:328
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::MCPhysReg
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:20
llvm::RegScavenger
Definition: RegisterScavenging.h:34
llvm::MachineFrameInfo::getObjectAlign
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:465
llvm::AMDGPUMachineFunction::isEntryFunction
bool isEntryFunction() const
Definition: AMDGPUMachineFunction.h:78
llvm::TargetRegisterInfo::eliminateFrameIndex
virtual void eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj, unsigned FIOperandNum, RegScavenger *RS=nullptr) const =0
This method must be overriden to eliminate abstract frame indices from instructions which may use the...
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::TargetStackID::ScalableVector
@ ScalableVector
Definition: TargetFrameLowering.h:30
llvm::isUInt< 32 >
constexpr bool isUInt< 32 >(uint64_t x)
Definition: MathExtras.h:411
llvm::MachineRegisterInfo::isAllocatable
bool isAllocatable(MCRegister PhysReg) const
isAllocatable - Returns true when PhysReg belongs to an allocatable register class and it hasn't been...
Definition: MachineRegisterInfo.h:933
llvm::TargetMachine::Options
TargetOptions Options
Definition: TargetMachine.h:115
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:638
llvm::SIMachineFunctionInfo::getScavengeFI
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI)
Definition: SIMachineFunctionInfo.cpp:469
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:225
llvm::RegScavenger::addScavengingFrameIndex
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
Definition: RegisterScavenging.h:123
llvm::MachineInstrBuilder::addMemOperand
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Definition: MachineInstrBuilder.h:202
llvm::SIMachineFunctionInfo::allocateVGPRSpillToAGPR
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR)
Reserve AGPRs or VGPRs to support spilling for FrameIndex FI.
Definition: SIMachineFunctionInfo.cpp:384
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::SIMachineFunctionInfo::getGITPtrLoReg
Register getGITPtrLoReg(const MachineFunction &MF) const
Definition: SIMachineFunctionInfo.cpp:494
llvm::MachineFunction
Definition: MachineFunction.h:230
llvm::MachineBasicBlock::getFirstTerminator
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Definition: MachineBasicBlock.cpp:242
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::MachineFrameInfo::hasPatchPoint
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
Definition: MachineFrameInfo.h:384
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::SIFrameLowering::isSupportedStackID
bool isSupportedStackID(TargetStackID::Value ID) const override
Definition: SIFrameLowering.cpp:669
AMDGPU.h
llvm::LivePhysRegs::init
void init(const TargetRegisterInfo &TRI)
(re-)initializes and clears the set.
Definition: LivePhysRegs.h:66
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: AArch64SLSHardening.cpp:75
llvm::BitVector::clearBitsInMask
void clearBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
clearBitsInMask - Clear any bits in this vector that are set in Mask.
Definition: BitVector.h:693
llvm::TargetStackID::NoAlloc
@ NoAlloc
Definition: TargetFrameLowering.h:32
getScratchScaleFactor
static unsigned getScratchScaleFactor(const GCNSubtarget &ST)
Definition: SIFrameLowering.cpp:391
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::AMDGPU::isCompute
bool isCompute(CallingConv::ID cc)
Definition: AMDGPUBaseInfo.cpp:1377
uint32_t
llvm::StackOffset
StackOffset is a class to represent an offset with 2 dimensions, named fixed and scalable,...
Definition: TypeSize.h:134
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
SIFrameLowering.h
llvm::SIMachineFunctionInfo::getScratchRSrcReg
Register getScratchRSrcReg() const
Returns the physical register reserved for use as the resource descriptor for scratch accesses.
Definition: SIMachineFunctionInfo.h:737
llvm::SIMachineFunctionInfo::getImplicitBufferPtrUserSGPR
Register getImplicitBufferPtrUserSGPR() const
Definition: SIMachineFunctionInfo.h:772
EnableSpillVGPRToAGPR
static cl::opt< bool > EnableSpillVGPRToAGPR("amdgpu-spill-vgpr-to-agpr", cl::desc("Enable spilling VGPRs to AGPRs"), cl::ReallyHidden, cl::init(true))
llvm::SIFrameLowering::processFunctionBeforeFrameFinalized
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS=nullptr) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
Definition: SIFrameLowering.cpp:1136
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:135
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::MachineFrameInfo::getMaxAlign
Align getMaxAlign() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
Definition: MachineFrameInfo.h:567
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::MachineBasicBlock::addLiveIn
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
Definition: MachineBasicBlock.h:367
llvm::MachineFrameInfo::isFrameAddressTaken
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
Definition: MachineFrameInfo.h:366
llvm::RegScavenger::enterBasicBlock
void enterBasicBlock(MachineBasicBlock &MBB)
Start tracking liveness from the begin of basic block MBB.
Definition: RegisterScavenging.cpp:84
llvm::MachineFrameInfo::hasCalls
bool hasCalls() const
Return true if the current function has any function calls.
Definition: MachineFrameInfo.h:579
llvm::ISD::FrameIndex
@ FrameIndex
Definition: ISDOpcodes.h:80
llvm::MachineRegisterInfo::replaceRegWith
void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
Definition: MachineRegisterInfo.cpp:380
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
spilledToMemory
static bool spilledToMemory(const MachineFunction &MF, int SaveIndex)
Definition: SIFrameLowering.cpp:730
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:588
allSGPRSpillsAreDead
static bool allSGPRSpillsAreDead(const MachineFunction &MF)
Definition: SIFrameLowering.cpp:1110
llvm::MachineBasicBlock::sortUniqueLiveIns
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
Definition: MachineBasicBlock.cpp:581
uint16_t
llvm::MachineFunction::getTarget
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Definition: MachineFunction.h:618
llvm::SIFrameLowering::determineCalleeSavesSGPR
void determineCalleeSavesSGPR(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
Definition: SIFrameLowering.cpp:1284
MachineFrameInfo.h
llvm::Align::value
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
llvm::SIMachineFunctionInfo::BasePointerSaveIndex
Optional< int > BasePointerSaveIndex
Definition: SIMachineFunctionInfo.h:500
llvm::SIFrameLowering::eliminateCallFramePseudoInstr
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
Definition: SIFrameLowering.cpp:1355
llvm::AMDGPUAS::CONSTANT_ADDRESS
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
Definition: AMDGPU.h:385
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::TargetStackID::Default
@ Default
Definition: TargetFrameLowering.h:28
llvm::SIMachineFunctionInfo::getAGPRSpillVGPRs
ArrayRef< MCPhysReg > getAGPRSpillVGPRs() const
Definition: SIMachineFunctionInfo.h:533
llvm::SIMachineFunctionInfo::setScratchRSrcReg
void setScratchRSrcReg(Register Reg)
Definition: SIMachineFunctionInfo.h:741
llvm::TargetStackID::SGPRSpill
@ SGPRSpill
Definition: TargetFrameLowering.h:29
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:137
llvm::BitVector::reset
BitVector & reset()
Definition: BitVector.h:384
llvm::SIInstrInfo
Definition: SIInstrInfo.h:38
llvm::TargetRegisterInfo::hasStackRealignment
bool hasStackRealignment(const MachineFunction &MF) const
True if stack realignment is required and still possible.
Definition: TargetRegisterInfo.h:938
llvm::MachineFrameInfo
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Definition: MachineFrameInfo.h:107
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1003
llvm::SIMachineFunctionInfo::removeDeadFrameIndices
void removeDeadFrameIndices(MachineFrameInfo &MFI)
Definition: SIMachineFunctionInfo.cpp:449
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:268
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
llvm::SIMachineFunctionInfo
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Definition: SIMachineFunctionInfo.h:335
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
llvm::SIFrameLowering::emitEntryFunctionPrologue
void emitEntryFunctionPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const
Definition: SIFrameLowering.cpp:395
llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER
@ PRIVATE_SEGMENT_BUFFER
Definition: AMDGPUArgumentUsageInfo.h:100
llvm::TargetRegisterInfo::getSubReg
MCRegister getSubReg(MCRegister Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo.
Definition: TargetRegisterInfo.h:1096
llvm::AMDGPU::Hwreg::ID_FLAT_SCR_LO
@ ID_FLAT_SCR_LO
Definition: SIDefines.h:368
buildEpilogRestore
static void buildEpilogRestore(const GCNSubtarget &ST, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &FuncInfo, LivePhysRegs &LiveRegs, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I, Register SpillReg, int FI)
Definition: SIFrameLowering.cpp:145
llvm::TargetFrameLowering::determineCalleeSaves
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
Definition: TargetFrameLoweringImpl.cpp:78
llvm::MachineFrameInfo::hasStackMap
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
Definition: MachineFrameInfo.h:378
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::cl::desc
Definition: CommandLine.h:414
RegisterScavenging.h
findScratchNonCalleeSaveRegister
static MCRegister findScratchNonCalleeSaveRegister(MachineRegisterInfo &MRI, LivePhysRegs &LiveRegs, const TargetRegisterClass &RC, bool Unused=false)
Definition: SIFrameLowering.cpp:33
llvm::printReg
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Definition: TargetRegisterInfo.cpp:110
llvm::SIMachineFunctionInfo::SGPRSpillVGPR
Definition: SIMachineFunctionInfo.h:452
llvm::MachineInstrBundleIterator
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i....
Definition: MachineInstrBundleIterator.h:108
llvm::SIMachineFunctionInfo::getSGPRToVGPRSpills
ArrayRef< SpilledReg > getSGPRToVGPRSpills(int FrameIndex) const
Definition: SIMachineFunctionInfo.h:517
llvm::SIFrameLowering::getFrameIndexReference
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - This method should return the base register and offset used to reference a f...
Definition: SIFrameLowering.cpp:1127
llvm::CallingConv::AMDGPU_CS
@ AMDGPU_CS
Calling convention used for Mesa/AMDPAL compute shaders.
Definition: CallingConv.h:213
llvm::SIMachineFunctionInfo::hasImplicitBufferPtr
bool hasImplicitBufferPtr() const
Definition: SIMachineFunctionInfo.h:687
llvm::SIMachineFunctionInfo::hasFlatScratchInit
bool hasFlatScratchInit() const
Definition: SIMachineFunctionInfo.h:647
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:270
llvm::SIMachineFunctionInfo::getFrameOffsetReg
Register getFrameOffsetReg() const
Definition: SIMachineFunctionInfo.h:746
llvm::MCRegister
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:23
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38
LivePhysRegs.h