LLVM  15.0.0git
AArch64FrameLowering.cpp
Go to the documentation of this file.
1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of TargetFrameLowering class.
10 //
11 // On AArch64, stack frames are structured as follows:
12 //
13 // The stack grows downward.
14 //
15 // All of the individual frame areas on the frame below are optional, i.e. it's
16 // possible to create a function so that the particular area isn't present
17 // in the frame.
18 //
19 // At function entry, the "frame" looks as follows:
20 //
21 // | | Higher address
22 // |-----------------------------------|
23 // | |
24 // | arguments passed on the stack |
25 // | |
26 // |-----------------------------------| <- sp
27 // | | Lower address
28 //
29 //
30 // After the prologue has run, the frame has the following general structure.
31 // Note that this doesn't depict the case where a red-zone is used. Also,
32 // technically the last frame area (VLAs) doesn't get created until in the
33 // main function body, after the prologue is run. However, it's depicted here
34 // for completeness.
35 //
36 // | | Higher address
37 // |-----------------------------------|
38 // | |
39 // | arguments passed on the stack |
40 // | |
41 // |-----------------------------------|
42 // | |
43 // | (Win64 only) varargs from reg |
44 // | |
45 // |-----------------------------------|
46 // | |
47 // | callee-saved gpr registers | <--.
48 // | | | On Darwin platforms these
49 // |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
50 // | prev_lr | | (frame record first)
51 // | prev_fp | <--'
52 // | async context if needed |
53 // | (a.k.a. "frame record") |
54 // |-----------------------------------| <- fp(=x29)
55 // | |
56 // | callee-saved fp/simd/SVE regs |
57 // | |
58 // |-----------------------------------|
59 // | |
60 // | SVE stack objects |
61 // | |
62 // |-----------------------------------|
63 // |.empty.space.to.make.part.below....|
64 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
65 // |.the.standard.16-byte.alignment....| compile time; if present)
66 // |-----------------------------------|
67 // | |
68 // | local variables of fixed size |
69 // | including spill slots |
70 // |-----------------------------------| <- bp(not defined by ABI,
71 // |.variable-sized.local.variables....| LLVM chooses X19)
72 // |.(VLAs)............................| (size of this area is unknown at
73 // |...................................| compile time)
74 // |-----------------------------------| <- sp
75 // | | Lower address
76 //
77 //
78 // To access the data in a frame, at-compile time, a constant offset must be
79 // computable from one of the pointers (fp, bp, sp) to access it. The size
80 // of the areas with a dotted background cannot be computed at compile-time
81 // if they are present, making it required to have all three of fp, bp and
82 // sp to be set up to be able to access all contents in the frame areas,
83 // assuming all of the frame areas are non-empty.
84 //
85 // For most functions, some of the frame areas are empty. For those functions,
86 // it may not be necessary to set up fp or bp:
87 // * A base pointer is definitely needed when there are both VLAs and local
88 // variables with more-than-default alignment requirements.
89 // * A frame pointer is definitely needed when there are local variables with
90 // more-than-default alignment requirements.
91 //
92 // For Darwin platforms the frame-record (fp, lr) is stored at the top of the
93 // callee-saved area, since the unwind encoding does not allow for encoding
94 // this dynamically and existing tools depend on this layout. For other
95 // platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
96 // area to allow SVE stack objects (allocated directly below the callee-saves,
97 // if available) to be accessed directly from the framepointer.
98 // The SVE spill/fill instructions have VL-scaled addressing modes such
99 // as:
100 // ldr z8, [fp, #-7 mul vl]
101 // For SVE the size of the vector length (VL) is not known at compile-time, so
102 // '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
103 // layout, we don't need to add an unscaled offset to the framepointer before
104 // accessing the SVE object in the frame.
105 //
106 // In some cases when a base pointer is not strictly needed, it is generated
107 // anyway when offsets from the frame pointer to access local variables become
108 // so large that the offset can't be encoded in the immediate fields of loads
109 // or stores.
110 //
111 // Outgoing function arguments must be at the bottom of the stack frame when
112 // calling another function. If we do not have variable-sized stack objects, we
113 // can allocate a "reserved call frame" area at the bottom of the local
114 // variable area, large enough for all outgoing calls. If we do have VLAs, then
115 // the stack pointer must be decremented and incremented around each call to
116 // make space for the arguments below the VLAs.
117 //
118 // FIXME: also explain the redzone concept.
119 //
120 // An example of the prologue:
121 //
122 // .globl __foo
123 // .align 2
124 // __foo:
125 // Ltmp0:
126 // .cfi_startproc
127 // .cfi_personality 155, ___gxx_personality_v0
128 // Leh_func_begin:
129 // .cfi_lsda 16, Lexception33
130 //
131 // stp xa,bx, [sp, -#offset]!
132 // ...
133 // stp x28, x27, [sp, #offset-32]
134 // stp fp, lr, [sp, #offset-16]
135 // add fp, sp, #offset - 16
136 // sub sp, sp, #1360
137 //
138 // The Stack:
139 // +-------------------------------------------+
140 // 10000 | ........ | ........ | ........ | ........ |
141 // 10004 | ........ | ........ | ........ | ........ |
142 // +-------------------------------------------+
143 // 10008 | ........ | ........ | ........ | ........ |
144 // 1000c | ........ | ........ | ........ | ........ |
145 // +===========================================+
146 // 10010 | X28 Register |
147 // 10014 | X28 Register |
148 // +-------------------------------------------+
149 // 10018 | X27 Register |
150 // 1001c | X27 Register |
151 // +===========================================+
152 // 10020 | Frame Pointer |
153 // 10024 | Frame Pointer |
154 // +-------------------------------------------+
155 // 10028 | Link Register |
156 // 1002c | Link Register |
157 // +===========================================+
158 // 10030 | ........ | ........ | ........ | ........ |
159 // 10034 | ........ | ........ | ........ | ........ |
160 // +-------------------------------------------+
161 // 10038 | ........ | ........ | ........ | ........ |
162 // 1003c | ........ | ........ | ........ | ........ |
163 // +-------------------------------------------+
164 //
165 // [sp] = 10030 :: >>initial value<<
166 // sp = 10020 :: stp fp, lr, [sp, #-16]!
167 // fp = sp == 10020 :: mov fp, sp
168 // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
169 // sp == 10010 :: >>final value<<
170 //
171 // The frame pointer (w29) points to address 10020. If we use an offset of
172 // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
173 // for w27, and -32 for w28:
174 //
175 // Ltmp1:
176 // .cfi_def_cfa w29, 16
177 // Ltmp2:
178 // .cfi_offset w30, -8
179 // Ltmp3:
180 // .cfi_offset w29, -16
181 // Ltmp4:
182 // .cfi_offset w27, -24
183 // Ltmp5:
184 // .cfi_offset w28, -32
185 //
186 //===----------------------------------------------------------------------===//
187 
188 #include "AArch64FrameLowering.h"
189 #include "AArch64InstrInfo.h"
191 #include "AArch64RegisterInfo.h"
192 #include "AArch64Subtarget.h"
193 #include "AArch64TargetMachine.h"
195 #include "llvm/ADT/ScopeExit.h"
196 #include "llvm/ADT/SmallVector.h"
197 #include "llvm/ADT/Statistic.h"
213 #include "llvm/IR/Attributes.h"
214 #include "llvm/IR/CallingConv.h"
215 #include "llvm/IR/DataLayout.h"
216 #include "llvm/IR/DebugLoc.h"
217 #include "llvm/IR/Function.h"
218 #include "llvm/MC/MCAsmInfo.h"
219 #include "llvm/MC/MCDwarf.h"
221 #include "llvm/Support/Debug.h"
223 #include "llvm/Support/MathExtras.h"
227 #include <cassert>
228 #include <cstdint>
229 #include <iterator>
230 #include <vector>
231 
232 using namespace llvm;
233 
234 #define DEBUG_TYPE "frame-info"
235 
236 static cl::opt<bool> EnableRedZone("aarch64-redzone",
237  cl::desc("enable use of redzone on AArch64"),
238  cl::init(false), cl::Hidden);
239 
240 static cl::opt<bool>
241  ReverseCSRRestoreSeq("reverse-csr-restore-seq",
242  cl::desc("reverse the CSR restore sequence"),
243  cl::init(false), cl::Hidden);
244 
246  "stack-tagging-merge-settag",
247  cl::desc("merge settag instruction in function epilog"), cl::init(true),
248  cl::Hidden);
249 
250 static cl::opt<bool> OrderFrameObjects("aarch64-order-frame-objects",
251  cl::desc("sort stack allocations"),
252  cl::init(true), cl::Hidden);
253 
255  "homogeneous-prolog-epilog", cl::init(false), cl::ZeroOrMore, cl::Hidden,
256  cl::desc("Emit homogeneous prologue and epilogue for the size "
257  "optimization (default = off)"));
258 
259 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
260 
261 /// Returns how much of the incoming argument stack area (in bytes) we should
262 /// clean up in an epilogue. For the C calling convention this will be 0, for
263 /// guaranteed tail call conventions it can be positive (a normal return or a
264 /// tail call to a function that uses less stack space for arguments) or
265 /// negative (for a tail call to a function that needs more stack space than us
266 /// for arguments).
270  bool IsTailCallReturn = false;
271  if (MBB.end() != MBBI) {
272  unsigned RetOpcode = MBBI->getOpcode();
273  IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
274  RetOpcode == AArch64::TCRETURNri ||
275  RetOpcode == AArch64::TCRETURNriBTI;
276  }
278 
279  int64_t ArgumentPopSize = 0;
280  if (IsTailCallReturn) {
281  MachineOperand &StackAdjust = MBBI->getOperand(1);
282 
283  // For a tail-call in a callee-pops-arguments environment, some or all of
284  // the stack may actually be in use for the call's arguments, this is
285  // calculated during LowerCall and consumed here...
286  ArgumentPopSize = StackAdjust.getImm();
287  } else {
288  // ... otherwise the amount to pop is *all* of the argument space,
289  // conveniently stored in the MachineFunctionInfo by
290  // LowerFormalArguments. This will, of course, be zero for the C calling
291  // convention.
292  ArgumentPopSize = AFI->getArgumentStackToRestore();
293  }
294 
295  return ArgumentPopSize;
296 }
297 
299 static bool needsWinCFI(const MachineFunction &MF);
302 
303 /// Returns true if a homogeneous prolog or epilog code can be emitted
304 /// for the size optimization. If possible, a frame helper call is injected.
305 /// When Exit block is given, this check is for epilog.
306 bool AArch64FrameLowering::homogeneousPrologEpilog(
307  MachineFunction &MF, MachineBasicBlock *Exit) const {
308  if (!MF.getFunction().hasMinSize())
309  return false;
311  return false;
313  return false;
314  if (EnableRedZone)
315  return false;
316 
317  // TODO: Window is supported yet.
318  if (needsWinCFI(MF))
319  return false;
320  // TODO: SVE is not supported yet.
321  if (getSVEStackSize(MF))
322  return false;
323 
324  // Bail on stack adjustment needed on return for simplicity.
325  const MachineFrameInfo &MFI = MF.getFrameInfo();
326  const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
327  if (MFI.hasVarSizedObjects() || RegInfo->hasStackRealignment(MF))
328  return false;
329  if (Exit && getArgumentStackToRestore(MF, *Exit))
330  return false;
331 
332  return true;
333 }
334 
335 /// Returns true if CSRs should be paired.
336 bool AArch64FrameLowering::producePairRegisters(MachineFunction &MF) const {
337  return produceCompactUnwindFrame(MF) || homogeneousPrologEpilog(MF);
338 }
339 
340 /// This is the biggest offset to the stack pointer we can encode in aarch64
341 /// instructions (without using a separate calculation and a temp register).
342 /// Note that the exception here are vector stores/loads which cannot encode any
343 /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
344 static const unsigned DefaultSafeSPDisplacement = 255;
345 
346 /// Look at each instruction that references stack frames and return the stack
347 /// size limit beyond which some of these instructions will require a scratch
348 /// register during their expansion later.
350  // FIXME: For now, just conservatively guestimate based on unscaled indexing
351  // range. We'll end up allocating an unnecessary spill slot a lot, but
352  // realistically that's not a big deal at this stage of the game.
353  for (MachineBasicBlock &MBB : MF) {
354  for (MachineInstr &MI : MBB) {
355  if (MI.isDebugInstr() || MI.isPseudo() ||
356  MI.getOpcode() == AArch64::ADDXri ||
357  MI.getOpcode() == AArch64::ADDSXri)
358  continue;
359 
360  for (const MachineOperand &MO : MI.operands()) {
361  if (!MO.isFI())
362  continue;
363 
364  StackOffset Offset;
365  if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
367  return 0;
368  }
369  }
370  }
372 }
373 
377 }
378 
379 /// Returns the size of the fixed object area (allocated next to sp on entry)
380 /// On Win64 this may include a var args area and an UnwindHelp object for EH.
381 static unsigned getFixedObjectSize(const MachineFunction &MF,
382  const AArch64FunctionInfo *AFI, bool IsWin64,
383  bool IsFunclet) {
384  if (!IsWin64 || IsFunclet) {
385  return AFI->getTailCallReservedStack();
386  } else {
387  if (AFI->getTailCallReservedStack() != 0)
388  report_fatal_error("cannot generate ABI-changing tail call for Win64");
389  // Var args are stored here in the primary function.
390  const unsigned VarArgsArea = AFI->getVarArgsGPRSize();
391  // To support EH funclets we allocate an UnwindHelp object
392  const unsigned UnwindHelpObject = (MF.hasEHFunclets() ? 8 : 0);
393  return alignTo(VarArgsArea + UnwindHelpObject, 16);
394  }
395 }
396 
397 /// Returns the size of the entire SVE stackframe (calleesaves + spills).
400  return StackOffset::getScalable((int64_t)AFI->getStackSizeSVE());
401 }
402 
404  if (!EnableRedZone)
405  return false;
406 
407  // Don't use the red zone if the function explicitly asks us not to.
408  // This is typically used for kernel code.
409  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
410  const unsigned RedZoneSize =
411  Subtarget.getTargetLowering()->getRedZoneSize(MF.getFunction());
412  if (!RedZoneSize)
413  return false;
414 
415  const MachineFrameInfo &MFI = MF.getFrameInfo();
417  uint64_t NumBytes = AFI->getLocalStackSize();
418 
419  return !(MFI.hasCalls() || hasFP(MF) || NumBytes > RedZoneSize ||
420  getSVEStackSize(MF));
421 }
422 
423 /// hasFP - Return true if the specified function should have a dedicated frame
424 /// pointer register.
426  const MachineFrameInfo &MFI = MF.getFrameInfo();
427  const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
428  // Win64 EH requires a frame pointer if funclets are present, as the locals
429  // are accessed off the frame pointer in both the parent function and the
430  // funclets.
431  if (MF.hasEHFunclets())
432  return true;
433  // Retain behavior of always omitting the FP for leaf functions when possible.
435  return true;
436  if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
437  MFI.hasStackMap() || MFI.hasPatchPoint() ||
438  RegInfo->hasStackRealignment(MF))
439  return true;
440  // With large callframes around we may need to use FP to access the scavenging
441  // emergency spillslot.
442  //
443  // Unfortunately some calls to hasFP() like machine verifier ->
444  // getReservedReg() -> hasFP in the middle of global isel are too early
445  // to know the max call frame size. Hopefully conservatively returning "true"
446  // in those cases is fine.
447  // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
448  if (!MFI.isMaxCallFrameSizeComputed() ||
450  return true;
451 
452  return false;
453 }
454 
455 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
456 /// not required, we reserve argument space for call sites in the function
457 /// immediately on entry to the current function. This eliminates the need for
458 /// add/sub sp brackets around call sites. Returns true if the call frame is
459 /// included as part of the stack frame.
460 bool
462  return !MF.getFrameInfo().hasVarSizedObjects();
463 }
464 
468  const AArch64InstrInfo *TII =
469  static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
470  DebugLoc DL = I->getDebugLoc();
471  unsigned Opc = I->getOpcode();
472  bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
473  uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
474 
475  if (!hasReservedCallFrame(MF)) {
476  int64_t Amount = I->getOperand(0).getImm();
477  Amount = alignTo(Amount, getStackAlign());
478  if (!IsDestroy)
479  Amount = -Amount;
480 
481  // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
482  // doesn't have to pop anything), then the first operand will be zero too so
483  // this adjustment is a no-op.
484  if (CalleePopAmount == 0) {
485  // FIXME: in-function stack adjustment for calls is limited to 24-bits
486  // because there's no guaranteed temporary register available.
487  //
488  // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
489  // 1) For offset <= 12-bit, we use LSL #0
490  // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
491  // LSL #0, and the other uses LSL #12.
492  //
493  // Most call frames will be allocated at the start of a function so
494  // this is OK, but it is a limitation that needs dealing with.
495  assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
496  emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
497  StackOffset::getFixed(Amount), TII);
498  }
499  } else if (CalleePopAmount != 0) {
500  // If the calling convention demands that the callee pops arguments from the
501  // stack, we want to add it back if we have a reserved call frame.
502  assert(CalleePopAmount < 0xffffff && "call frame too large");
503  emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
504  StackOffset::getFixed(-(int64_t)CalleePopAmount), TII);
505  }
506  return MBB.erase(I);
507 }
508 
509 void AArch64FrameLowering::emitCalleeSavedGPRLocations(
511  MachineFunction &MF = *MBB.getParent();
512  MachineFrameInfo &MFI = MF.getFrameInfo();
513 
514  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
515  if (CSI.empty())
516  return;
517 
518  const TargetSubtargetInfo &STI = MF.getSubtarget();
519  const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
520  const TargetInstrInfo &TII = *STI.getInstrInfo();
522 
523  for (const auto &Info : CSI) {
524  if (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector)
525  continue;
526 
527  assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
528  unsigned DwarfReg = TRI.getDwarfRegNum(Info.getReg(), true);
529 
530  int64_t Offset =
531  MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
532  unsigned CFIIndex = MF.addFrameInst(
533  MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
534  BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
535  .addCFIIndex(CFIIndex)
537  }
538 }
539 
540 void AArch64FrameLowering::emitCalleeSavedSVELocations(
542  MachineFunction &MF = *MBB.getParent();
543  MachineFrameInfo &MFI = MF.getFrameInfo();
544 
545  // Add callee saved registers to move list.
546  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
547  if (CSI.empty())
548  return;
549 
550  const TargetSubtargetInfo &STI = MF.getSubtarget();
551  const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
552  const TargetInstrInfo &TII = *STI.getInstrInfo();
555 
556  for (const auto &Info : CSI) {
557  if (!(MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
558  continue;
559 
560  // Not all unwinders may know about SVE registers, so assume the lowest
561  // common demoninator.
562  assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
563  unsigned Reg = Info.getReg();
564  if (!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
565  continue;
566 
568  StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) -
570 
571  unsigned CFIIndex = MF.addFrameInst(createCFAOffset(TRI, Reg, Offset));
572  BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
573  .addCFIIndex(CFIIndex)
575  }
576 }
577 
580  emitCalleeSavedGPRLocations(MBB, MBBI);
581  emitCalleeSavedSVELocations(MBB, MBBI);
582 }
583 
584 static void insertCFISameValue(const MCInstrDesc &Desc, MachineFunction &MF,
587  unsigned DwarfReg) {
588  unsigned CFIIndex =
589  MF.addFrameInst(MCCFIInstruction::createSameValue(nullptr, DwarfReg));
590  BuildMI(MBB, InsertPt, DebugLoc(), Desc).addCFIIndex(CFIIndex);
591 }
592 
594  MachineBasicBlock &MBB) const {
595 
596  MachineFunction &MF = *MBB.getParent();
597  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
598  const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
599  const auto &TRI =
600  static_cast<const AArch64RegisterInfo &>(*Subtarget.getRegisterInfo());
601  const auto &MFI = *MF.getInfo<AArch64FunctionInfo>();
602 
603  const MCInstrDesc &CFIDesc = TII.get(TargetOpcode::CFI_INSTRUCTION);
604  DebugLoc DL;
605 
606  // Reset the CFA to `SP + 0`.
608  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
609  nullptr, TRI.getDwarfRegNum(AArch64::SP, true), 0));
610  BuildMI(MBB, InsertPt, DL, CFIDesc).addCFIIndex(CFIIndex);
611 
612  // Flip the RA sign state.
613  if (MFI.shouldSignReturnAddress()) {
614  CFIIndex = MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
615  BuildMI(MBB, InsertPt, DL, CFIDesc).addCFIIndex(CFIIndex);
616  }
617 
618  // Shadow call stack uses X18, reset it.
620  insertCFISameValue(CFIDesc, MF, MBB, InsertPt,
621  TRI.getDwarfRegNum(AArch64::X18, true));
622 
623  // Emit .cfi_same_value for callee-saved registers.
624  const std::vector<CalleeSavedInfo> &CSI =
626  for (const auto &Info : CSI) {
627  unsigned Reg = Info.getReg();
628  if (!TRI.regNeedsCFI(Reg, Reg))
629  continue;
630  insertCFISameValue(CFIDesc, MF, MBB, InsertPt,
631  TRI.getDwarfRegNum(Reg, true));
632  }
633 }
634 
637  bool SVE) {
638  MachineFunction &MF = *MBB.getParent();
639  MachineFrameInfo &MFI = MF.getFrameInfo();
640 
641  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
642  if (CSI.empty())
643  return;
644 
645  const TargetSubtargetInfo &STI = MF.getSubtarget();
646  const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
647  const TargetInstrInfo &TII = *STI.getInstrInfo();
649 
650  for (const auto &Info : CSI) {
651  if (SVE !=
652  (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
653  continue;
654 
655  unsigned Reg = Info.getReg();
656  if (SVE &&
657  !static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
658  continue;
659 
660  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore(
661  nullptr, TRI.getDwarfRegNum(Info.getReg(), true)));
662  BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
663  .addCFIIndex(CFIIndex)
665  }
666 }
667 
668 void AArch64FrameLowering::emitCalleeSavedGPRRestores(
671 }
672 
673 void AArch64FrameLowering::emitCalleeSavedSVERestores(
676 }
677 
678 // Find a scratch register that we can use at the start of the prologue to
679 // re-align the stack pointer. We avoid using callee-save registers since they
680 // may appear to be free when this is called from canUseAsPrologue (during
681 // shrink wrapping), but then no longer be free when this is called from
682 // emitPrologue.
683 //
684 // FIXME: This is a bit conservative, since in the above case we could use one
685 // of the callee-save registers as a scratch temp to re-align the stack pointer,
686 // but we would then have to make sure that we were in fact saving at least one
687 // callee-save register in the prologue, which is additional complexity that
688 // doesn't seem worth the benefit.
690  MachineFunction *MF = MBB->getParent();
691 
692  // If MBB is an entry block, use X9 as the scratch register
693  if (&MF->front() == MBB)
694  return AArch64::X9;
695 
696  const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
697  const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
698  LivePhysRegs LiveRegs(TRI);
699  LiveRegs.addLiveIns(*MBB);
700 
701  // Mark callee saved registers as used so we will not choose them.
702  const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
703  for (unsigned i = 0; CSRegs[i]; ++i)
704  LiveRegs.addReg(CSRegs[i]);
705 
706  // Prefer X9 since it was historically used for the prologue scratch reg.
707  const MachineRegisterInfo &MRI = MF->getRegInfo();
708  if (LiveRegs.available(MRI, AArch64::X9))
709  return AArch64::X9;
710 
711  for (unsigned Reg : AArch64::GPR64RegClass) {
712  if (LiveRegs.available(MRI, Reg))
713  return Reg;
714  }
715  return AArch64::NoRegister;
716 }
717 
719  const MachineBasicBlock &MBB) const {
720  const MachineFunction *MF = MBB.getParent();
721  MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
722  const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
723  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
724 
725  // Don't need a scratch register if we're not going to re-align the stack.
726  if (!RegInfo->hasStackRealignment(*MF))
727  return true;
728  // Otherwise, we can use any block as long as it has a scratch register
729  // available.
730  return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
731 }
732 
734  uint64_t StackSizeInBytes) {
735  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
736  if (!Subtarget.isTargetWindows())
737  return false;
738  const Function &F = MF.getFunction();
739  // TODO: When implementing stack protectors, take that into account
740  // for the probe threshold.
741  unsigned StackProbeSize = 4096;
742  if (F.hasFnAttribute("stack-probe-size"))
743  F.getFnAttribute("stack-probe-size")
744  .getValueAsString()
745  .getAsInteger(0, StackProbeSize);
746  return (StackSizeInBytes >= StackProbeSize) &&
747  !F.hasFnAttribute("no-stack-arg-probe");
748 }
749 
750 static bool needsWinCFI(const MachineFunction &MF) {
751  const Function &F = MF.getFunction();
752  return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
753  F.needsUnwindTableEntry();
754 }
755 
756 bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
757  MachineFunction &MF, uint64_t StackBumpBytes) const {
759  const MachineFrameInfo &MFI = MF.getFrameInfo();
760  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
761  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
762  if (homogeneousPrologEpilog(MF))
763  return false;
764 
765  if (AFI->getLocalStackSize() == 0)
766  return false;
767 
768  // For WinCFI, if optimizing for size, prefer to not combine the stack bump
769  // (to force a stp with predecrement) to match the packed unwind format,
770  // provided that there actually are any callee saved registers to merge the
771  // decrement with.
772  // This is potentially marginally slower, but allows using the packed
773  // unwind format for functions that both have a local area and callee saved
774  // registers. Using the packed unwind format notably reduces the size of
775  // the unwind info.
776  if (needsWinCFI(MF) && AFI->getCalleeSavedStackSize() > 0 &&
777  MF.getFunction().hasOptSize())
778  return false;
779 
780  // 512 is the maximum immediate for stp/ldp that will be used for
781  // callee-save save/restores
782  if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
783  return false;
784 
785  if (MFI.hasVarSizedObjects())
786  return false;
787 
788  if (RegInfo->hasStackRealignment(MF))
789  return false;
790 
791  // This isn't strictly necessary, but it simplifies things a bit since the
792  // current RedZone handling code assumes the SP is adjusted by the
793  // callee-save save/restore code.
794  if (canUseRedZone(MF))
795  return false;
796 
797  // When there is an SVE area on the stack, always allocate the
798  // callee-saves and spills/locals separately.
799  if (getSVEStackSize(MF))
800  return false;
801 
802  return true;
803 }
804 
805 bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
806  MachineBasicBlock &MBB, unsigned StackBumpBytes) const {
807  if (!shouldCombineCSRLocalStackBump(*MBB.getParent(), StackBumpBytes))
808  return false;
809 
810  if (MBB.empty())
811  return true;
812 
813  // Disable combined SP bump if the last instruction is an MTE tag store. It
814  // is almost always better to merge SP adjustment into those instructions.
817  while (LastI != Begin) {
818  --LastI;
819  if (LastI->isTransient())
820  continue;
821  if (!LastI->getFlag(MachineInstr::FrameDestroy))
822  break;
823  }
824  switch (LastI->getOpcode()) {
825  case AArch64::STGloop:
826  case AArch64::STZGloop:
827  case AArch64::STGOffset:
828  case AArch64::STZGOffset:
829  case AArch64::ST2GOffset:
830  case AArch64::STZ2GOffset:
831  return false;
832  default:
833  return true;
834  }
835  llvm_unreachable("unreachable");
836 }
837 
838 // Given a load or a store instruction, generate an appropriate unwinding SEH
839 // code on Windows.
841  const TargetInstrInfo &TII,
843  unsigned Opc = MBBI->getOpcode();
845  MachineFunction &MF = *MBB->getParent();
846  DebugLoc DL = MBBI->getDebugLoc();
847  unsigned ImmIdx = MBBI->getNumOperands() - 1;
848  int Imm = MBBI->getOperand(ImmIdx).getImm();
850  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
851  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
852 
853  switch (Opc) {
854  default:
855  llvm_unreachable("No SEH Opcode for this instruction");
856  case AArch64::LDPDpost:
857  Imm = -Imm;
859  case AArch64::STPDpre: {
860  unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
861  unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
862  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
863  .addImm(Reg0)
864  .addImm(Reg1)
865  .addImm(Imm * 8)
866  .setMIFlag(Flag);
867  break;
868  }
869  case AArch64::LDPXpost:
870  Imm = -Imm;
872  case AArch64::STPXpre: {
873  Register Reg0 = MBBI->getOperand(1).getReg();
874  Register Reg1 = MBBI->getOperand(2).getReg();
875  if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
876  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
877  .addImm(Imm * 8)
878  .setMIFlag(Flag);
879  else
880  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
881  .addImm(RegInfo->getSEHRegNum(Reg0))
882  .addImm(RegInfo->getSEHRegNum(Reg1))
883  .addImm(Imm * 8)
884  .setMIFlag(Flag);
885  break;
886  }
887  case AArch64::LDRDpost:
888  Imm = -Imm;
890  case AArch64::STRDpre: {
891  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
892  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
893  .addImm(Reg)
894  .addImm(Imm)
895  .setMIFlag(Flag);
896  break;
897  }
898  case AArch64::LDRXpost:
899  Imm = -Imm;
901  case AArch64::STRXpre: {
902  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
903  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
904  .addImm(Reg)
905  .addImm(Imm)
906  .setMIFlag(Flag);
907  break;
908  }
909  case AArch64::STPDi:
910  case AArch64::LDPDi: {
911  unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
912  unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
913  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
914  .addImm(Reg0)
915  .addImm(Reg1)
916  .addImm(Imm * 8)
917  .setMIFlag(Flag);
918  break;
919  }
920  case AArch64::STPXi:
921  case AArch64::LDPXi: {
922  Register Reg0 = MBBI->getOperand(0).getReg();
923  Register Reg1 = MBBI->getOperand(1).getReg();
924  if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
925  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
926  .addImm(Imm * 8)
927  .setMIFlag(Flag);
928  else
929  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
930  .addImm(RegInfo->getSEHRegNum(Reg0))
931  .addImm(RegInfo->getSEHRegNum(Reg1))
932  .addImm(Imm * 8)
933  .setMIFlag(Flag);
934  break;
935  }
936  case AArch64::STRXui:
937  case AArch64::LDRXui: {
938  int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
939  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
940  .addImm(Reg)
941  .addImm(Imm * 8)
942  .setMIFlag(Flag);
943  break;
944  }
945  case AArch64::STRDui:
946  case AArch64::LDRDui: {
947  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
948  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
949  .addImm(Reg)
950  .addImm(Imm * 8)
951  .setMIFlag(Flag);
952  break;
953  }
954  }
955  auto I = MBB->insertAfter(MBBI, MIB);
956  return I;
957 }
958 
959 // Fix up the SEH opcode associated with the save/restore instruction.
961  unsigned LocalStackSize) {
962  MachineOperand *ImmOpnd = nullptr;
963  unsigned ImmIdx = MBBI->getNumOperands() - 1;
964  switch (MBBI->getOpcode()) {
965  default:
966  llvm_unreachable("Fix the offset in the SEH instruction");
967  case AArch64::SEH_SaveFPLR:
968  case AArch64::SEH_SaveRegP:
969  case AArch64::SEH_SaveReg:
970  case AArch64::SEH_SaveFRegP:
971  case AArch64::SEH_SaveFReg:
972  ImmOpnd = &MBBI->getOperand(ImmIdx);
973  break;
974  }
975  if (ImmOpnd)
976  ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
977 }
978 
979 // Convert callee-save register save/restore instruction to do stack pointer
980 // decrement/increment to allocate/deallocate the callee-save stack area by
981 // converting store/load to use pre/post increment version.
984  const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
985  bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI,
987  int CFAOffset = 0) {
988  unsigned NewOpc;
989  switch (MBBI->getOpcode()) {
990  default:
991  llvm_unreachable("Unexpected callee-save save/restore opcode!");
992  case AArch64::STPXi:
993  NewOpc = AArch64::STPXpre;
994  break;
995  case AArch64::STPDi:
996  NewOpc = AArch64::STPDpre;
997  break;
998  case AArch64::STPQi:
999  NewOpc = AArch64::STPQpre;
1000  break;
1001  case AArch64::STRXui:
1002  NewOpc = AArch64::STRXpre;
1003  break;
1004  case AArch64::STRDui:
1005  NewOpc = AArch64::STRDpre;
1006  break;
1007  case AArch64::STRQui:
1008  NewOpc = AArch64::STRQpre;
1009  break;
1010  case AArch64::LDPXi:
1011  NewOpc = AArch64::LDPXpost;
1012  break;
1013  case AArch64::LDPDi:
1014  NewOpc = AArch64::LDPDpost;
1015  break;
1016  case AArch64::LDPQi:
1017  NewOpc = AArch64::LDPQpost;
1018  break;
1019  case AArch64::LDRXui:
1020  NewOpc = AArch64::LDRXpost;
1021  break;
1022  case AArch64::LDRDui:
1023  NewOpc = AArch64::LDRDpost;
1024  break;
1025  case AArch64::LDRQui:
1026  NewOpc = AArch64::LDRQpost;
1027  break;
1028  }
1029  // Get rid of the SEH code associated with the old instruction.
1030  if (NeedsWinCFI) {
1031  auto SEH = std::next(MBBI);
1033  SEH->eraseFromParent();
1034  }
1035 
1036  TypeSize Scale = TypeSize::Fixed(1);
1037  unsigned Width;
1038  int64_t MinOffset, MaxOffset;
1039  bool Success = static_cast<const AArch64InstrInfo *>(TII)->getMemOpInfo(
1040  NewOpc, Scale, Width, MinOffset, MaxOffset);
1041  (void)Success;
1042  assert(Success && "unknown load/store opcode");
1043 
1044  // If the first store isn't right where we want SP then we can't fold the
1045  // update in so create a normal arithmetic instruction instead.
1046  MachineFunction &MF = *MBB.getParent();
1047  if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0 ||
1048  CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) {
1049  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1050  StackOffset::getFixed(CSStackSizeInc), TII, FrameFlag,
1051  false, false, nullptr, EmitCFI,
1052  StackOffset::getFixed(CFAOffset));
1053 
1054  return std::prev(MBBI);
1055  }
1056 
1057  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
1058  MIB.addReg(AArch64::SP, RegState::Define);
1059 
1060  // Copy all operands other than the immediate offset.
1061  unsigned OpndIdx = 0;
1062  for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
1063  ++OpndIdx)
1064  MIB.add(MBBI->getOperand(OpndIdx));
1065 
1066  assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
1067  "Unexpected immediate offset in first/last callee-save save/restore "
1068  "instruction!");
1069  assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
1070  "Unexpected base register in callee-save save/restore instruction!");
1071  assert(CSStackSizeInc % Scale == 0);
1072  MIB.addImm(CSStackSizeInc / (int)Scale);
1073 
1074  MIB.setMIFlags(MBBI->getFlags());
1075  MIB.setMemRefs(MBBI->memoperands());
1076 
1077  // Generate a new SEH code that corresponds to the new instruction.
1078  if (NeedsWinCFI) {
1079  *HasWinCFI = true;
1080  InsertSEH(*MIB, *TII, FrameFlag);
1081  }
1082 
1083  if (EmitCFI) {
1084  unsigned CFIIndex = MF.addFrameInst(
1085  MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset - CSStackSizeInc));
1086  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1087  .addCFIIndex(CFIIndex)
1088  .setMIFlags(FrameFlag);
1089  }
1090 
1091  return std::prev(MBB.erase(MBBI));
1092 }
1093 
1094 // Fixup callee-save register save/restore instructions to take into account
1095 // combined SP bump by adding the local stack size to the stack offsets.
1097  uint64_t LocalStackSize,
1098  bool NeedsWinCFI,
1099  bool *HasWinCFI) {
1101  return;
1102 
1103  unsigned Opc = MI.getOpcode();
1104  unsigned Scale;
1105  switch (Opc) {
1106  case AArch64::STPXi:
1107  case AArch64::STRXui:
1108  case AArch64::STPDi:
1109  case AArch64::STRDui:
1110  case AArch64::LDPXi:
1111  case AArch64::LDRXui:
1112  case AArch64::LDPDi:
1113  case AArch64::LDRDui:
1114  Scale = 8;
1115  break;
1116  case AArch64::STPQi:
1117  case AArch64::STRQui:
1118  case AArch64::LDPQi:
1119  case AArch64::LDRQui:
1120  Scale = 16;
1121  break;
1122  default:
1123  llvm_unreachable("Unexpected callee-save save/restore opcode!");
1124  }
1125 
1126  unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
1127  assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
1128  "Unexpected base register in callee-save save/restore instruction!");
1129  // Last operand is immediate offset that needs fixing.
1130  MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
1131  // All generated opcodes have scaled offsets.
1132  assert(LocalStackSize % Scale == 0);
1133  OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
1134 
1135  if (NeedsWinCFI) {
1136  *HasWinCFI = true;
1137  auto MBBI = std::next(MachineBasicBlock::iterator(MI));
1138  assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
1140  "Expecting a SEH instruction");
1141  fixupSEHOpcode(MBBI, LocalStackSize);
1142  }
1143 }
1144 
1145 static bool isTargetWindows(const MachineFunction &MF) {
1147 }
1148 
1149 // Convenience function to determine whether I is an SVE callee save.
1151  switch (I->getOpcode()) {
1152  default:
1153  return false;
1154  case AArch64::STR_ZXI:
1155  case AArch64::STR_PXI:
1156  case AArch64::LDR_ZXI:
1157  case AArch64::LDR_PXI:
1158  return I->getFlag(MachineInstr::FrameSetup) ||
1159  I->getFlag(MachineInstr::FrameDestroy);
1160  }
1161 }
1162 
1164  if (!(llvm::any_of(
1166  [](const auto &Info) { return Info.getReg() == AArch64::LR; }) &&
1167  MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)))
1168  return false;
1169 
1171  report_fatal_error("Must reserve x18 to use shadow call stack");
1172 
1173  return true;
1174 }
1175 
1177  MachineFunction &MF,
1180  const DebugLoc &DL, bool NeedsWinCFI,
1181  bool NeedsUnwindInfo) {
1182  // Shadow call stack prolog: str x30, [x18], #8
1183  BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXpost))
1184  .addReg(AArch64::X18, RegState::Define)
1185  .addReg(AArch64::LR)
1186  .addReg(AArch64::X18)
1187  .addImm(8)
1189 
1190  // This instruction also makes x18 live-in to the entry block.
1191  MBB.addLiveIn(AArch64::X18);
1192 
1193  if (NeedsWinCFI)
1194  BuildMI(MBB, MBBI, DL, TII.get(AArch64::SEH_Nop))
1196 
1197  if (NeedsUnwindInfo) {
1198  // Emit a CFI instruction that causes 8 to be subtracted from the value of
1199  // x18 when unwinding past this frame.
1200  static const char CFIInst[] = {
1201  dwarf::DW_CFA_val_expression,
1202  18, // register
1203  2, // length
1204  static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
1205  static_cast<char>(-8) & 0x7f, // addend (sleb128)
1206  };
1207  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
1208  nullptr, StringRef(CFIInst, sizeof(CFIInst))));
1209  BuildMI(MBB, MBBI, DL, TII.get(AArch64::CFI_INSTRUCTION))
1210  .addCFIIndex(CFIIndex)
1212  }
1213 }
1214 
1216  MachineFunction &MF,
1219  const DebugLoc &DL) {
1220  // Shadow call stack epilog: ldr x30, [x18, #-8]!
1221  BuildMI(MBB, MBBI, DL, TII.get(AArch64::LDRXpre))
1222  .addReg(AArch64::X18, RegState::Define)
1223  .addReg(AArch64::LR, RegState::Define)
1224  .addReg(AArch64::X18)
1225  .addImm(-8)
1227 
1229  unsigned CFIIndex =
1231  BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
1232  .addCFIIndex(CFIIndex)
1234  }
1235 }
1236 
1238  MachineBasicBlock &MBB) const {
1240  const MachineFrameInfo &MFI = MF.getFrameInfo();
1241  const Function &F = MF.getFunction();
1242  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1243  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1244  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1245  MachineModuleInfo &MMI = MF.getMMI();
1247  bool EmitCFI = AFI->needsDwarfUnwindInfo();
1248  bool HasFP = hasFP(MF);
1249  bool NeedsWinCFI = needsWinCFI(MF);
1250  bool HasWinCFI = false;
1251  auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); });
1252 
1253  bool IsFunclet = MBB.isEHFuncletEntry();
1254 
1255  // At this point, we're going to decide whether or not the function uses a
1256  // redzone. In most cases, the function doesn't have a redzone so let's
1257  // assume that's false and set it to true in the case that there's a redzone.
1258  AFI->setHasRedZone(false);
1259 
1260  // Debug location must be unknown since the first debug location is used
1261  // to determine the end of the prologue.
1262  DebugLoc DL;
1263 
1264  const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
1266  emitShadowCallStackPrologue(*TII, MF, MBB, MBBI, DL, NeedsWinCFI,
1267  MFnI.needsDwarfUnwindInfo());
1268 
1269  if (MFnI.shouldSignReturnAddress()) {
1270  unsigned PACI;
1271  if (MFnI.shouldSignWithBKey()) {
1272  BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
1274  PACI = Subtarget.hasPAuth() ? AArch64::PACIB : AArch64::PACIBSP;
1275  } else {
1276  PACI = Subtarget.hasPAuth() ? AArch64::PACIA : AArch64::PACIASP;
1277  }
1278 
1279  auto MI = BuildMI(MBB, MBBI, DL, TII->get(PACI));
1280  if (Subtarget.hasPAuth())
1281  MI.addReg(AArch64::LR, RegState::Define)
1282  .addReg(AArch64::LR)
1283  .addReg(AArch64::SP, RegState::InternalRead);
1284  MI.setMIFlag(MachineInstr::FrameSetup);
1285  if (EmitCFI) {
1286  unsigned CFIIndex =
1288  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1289  .addCFIIndex(CFIIndex)
1291  }
1292  }
1293 
1294  // We signal the presence of a Swift extended frame to external tools by
1295  // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple
1296  // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI
1297  // bits so that is still true.
1298  if (HasFP && AFI->hasSwiftAsyncContext()) {
1299  switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
1301  if (Subtarget.swiftAsyncContextIsDynamicallySet()) {
1302  // The special symbol below is absolute and has a *value* that can be
1303  // combined with the frame pointer to signal an extended frame.
1304  BuildMI(MBB, MBBI, DL, TII->get(AArch64::LOADgot), AArch64::X16)
1305  .addExternalSymbol("swift_async_extendedFramePointerFlags",
1307  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::FP)
1308  .addUse(AArch64::FP)
1309  .addUse(AArch64::X16)
1310  .addImm(Subtarget.isTargetILP32() ? 32 : 0);
1311  break;
1312  }
1314 
1316  // ORR x29, x29, #0x1000_0000_0000_0000
1317  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP)
1318  .addUse(AArch64::FP)
1319  .addImm(0x1100)
1321  break;
1322 
1324  break;
1325  }
1326  }
1327 
1328  // All calls are tail calls in GHC calling conv, and functions have no
1329  // prologue/epilogue.
1331  return;
1332 
1333  // Set tagged base pointer to the requested stack slot.
1334  // Ideally it should match SP value after prologue.
1336  if (TBPI)
1337  AFI->setTaggedBasePointerOffset(-MFI.getObjectOffset(*TBPI));
1338  else
1340 
1341  const StackOffset &SVEStackSize = getSVEStackSize(MF);
1342 
1343  // getStackSize() includes all the locals in its size calculation. We don't
1344  // include these locals when computing the stack size of a funclet, as they
1345  // are allocated in the parent's stack frame and accessed via the frame
1346  // pointer from the funclet. We only save the callee saved registers in the
1347  // funclet, which are really the callee saved registers of the parent
1348  // function, including the funclet.
1349  int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1350  : MFI.getStackSize();
1351  if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
1352  assert(!HasFP && "unexpected function without stack frame but with FP");
1353  assert(!SVEStackSize &&
1354  "unexpected function without stack frame but with SVE objects");
1355  // All of the stack allocation is for locals.
1356  AFI->setLocalStackSize(NumBytes);
1357  if (!NumBytes)
1358  return;
1359  // REDZONE: If the stack size is less than 128 bytes, we don't need
1360  // to actually allocate.
1361  if (canUseRedZone(MF)) {
1362  AFI->setHasRedZone(true);
1363  ++NumRedZoneFunctions;
1364  } else {
1365  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1366  StackOffset::getFixed(-NumBytes), TII,
1367  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
1368  if (EmitCFI) {
1369  // Label used to tie together the PROLOG_LABEL and the MachineMoves.
1370  MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
1371  // Encode the stack size of the leaf function.
1372  unsigned CFIIndex = MF.addFrameInst(
1373  MCCFIInstruction::cfiDefCfaOffset(FrameLabel, NumBytes));
1374  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1375  .addCFIIndex(CFIIndex)
1377  }
1378  }
1379 
1380  if (NeedsWinCFI) {
1381  HasWinCFI = true;
1382  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1384  }
1385 
1386  return;
1387  }
1388 
1389  bool IsWin64 =
1390  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1391  unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1392 
1393  auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1394  // All of the remaining stack allocations are for locals.
1395  AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1396  bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1397  bool HomPrologEpilog = homogeneousPrologEpilog(MF);
1398  if (CombineSPBump) {
1399  assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1400  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1401  StackOffset::getFixed(-NumBytes), TII,
1402  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI,
1403  EmitCFI);
1404  NumBytes = 0;
1405  } else if (HomPrologEpilog) {
1406  // Stack has been already adjusted.
1407  NumBytes -= PrologueSaveSize;
1408  } else if (PrologueSaveSize != 0) {
1410  MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI,
1411  EmitCFI);
1412  NumBytes -= PrologueSaveSize;
1413  }
1414  assert(NumBytes >= 0 && "Negative stack allocation size!?");
1415 
1416  // Move past the saves of the callee-saved registers, fixing up the offsets
1417  // and pre-inc if we decided to combine the callee-save and local stack
1418  // pointer bump above.
1420  while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) &&
1421  !IsSVECalleeSave(MBBI)) {
1422  if (CombineSPBump)
1424  NeedsWinCFI, &HasWinCFI);
1425  ++MBBI;
1426  }
1427 
1428  // For funclets the FP belongs to the containing function.
1429  if (!IsFunclet && HasFP) {
1430  // Only set up FP if we actually need to.
1431  int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset();
1432 
1433  if (CombineSPBump)
1434  FPOffset += AFI->getLocalStackSize();
1435 
1436  if (AFI->hasSwiftAsyncContext()) {
1437  // Before we update the live FP we have to ensure there's a valid (or
1438  // null) asynchronous context in its slot just before FP in the frame
1439  // record, so store it now.
1440  const auto &Attrs = MF.getFunction().getAttributes();
1441  bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync);
1442  if (HaveInitialContext)
1443  MBB.addLiveIn(AArch64::X22);
1444  BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext))
1445  .addUse(HaveInitialContext ? AArch64::X22 : AArch64::XZR)
1446  .addUse(AArch64::SP)
1447  .addImm(FPOffset - 8)
1449  }
1450 
1451  if (HomPrologEpilog) {
1452  auto Prolog = MBBI;
1453  --Prolog;
1454  assert(Prolog->getOpcode() == AArch64::HOM_Prolog);
1455  Prolog->addOperand(MachineOperand::CreateImm(FPOffset));
1456  } else {
1457  // Issue sub fp, sp, FPOffset or
1458  // mov fp,sp when FPOffset is zero.
1459  // Note: All stores of callee-saved registers are marked as "FrameSetup".
1460  // This code marks the instruction(s) that set the FP also.
1461  emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP,
1462  StackOffset::getFixed(FPOffset), TII,
1463  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
1464  }
1465  if (EmitCFI) {
1466  // Define the current CFA rule to use the provided FP.
1467  const int OffsetToFirstCalleeSaveFromFP =
1469  AFI->getCalleeSavedStackSize();
1470  Register FramePtr = RegInfo->getFrameRegister(MF);
1471  unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
1472  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
1473  nullptr, Reg, FixedObject - OffsetToFirstCalleeSaveFromFP));
1474  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1475  .addCFIIndex(CFIIndex)
1477  }
1478  }
1479 
1480  // Now emit the moves for whatever callee saved regs we have (including FP,
1481  // LR if those are saved). Frame instructions for SVE register are emitted
1482  // later, after the instruction which actually save SVE regs.
1483  if (EmitCFI)
1484  emitCalleeSavedGPRLocations(MBB, MBBI);
1485 
1486  if (windowsRequiresStackProbe(MF, NumBytes)) {
1487  uint64_t NumWords = NumBytes >> 4;
1488  if (NeedsWinCFI) {
1489  HasWinCFI = true;
1490  // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
1491  // exceed this amount. We need to move at most 2^24 - 1 into x15.
1492  // This is at most two instructions, MOVZ follwed by MOVK.
1493  // TODO: Fix to use multiple stack alloc unwind codes for stacks
1494  // exceeding 256MB in size.
1495  if (NumBytes >= (1 << 28))
1496  report_fatal_error("Stack size cannot exceed 256MB for stack "
1497  "unwinding purposes");
1498 
1499  uint32_t LowNumWords = NumWords & 0xFFFF;
1500  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
1501  .addImm(LowNumWords)
1504  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1506  if ((NumWords & 0xFFFF0000) != 0) {
1507  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
1508  .addReg(AArch64::X15)
1509  .addImm((NumWords & 0xFFFF0000) >> 16) // High half
1512  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1514  }
1515  } else {
1516  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
1517  .addImm(NumWords)
1519  }
1520 
1521  switch (MF.getTarget().getCodeModel()) {
1522  case CodeModel::Tiny:
1523  case CodeModel::Small:
1524  case CodeModel::Medium:
1525  case CodeModel::Kernel:
1526  BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
1527  .addExternalSymbol("__chkstk")
1528  .addReg(AArch64::X15, RegState::Implicit)
1533  if (NeedsWinCFI) {
1534  HasWinCFI = true;
1535  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1537  }
1538  break;
1539  case CodeModel::Large:
1540  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
1541  .addReg(AArch64::X16, RegState::Define)
1542  .addExternalSymbol("__chkstk")
1543  .addExternalSymbol("__chkstk")
1545  if (NeedsWinCFI) {
1546  HasWinCFI = true;
1547  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1549  }
1550 
1551  BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF)))
1552  .addReg(AArch64::X16, RegState::Kill)
1553  .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
1558  if (NeedsWinCFI) {
1559  HasWinCFI = true;
1560  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1562  }
1563  break;
1564  }
1565 
1566  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
1567  .addReg(AArch64::SP, RegState::Kill)
1568  .addReg(AArch64::X15, RegState::Kill)
1571  if (NeedsWinCFI) {
1572  HasWinCFI = true;
1573  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1574  .addImm(NumBytes)
1576  }
1577  NumBytes = 0;
1578  }
1579 
1580  StackOffset AllocateBefore = SVEStackSize, AllocateAfter = {};
1581  MachineBasicBlock::iterator CalleeSavesBegin = MBBI, CalleeSavesEnd = MBBI;
1582 
1583  // Process the SVE callee-saves to determine what space needs to be
1584  // allocated.
1585  if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
1586  // Find callee save instructions in frame.
1587  CalleeSavesBegin = MBBI;
1588  assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
1590  ++MBBI;
1591  CalleeSavesEnd = MBBI;
1592 
1593  AllocateBefore = StackOffset::getScalable(CalleeSavedSize);
1594  AllocateAfter = SVEStackSize - AllocateBefore;
1595  }
1596 
1597  // Allocate space for the callee saves (if any).
1599  MBB, CalleeSavesBegin, DL, AArch64::SP, AArch64::SP, -AllocateBefore, TII,
1600  MachineInstr::FrameSetup, false, false, nullptr,
1601  EmitCFI && !HasFP && AllocateBefore,
1602  StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes));
1603 
1604  if (EmitCFI)
1605  emitCalleeSavedSVELocations(MBB, CalleeSavesEnd);
1606 
1607  // Finally allocate remaining SVE stack space.
1608  emitFrameOffset(MBB, CalleeSavesEnd, DL, AArch64::SP, AArch64::SP,
1609  -AllocateAfter, TII, MachineInstr::FrameSetup, false, false,
1610  nullptr, EmitCFI && !HasFP && AllocateAfter,
1611  AllocateBefore + StackOffset::getFixed(
1612  (int64_t)MFI.getStackSize() - NumBytes));
1613 
1614  // Allocate space for the rest of the frame.
1615  if (NumBytes) {
1616  // Alignment is required for the parent frame, not the funclet
1617  const bool NeedsRealignment =
1618  !IsFunclet && RegInfo->hasStackRealignment(MF);
1619  unsigned scratchSPReg = AArch64::SP;
1620 
1621  if (NeedsRealignment) {
1622  scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
1623  assert(scratchSPReg != AArch64::NoRegister);
1624  }
1625 
1626  // If we're a leaf function, try using the red zone.
1627  if (!canUseRedZone(MF)) {
1628  // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
1629  // the correct value here, as NumBytes also includes padding bytes,
1630  // which shouldn't be counted here.
1632  MBB, MBBI, DL, scratchSPReg, AArch64::SP,
1634  false, NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP,
1635  SVEStackSize +
1636  StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes));
1637  }
1638  if (NeedsRealignment) {
1639  const unsigned NrBitsToZero = Log2(MFI.getMaxAlign());
1640  assert(NrBitsToZero > 1);
1641  assert(scratchSPReg != AArch64::SP);
1642 
1643  // SUB X9, SP, NumBytes
1644  // -- X9 is temporary register, so shouldn't contain any live data here,
1645  // -- free to use. This is already produced by emitFrameOffset above.
1646  // AND SP, X9, 0b11111...0000
1647  // The logical immediates have a non-trivial encoding. The following
1648  // formula computes the encoded immediate with all ones but
1649  // NrBitsToZero zero bits as least significant bits.
1650  uint32_t andMaskEncoded = (1 << 12) // = N
1651  | ((64 - NrBitsToZero) << 6) // immr
1652  | ((64 - NrBitsToZero - 1) << 0); // imms
1653 
1654  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
1655  .addReg(scratchSPReg, RegState::Kill)
1656  .addImm(andMaskEncoded);
1657  AFI->setStackRealigned(true);
1658  if (NeedsWinCFI) {
1659  HasWinCFI = true;
1660  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1661  .addImm(NumBytes & andMaskEncoded)
1663  }
1664  }
1665  }
1666 
1667  // If we need a base pointer, set it up here. It's whatever the value of the
1668  // stack pointer is at this point. Any variable size objects will be allocated
1669  // after this, so we can still use the base pointer to reference locals.
1670  //
1671  // FIXME: Clarify FrameSetup flags here.
1672  // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
1673  // needed.
1674  // For funclets the BP belongs to the containing function.
1675  if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
1676  TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
1677  false);
1678  if (NeedsWinCFI) {
1679  HasWinCFI = true;
1680  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1682  }
1683  }
1684 
1685  // The very last FrameSetup instruction indicates the end of prologue. Emit a
1686  // SEH opcode indicating the prologue end.
1687  if (NeedsWinCFI && HasWinCFI) {
1688  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1690  }
1691 
1692  // SEH funclets are passed the frame pointer in X1. If the parent
1693  // function uses the base register, then the base register is used
1694  // directly, and is not retrieved from X1.
1695  if (IsFunclet && F.hasPersonalityFn()) {
1696  EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
1697  if (isAsynchronousEHPersonality(Per)) {
1698  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
1699  .addReg(AArch64::X1)
1701  MBB.addLiveIn(AArch64::X1);
1702  }
1703  }
1704 }
1705 
1708  const auto &MFI = *MF.getInfo<AArch64FunctionInfo>();
1709  if (!MFI.shouldSignReturnAddress())
1710  return;
1711  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1712  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1713 
1715  DebugLoc DL;
1716  if (MBBI != MBB.end())
1717  DL = MBBI->getDebugLoc();
1718 
1719  // The AUTIASP instruction assembles to a hint instruction before v8.3a so
1720  // this instruction can safely used for any v8a architecture.
1721  // From v8.3a onwards there are optimised authenticate LR and return
1722  // instructions, namely RETA{A,B}, that can be used instead. In this case the
1723  // DW_CFA_AARCH64_negate_ra_state can't be emitted.
1724  if (Subtarget.hasPAuth() && MBBI != MBB.end() &&
1725  MBBI->getOpcode() == AArch64::RET_ReallyLR) {
1726  BuildMI(MBB, MBBI, DL,
1727  TII->get(MFI.shouldSignWithBKey() ? AArch64::RETAB : AArch64::RETAA))
1728  .copyImplicitOps(*MBBI);
1729  MBB.erase(MBBI);
1730  } else {
1731  BuildMI(
1732  MBB, MBBI, DL,
1733  TII->get(MFI.shouldSignWithBKey() ? AArch64::AUTIBSP : AArch64::AUTIASP))
1735 
1736  unsigned CFIIndex =
1738  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1739  .addCFIIndex(CFIIndex)
1741  }
1742 }
1743 
1744 static bool isFuncletReturnInstr(const MachineInstr &MI) {
1745  switch (MI.getOpcode()) {
1746  default:
1747  return false;
1748  case AArch64::CATCHRET:
1749  case AArch64::CLEANUPRET:
1750  return true;
1751  }
1752 }
1753 
1755  MachineBasicBlock &MBB) const {
1757  MachineFrameInfo &MFI = MF.getFrameInfo();
1758  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1759  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1760  DebugLoc DL;
1761  bool NeedsWinCFI = needsWinCFI(MF);
1762  bool EmitCFI = MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo();
1763  bool HasWinCFI = false;
1764  bool IsFunclet = false;
1765  auto WinCFI = make_scope_exit([&]() { assert(HasWinCFI == MF.hasWinCFI()); });
1766 
1767  if (MBB.end() != MBBI) {
1768  DL = MBBI->getDebugLoc();
1769  IsFunclet = isFuncletReturnInstr(*MBBI);
1770  }
1771 
1772  auto FinishingTouches = make_scope_exit([&]() {
1776  if (EmitCFI)
1777  emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
1778  });
1779 
1780  int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1781  : MFI.getStackSize();
1783 
1784  // All calls are tail calls in GHC calling conv, and functions have no
1785  // prologue/epilogue.
1787  return;
1788 
1789  // How much of the stack used by incoming arguments this function is expected
1790  // to restore in this particular epilogue.
1791  int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB);
1792  bool IsWin64 =
1793  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1794  unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1795 
1796  int64_t AfterCSRPopSize = ArgumentStackToRestore;
1797  auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1798  // We cannot rely on the local stack size set in emitPrologue if the function
1799  // has funclets, as funclets have different local stack size requirements, and
1800  // the current value set in emitPrologue may be that of the containing
1801  // function.
1802  if (MF.hasEHFunclets())
1803  AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1804  if (homogeneousPrologEpilog(MF, &MBB)) {
1805  assert(!NeedsWinCFI);
1806  auto LastPopI = MBB.getFirstTerminator();
1807  if (LastPopI != MBB.begin()) {
1808  auto HomogeneousEpilog = std::prev(LastPopI);
1809  if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog)
1810  LastPopI = HomogeneousEpilog;
1811  }
1812 
1813  // Adjust local stack
1814  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1816  MachineInstr::FrameDestroy, false, NeedsWinCFI);
1817 
1818  // SP has been already adjusted while restoring callee save regs.
1819  // We've bailed-out the case with adjusting SP for arguments.
1820  assert(AfterCSRPopSize == 0);
1821  return;
1822  }
1823  bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
1824  // Assume we can't combine the last pop with the sp restore.
1825 
1826  bool CombineAfterCSRBump = false;
1827  if (!CombineSPBump && PrologueSaveSize != 0) {
1829  while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
1831  Pop = std::prev(Pop);
1832  // Converting the last ldp to a post-index ldp is valid only if the last
1833  // ldp's offset is 0.
1834  const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
1835  // If the offset is 0 and the AfterCSR pop is not actually trying to
1836  // allocate more stack for arguments (in space that an untimely interrupt
1837  // may clobber), convert it to a post-index ldp.
1838  if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) {
1840  MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, EmitCFI,
1841  MachineInstr::FrameDestroy, PrologueSaveSize);
1842  } else {
1843  // If not, make sure to emit an add after the last ldp.
1844  // We're doing this by transfering the size to be restored from the
1845  // adjustment *before* the CSR pops to the adjustment *after* the CSR
1846  // pops.
1847  AfterCSRPopSize += PrologueSaveSize;
1848  CombineAfterCSRBump = true;
1849  }
1850  }
1851 
1852  // Move past the restores of the callee-saved registers.
1853  // If we plan on combining the sp bump of the local stack size and the callee
1854  // save stack size, we might need to adjust the CSR save and restore offsets.
1857  while (LastPopI != Begin) {
1858  --LastPopI;
1859  if (!LastPopI->getFlag(MachineInstr::FrameDestroy) ||
1860  IsSVECalleeSave(LastPopI)) {
1861  ++LastPopI;
1862  break;
1863  } else if (CombineSPBump)
1865  NeedsWinCFI, &HasWinCFI);
1866  }
1867 
1868  if (MF.hasWinCFI()) {
1869  // If the prologue didn't contain any SEH opcodes and didn't set the
1870  // MF.hasWinCFI() flag, assume the epilogue won't either, and skip the
1871  // EpilogStart - to avoid generating CFI for functions that don't need it.
1872  // (And as we didn't generate any prologue at all, it would be asymmetrical
1873  // to the epilogue.) By the end of the function, we assert that
1874  // HasWinCFI is equal to MF.hasWinCFI(), to verify this assumption.
1875  HasWinCFI = true;
1876  BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
1878  }
1879 
1880  if (hasFP(MF) && AFI->hasSwiftAsyncContext()) {
1881  switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
1883  // Avoid the reload as it is GOT relative, and instead fall back to the
1884  // hardcoded value below. This allows a mismatch between the OS and
1885  // application without immediately terminating on the difference.
1888  // We need to reset FP to its untagged state on return. Bit 60 is
1889  // currently used to show the presence of an extended frame.
1890 
1891  // BIC x29, x29, #0x1000_0000_0000_0000
1892  BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::ANDXri),
1893  AArch64::FP)
1894  .addUse(AArch64::FP)
1895  .addImm(0x10fe)
1897  break;
1898 
1900  break;
1901  }
1902  }
1903 
1904  const StackOffset &SVEStackSize = getSVEStackSize(MF);
1905 
1906  // If there is a single SP update, insert it before the ret and we're done.
1907  if (CombineSPBump) {
1908  assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1909 
1910  // When we are about to restore the CSRs, the CFA register is SP again.
1911  if (EmitCFI && hasFP(MF)) {
1912  const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
1913  unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
1914  unsigned CFIIndex =
1915  MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, NumBytes));
1916  BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1917  .addCFIIndex(CFIIndex)
1919  }
1920 
1921  emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
1922  StackOffset::getFixed(NumBytes + (int64_t)AfterCSRPopSize),
1923  TII, MachineInstr::FrameDestroy, false, NeedsWinCFI,
1924  &HasWinCFI, EmitCFI, StackOffset::getFixed(NumBytes));
1925  if (HasWinCFI)
1927  TII->get(AArch64::SEH_EpilogEnd))
1929  return;
1930  }
1931 
1932  NumBytes -= PrologueSaveSize;
1933  assert(NumBytes >= 0 && "Negative stack allocation size!?");
1934 
1935  // Process the SVE callee-saves to determine what space needs to be
1936  // deallocated.
1937  StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
1938  MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
1939  if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
1940  RestoreBegin = std::prev(RestoreEnd);
1941  while (RestoreBegin != MBB.begin() &&
1942  IsSVECalleeSave(std::prev(RestoreBegin)))
1943  --RestoreBegin;
1944 
1945  assert(IsSVECalleeSave(RestoreBegin) &&
1946  IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
1947 
1948  StackOffset CalleeSavedSizeAsOffset =
1949  StackOffset::getScalable(CalleeSavedSize);
1950  DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
1951  DeallocateAfter = CalleeSavedSizeAsOffset;
1952  }
1953 
1954  // Deallocate the SVE area.
1955  if (SVEStackSize) {
1956  // If we have stack realignment or variable sized objects on the stack,
1957  // restore the stack pointer from the frame pointer prior to SVE CSR
1958  // restoration.
1959  if (AFI->isStackRealigned() || MFI.hasVarSizedObjects()) {
1960  if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
1961  // Set SP to start of SVE callee-save area from which they can
1962  // be reloaded. The code below will deallocate the stack space
1963  // space by moving FP -> SP.
1964  emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
1965  StackOffset::getScalable(-CalleeSavedSize), TII,
1967  }
1968  } else {
1969  if (AFI->getSVECalleeSavedStackSize()) {
1970  // Deallocate the non-SVE locals first before we can deallocate (and
1971  // restore callee saves) from the SVE area.
1973  MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
1975  false, false, nullptr, EmitCFI && !hasFP(MF),
1976  SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize));
1977  NumBytes = 0;
1978  }
1979 
1980  emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
1981  DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
1982  false, nullptr, EmitCFI && !hasFP(MF),
1983  SVEStackSize +
1984  StackOffset::getFixed(NumBytes + PrologueSaveSize));
1985 
1986  emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
1987  DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
1988  false, nullptr, EmitCFI && !hasFP(MF),
1989  DeallocateAfter +
1990  StackOffset::getFixed(NumBytes + PrologueSaveSize));
1991  }
1992  if (EmitCFI)
1993  emitCalleeSavedSVERestores(MBB, RestoreEnd);
1994  }
1995 
1996  if (!hasFP(MF)) {
1997  bool RedZone = canUseRedZone(MF);
1998  // If this was a redzone leaf function, we don't need to restore the
1999  // stack pointer (but we may need to pop stack args for fastcc).
2000  if (RedZone && AfterCSRPopSize == 0)
2001  return;
2002 
2003  // Pop the local variables off the stack. If there are no callee-saved
2004  // registers, it means we are actually positioned at the terminator and can
2005  // combine stack increment for the locals and the stack increment for
2006  // callee-popped arguments into (possibly) a single instruction and be done.
2007  bool NoCalleeSaveRestore = PrologueSaveSize == 0;
2008  int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
2009  if (NoCalleeSaveRestore)
2010  StackRestoreBytes += AfterCSRPopSize;
2011 
2013  MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
2014  StackOffset::getFixed(StackRestoreBytes), TII,
2015  MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI,
2016  StackOffset::getFixed((RedZone ? 0 : NumBytes) + PrologueSaveSize));
2017 
2018  // If we were able to combine the local stack pop with the argument pop,
2019  // then we're done.
2020  if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
2021  if (HasWinCFI) {
2023  TII->get(AArch64::SEH_EpilogEnd))
2025  }
2026  return;
2027  }
2028 
2029  NumBytes = 0;
2030  }
2031 
2032  // Restore the original stack pointer.
2033  // FIXME: Rather than doing the math here, we should instead just use
2034  // non-post-indexed loads for the restores if we aren't actually going to
2035  // be able to save any instructions.
2036  if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
2038  MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
2040  TII, MachineInstr::FrameDestroy, false, NeedsWinCFI);
2041  } else if (NumBytes)
2042  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
2043  StackOffset::getFixed(NumBytes), TII,
2044  MachineInstr::FrameDestroy, false, NeedsWinCFI);
2045 
2046  // When we are about to restore the CSRs, the CFA register is SP again.
2047  if (EmitCFI && hasFP(MF)) {
2048  const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
2049  unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
2050  unsigned CFIIndex = MF.addFrameInst(
2051  MCCFIInstruction::cfiDefCfa(nullptr, Reg, PrologueSaveSize));
2052  BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
2053  .addCFIIndex(CFIIndex)
2055  }
2056 
2057  // This must be placed after the callee-save restore code because that code
2058  // assumes the SP is at the same location as it was after the callee-save save
2059  // code in the prologue.
2060  if (AfterCSRPopSize) {
2061  assert(AfterCSRPopSize > 0 && "attempting to reallocate arg stack that an "
2062  "interrupt may have clobbered");
2063 
2065  MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
2067  false, NeedsWinCFI, &HasWinCFI, EmitCFI,
2068  StackOffset::getFixed(CombineAfterCSRBump ? PrologueSaveSize : 0));
2069  }
2070  if (HasWinCFI)
2071  BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
2073 }
2074 
2075 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
2076 /// debug info. It's the same as what we use for resolving the code-gen
2077 /// references for now. FIXME: This can go wrong when references are
2078 /// SP-relative and simple call frames aren't used.
2081  Register &FrameReg) const {
2083  MF, FI, FrameReg,
2084  /*PreferFP=*/
2085  MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress),
2086  /*ForSimm=*/false);
2087 }
2088 
2091  int FI) const {
2093 }
2094 
2096  int64_t ObjectOffset) {
2097  const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2098  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2099  bool IsWin64 =
2100  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
2101  unsigned FixedObject =
2102  getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
2103  int64_t CalleeSaveSize = AFI->getCalleeSavedStackSize(MF.getFrameInfo());
2104  int64_t FPAdjust =
2105  CalleeSaveSize - AFI->getCalleeSaveBaseToFrameRecordOffset();
2106  return StackOffset::getFixed(ObjectOffset + FixedObject + FPAdjust);
2107 }
2108 
2110  int64_t ObjectOffset) {
2111  const auto &MFI = MF.getFrameInfo();
2112  return StackOffset::getFixed(ObjectOffset + (int64_t)MFI.getStackSize());
2113 }
2114 
2115  // TODO: This function currently does not work for scalable vectors.
2117  int FI) const {
2118  const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
2119  MF.getSubtarget().getRegisterInfo());
2120  int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI);
2121  return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
2122  ? getFPOffset(MF, ObjectOffset).getFixed()
2123  : getStackOffset(MF, ObjectOffset).getFixed();
2124 }
2125 
2127  const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP,
2128  bool ForSimm) const {
2129  const auto &MFI = MF.getFrameInfo();
2130  int64_t ObjectOffset = MFI.getObjectOffset(FI);
2131  bool isFixed = MFI.isFixedObjectIndex(FI);
2132  bool isSVE = MFI.getStackID(FI) == TargetStackID::ScalableVector;
2133  return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
2134  PreferFP, ForSimm);
2135 }
2136 
2138  const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
2139  Register &FrameReg, bool PreferFP, bool ForSimm) const {
2140  const auto &MFI = MF.getFrameInfo();
2141  const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
2142  MF.getSubtarget().getRegisterInfo());
2143  const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2144  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2145 
2146  int64_t FPOffset = getFPOffset(MF, ObjectOffset).getFixed();
2147  int64_t Offset = getStackOffset(MF, ObjectOffset).getFixed();
2148  bool isCSR =
2149  !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
2150 
2151  const StackOffset &SVEStackSize = getSVEStackSize(MF);
2152 
2153  // Use frame pointer to reference fixed objects. Use it for locals if
2154  // there are VLAs or a dynamically realigned SP (and thus the SP isn't
2155  // reliable as a base). Make sure useFPForScavengingIndex() does the
2156  // right thing for the emergency spill slot.
2157  bool UseFP = false;
2158  if (AFI->hasStackFrame() && !isSVE) {
2159  // We shouldn't prefer using the FP to access fixed-sized stack objects when
2160  // there are scalable (SVE) objects in between the FP and the fixed-sized
2161  // objects.
2162  PreferFP &= !SVEStackSize;
2163 
2164  // Note: Keeping the following as multiple 'if' statements rather than
2165  // merging to a single expression for readability.
2166  //
2167  // Argument access should always use the FP.
2168  if (isFixed) {
2169  UseFP = hasFP(MF);
2170  } else if (isCSR && RegInfo->hasStackRealignment(MF)) {
2171  // References to the CSR area must use FP if we're re-aligning the stack
2172  // since the dynamically-sized alignment padding is between the SP/BP and
2173  // the CSR area.
2174  assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
2175  UseFP = true;
2176  } else if (hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
2177  // If the FPOffset is negative and we're producing a signed immediate, we
2178  // have to keep in mind that the available offset range for negative
2179  // offsets is smaller than for positive ones. If an offset is available
2180  // via the FP and the SP, use whichever is closest.
2181  bool FPOffsetFits = !ForSimm || FPOffset >= -256;
2182  PreferFP |= Offset > -FPOffset && !SVEStackSize;
2183 
2184  if (MFI.hasVarSizedObjects()) {
2185  // If we have variable sized objects, we can use either FP or BP, as the
2186  // SP offset is unknown. We can use the base pointer if we have one and
2187  // FP is not preferred. If not, we're stuck with using FP.
2188  bool CanUseBP = RegInfo->hasBasePointer(MF);
2189  if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
2190  UseFP = PreferFP;
2191  else if (!CanUseBP) // Can't use BP. Forced to use FP.
2192  UseFP = true;
2193  // else we can use BP and FP, but the offset from FP won't fit.
2194  // That will make us scavenge registers which we can probably avoid by
2195  // using BP. If it won't fit for BP either, we'll scavenge anyway.
2196  } else if (FPOffset >= 0) {
2197  // Use SP or FP, whichever gives us the best chance of the offset
2198  // being in range for direct access. If the FPOffset is positive,
2199  // that'll always be best, as the SP will be even further away.
2200  UseFP = true;
2201  } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
2202  // Funclets access the locals contained in the parent's stack frame
2203  // via the frame pointer, so we have to use the FP in the parent
2204  // function.
2205  (void) Subtarget;
2206  assert(
2207  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
2208  "Funclets should only be present on Win64");
2209  UseFP = true;
2210  } else {
2211  // We have the choice between FP and (SP or BP).
2212  if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
2213  UseFP = true;
2214  }
2215  }
2216  }
2217 
2218  assert(
2219  ((isFixed || isCSR) || !RegInfo->hasStackRealignment(MF) || !UseFP) &&
2220  "In the presence of dynamic stack pointer realignment, "
2221  "non-argument/CSR objects cannot be accessed through the frame pointer");
2222 
2223  if (isSVE) {
2224  StackOffset FPOffset =
2226  StackOffset SPOffset =
2227  SVEStackSize +
2228  StackOffset::get(MFI.getStackSize() - AFI->getCalleeSavedStackSize(),
2229  ObjectOffset);
2230  // Always use the FP for SVE spills if available and beneficial.
2231  if (hasFP(MF) && (SPOffset.getFixed() ||
2232  FPOffset.getScalable() < SPOffset.getScalable() ||
2233  RegInfo->hasStackRealignment(MF))) {
2234  FrameReg = RegInfo->getFrameRegister(MF);
2235  return FPOffset;
2236  }
2237 
2238  FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
2239  : (unsigned)AArch64::SP;
2240  return SPOffset;
2241  }
2242 
2243  StackOffset ScalableOffset = {};
2244  if (UseFP && !(isFixed || isCSR))
2245  ScalableOffset = -SVEStackSize;
2246  if (!UseFP && (isFixed || isCSR))
2247  ScalableOffset = SVEStackSize;
2248 
2249  if (UseFP) {
2250  FrameReg = RegInfo->getFrameRegister(MF);
2251  return StackOffset::getFixed(FPOffset) + ScalableOffset;
2252  }
2253 
2254  // Use the base pointer if we have one.
2255  if (RegInfo->hasBasePointer(MF))
2256  FrameReg = RegInfo->getBaseRegister();
2257  else {
2258  assert(!MFI.hasVarSizedObjects() &&
2259  "Can't use SP when we have var sized objects.");
2260  FrameReg = AArch64::SP;
2261  // If we're using the red zone for this function, the SP won't actually
2262  // be adjusted, so the offsets will be negative. They're also all
2263  // within range of the signed 9-bit immediate instructions.
2264  if (canUseRedZone(MF))
2265  Offset -= AFI->getLocalStackSize();
2266  }
2267 
2268  return StackOffset::getFixed(Offset) + ScalableOffset;
2269 }
2270 
2271 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
2272  // Do not set a kill flag on values that are also marked as live-in. This
2273  // happens with the @llvm-returnaddress intrinsic and with arguments passed in
2274  // callee saved registers.
2275  // Omitting the kill flags is conservatively correct even if the live-in
2276  // is not used after all.
2277  bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
2278  return getKillRegState(!IsLiveIn);
2279 }
2280 
2282  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2284  return Subtarget.isTargetMachO() &&
2285  !(Subtarget.getTargetLowering()->supportSwiftError() &&
2286  Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
2288 }
2289 
2290 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
2291  bool NeedsWinCFI, bool IsFirst) {
2292  // If we are generating register pairs for a Windows function that requires
2293  // EH support, then pair consecutive registers only. There are no unwind
2294  // opcodes for saves/restores of non-consectuve register pairs.
2295  // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x,
2296  // save_lrpair.
2297  // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
2298 
2299  if (Reg2 == AArch64::FP)
2300  return true;
2301  if (!NeedsWinCFI)
2302  return false;
2303  if (Reg2 == Reg1 + 1)
2304  return false;
2305  // If pairing a GPR with LR, the pair can be described by the save_lrpair
2306  // opcode. If this is the first register pair, it would end up with a
2307  // predecrement, but there's no save_lrpair_x opcode, so we can only do this
2308  // if LR is paired with something else than the first register.
2309  // The save_lrpair opcode requires the first register to be an odd one.
2310  if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
2311  (Reg1 - AArch64::X19) % 2 == 0 && Reg2 == AArch64::LR && !IsFirst)
2312  return false;
2313  return true;
2314 }
2315 
2316 /// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
2317 /// WindowsCFI requires that only consecutive registers can be paired.
2318 /// LR and FP need to be allocated together when the frame needs to save
2319 /// the frame-record. This means any other register pairing with LR is invalid.
2320 static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2,
2321  bool UsesWinAAPCS, bool NeedsWinCFI,
2322  bool NeedsFrameRecord, bool IsFirst) {
2323  if (UsesWinAAPCS)
2324  return invalidateWindowsRegisterPairing(Reg1, Reg2, NeedsWinCFI, IsFirst);
2325 
2326  // If we need to store the frame record, don't pair any register
2327  // with LR other than FP.
2328  if (NeedsFrameRecord)
2329  return Reg2 == AArch64::LR;
2330 
2331  return false;
2332 }
2333 
2334 namespace {
2335 
2336 struct RegPairInfo {
2337  unsigned Reg1 = AArch64::NoRegister;
2338  unsigned Reg2 = AArch64::NoRegister;
2339  int FrameIdx;
2340  int Offset;
2341  enum RegType { GPR, FPR64, FPR128, PPR, ZPR } Type;
2342 
2343  RegPairInfo() = default;
2344 
2345  bool isPaired() const { return Reg2 != AArch64::NoRegister; }
2346 
2347  unsigned getScale() const {
2348  switch (Type) {
2349  case PPR:
2350  return 2;
2351  case GPR:
2352  case FPR64:
2353  return 8;
2354  case ZPR:
2355  case FPR128:
2356  return 16;
2357  }
2358  llvm_unreachable("Unsupported type");
2359  }
2360 
2361  bool isScalable() const { return Type == PPR || Type == ZPR; }
2362 };
2363 
2364 } // end anonymous namespace
2365 
2369  bool NeedsFrameRecord) {
2370 
2371  if (CSI.empty())
2372  return;
2373 
2374  bool IsWindows = isTargetWindows(MF);
2375  bool NeedsWinCFI = needsWinCFI(MF);
2377  MachineFrameInfo &MFI = MF.getFrameInfo();
2379  unsigned Count = CSI.size();
2380  (void)CC;
2381  // MachO's compact unwind format relies on all registers being stored in
2382  // pairs.
2385  (Count & 1) == 0) &&
2386  "Odd number of callee-saved regs to spill!");
2387  int ByteOffset = AFI->getCalleeSavedStackSize();
2388  int StackFillDir = -1;
2389  int RegInc = 1;
2390  unsigned FirstReg = 0;
2391  if (NeedsWinCFI) {
2392  // For WinCFI, fill the stack from the bottom up.
2393  ByteOffset = 0;
2394  StackFillDir = 1;
2395  // As the CSI array is reversed to match PrologEpilogInserter, iterate
2396  // backwards, to pair up registers starting from lower numbered registers.
2397  RegInc = -1;
2398  FirstReg = Count - 1;
2399  }
2400  int ScalableByteOffset = AFI->getSVECalleeSavedStackSize();
2401  bool NeedGapToAlignStack = AFI->hasCalleeSaveStackFreeSpace();
2402 
2403  // When iterating backwards, the loop condition relies on unsigned wraparound.
2404  for (unsigned i = FirstReg; i < Count; i += RegInc) {
2405  RegPairInfo RPI;
2406  RPI.Reg1 = CSI[i].getReg();
2407 
2408  if (AArch64::GPR64RegClass.contains(RPI.Reg1))
2409  RPI.Type = RegPairInfo::GPR;
2410  else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
2411  RPI.Type = RegPairInfo::FPR64;
2412  else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
2413  RPI.Type = RegPairInfo::FPR128;
2414  else if (AArch64::ZPRRegClass.contains(RPI.Reg1))
2415  RPI.Type = RegPairInfo::ZPR;
2416  else if (AArch64::PPRRegClass.contains(RPI.Reg1))
2417  RPI.Type = RegPairInfo::PPR;
2418  else
2419  llvm_unreachable("Unsupported register class.");
2420 
2421  // Add the next reg to the pair if it is in the same register class.
2422  if (unsigned(i + RegInc) < Count) {
2423  Register NextReg = CSI[i + RegInc].getReg();
2424  bool IsFirst = i == FirstReg;
2425  switch (RPI.Type) {
2426  case RegPairInfo::GPR:
2427  if (AArch64::GPR64RegClass.contains(NextReg) &&
2428  !invalidateRegisterPairing(RPI.Reg1, NextReg, IsWindows,
2429  NeedsWinCFI, NeedsFrameRecord, IsFirst))
2430  RPI.Reg2 = NextReg;
2431  break;
2432  case RegPairInfo::FPR64:
2433  if (AArch64::FPR64RegClass.contains(NextReg) &&
2434  !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI,
2435  IsFirst))
2436  RPI.Reg2 = NextReg;
2437  break;
2438  case RegPairInfo::FPR128:
2439  if (AArch64::FPR128RegClass.contains(NextReg))
2440  RPI.Reg2 = NextReg;
2441  break;
2442  case RegPairInfo::PPR:
2443  case RegPairInfo::ZPR:
2444  break;
2445  }
2446  }
2447 
2448  // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
2449  // list to come in sorted by frame index so that we can issue the store
2450  // pair instructions directly. Assert if we see anything otherwise.
2451  //
2452  // The order of the registers in the list is controlled by
2453  // getCalleeSavedRegs(), so they will always be in-order, as well.
2454  assert((!RPI.isPaired() ||
2455  (CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
2456  "Out of order callee saved regs!");
2457 
2458  assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
2459  RPI.Reg1 == AArch64::LR) &&
2460  "FrameRecord must be allocated together with LR");
2461 
2462  // Windows AAPCS has FP and LR reversed.
2463  assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
2464  RPI.Reg2 == AArch64::LR) &&
2465  "FrameRecord must be allocated together with LR");
2466 
2467  // MachO's compact unwind format relies on all registers being stored in
2468  // adjacent register pairs.
2471  (RPI.isPaired() &&
2472  ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
2473  RPI.Reg1 + 1 == RPI.Reg2))) &&
2474  "Callee-save registers not saved as adjacent register pair!");
2475 
2476  RPI.FrameIdx = CSI[i].getFrameIdx();
2477  if (NeedsWinCFI &&
2478  RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
2479  RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
2480 
2481  int Scale = RPI.getScale();
2482 
2483  int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2484  assert(OffsetPre % Scale == 0);
2485 
2486  if (RPI.isScalable())
2487  ScalableByteOffset += StackFillDir * Scale;
2488  else
2489  ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
2490 
2491  // Swift's async context is directly before FP, so allocate an extra
2492  // 8 bytes for it.
2493  if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
2494  RPI.Reg2 == AArch64::FP)
2495  ByteOffset += StackFillDir * 8;
2496 
2497  assert(!(RPI.isScalable() && RPI.isPaired()) &&
2498  "Paired spill/fill instructions don't exist for SVE vectors");
2499 
2500  // Round up size of non-pair to pair size if we need to pad the
2501  // callee-save area to ensure 16-byte alignment.
2502  if (NeedGapToAlignStack && !NeedsWinCFI &&
2503  !RPI.isScalable() && RPI.Type != RegPairInfo::FPR128 &&
2504  !RPI.isPaired() && ByteOffset % 16 != 0) {
2505  ByteOffset += 8 * StackFillDir;
2506  assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
2507  // A stack frame with a gap looks like this, bottom up:
2508  // d9, d8. x21, gap, x20, x19.
2509  // Set extra alignment on the x21 object to create the gap above it.
2510  MFI.setObjectAlignment(RPI.FrameIdx, Align(16));
2511  NeedGapToAlignStack = false;
2512  }
2513 
2514  int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2515  assert(OffsetPost % Scale == 0);
2516  // If filling top down (default), we want the offset after incrementing it.
2517  // If fillibg bootom up (WinCFI) we need the original offset.
2518  int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
2519 
2520  // The FP, LR pair goes 8 bytes into our expanded 24-byte slot so that the
2521  // Swift context can directly precede FP.
2522  if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
2523  RPI.Reg2 == AArch64::FP)
2524  Offset += 8;
2525  RPI.Offset = Offset / Scale;
2526 
2527  assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
2528  (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
2529  "Offset out of bounds for LDP/STP immediate");
2530 
2531  // Save the offset to frame record so that the FP register can point to the
2532  // innermost frame record (spilled FP and LR registers).
2533  if (NeedsFrameRecord && ((!IsWindows && RPI.Reg1 == AArch64::LR &&
2534  RPI.Reg2 == AArch64::FP) ||
2535  (IsWindows && RPI.Reg1 == AArch64::FP &&
2536  RPI.Reg2 == AArch64::LR)))
2538 
2539  RegPairs.push_back(RPI);
2540  if (RPI.isPaired())
2541  i += RegInc;
2542  }
2543  if (NeedsWinCFI) {
2544  // If we need an alignment gap in the stack, align the topmost stack
2545  // object. A stack frame with a gap looks like this, bottom up:
2546  // x19, d8. d9, gap.
2547  // Set extra alignment on the topmost stack object (the first element in
2548  // CSI, which goes top down), to create the gap above it.
2549  if (AFI->hasCalleeSaveStackFreeSpace())
2550  MFI.setObjectAlignment(CSI[0].getFrameIdx(), Align(16));
2551  // We iterated bottom up over the registers; flip RegPairs back to top
2552  // down order.
2553  std::reverse(RegPairs.begin(), RegPairs.end());
2554  }
2555 }
2556 
2559  ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2560  MachineFunction &MF = *MBB.getParent();
2561  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2562  bool NeedsWinCFI = needsWinCFI(MF);
2563  DebugLoc DL;
2564  SmallVector<RegPairInfo, 8> RegPairs;
2565 
2566  computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF));
2567 
2568  const MachineRegisterInfo &MRI = MF.getRegInfo();
2569  if (homogeneousPrologEpilog(MF)) {
2570  auto MIB = BuildMI(MBB, MI, DL, TII.get(AArch64::HOM_Prolog))
2572 
2573  for (auto &RPI : RegPairs) {
2574  MIB.addReg(RPI.Reg1);
2575  MIB.addReg(RPI.Reg2);
2576 
2577  // Update register live in.
2578  if (!MRI.isReserved(RPI.Reg1))
2579  MBB.addLiveIn(RPI.Reg1);
2580  if (!MRI.isReserved(RPI.Reg2))
2581  MBB.addLiveIn(RPI.Reg2);
2582  }
2583  return true;
2584  }
2585  for (const RegPairInfo &RPI : llvm::reverse(RegPairs)) {
2586  unsigned Reg1 = RPI.Reg1;
2587  unsigned Reg2 = RPI.Reg2;
2588  unsigned StrOpc;
2589 
2590  // Issue sequence of spills for cs regs. The first spill may be converted
2591  // to a pre-decrement store later by emitPrologue if the callee-save stack
2592  // area allocation can't be combined with the local stack area allocation.
2593  // For example:
2594  // stp x22, x21, [sp, #0] // addImm(+0)
2595  // stp x20, x19, [sp, #16] // addImm(+2)
2596  // stp fp, lr, [sp, #32] // addImm(+4)
2597  // Rationale: This sequence saves uop updates compared to a sequence of
2598  // pre-increment spills like stp xi,xj,[sp,#-16]!
2599  // Note: Similar rationale and sequence for restores in epilog.
2600  unsigned Size;
2601  Align Alignment;
2602  switch (RPI.Type) {
2603  case RegPairInfo::GPR:
2604  StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
2605  Size = 8;
2606  Alignment = Align(8);
2607  break;
2608  case RegPairInfo::FPR64:
2609  StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
2610  Size = 8;
2611  Alignment = Align(8);
2612  break;
2613  case RegPairInfo::FPR128:
2614  StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
2615  Size = 16;
2616  Alignment = Align(16);
2617  break;
2618  case RegPairInfo::ZPR:
2619  StrOpc = AArch64::STR_ZXI;
2620  Size = 16;
2621  Alignment = Align(16);
2622  break;
2623  case RegPairInfo::PPR:
2624  StrOpc = AArch64::STR_PXI;
2625  Size = 2;
2626  Alignment = Align(2);
2627  break;
2628  }
2629  LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
2630  if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2631  dbgs() << ") -> fi#(" << RPI.FrameIdx;
2632  if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2633  dbgs() << ")\n");
2634 
2635  assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
2636  "Windows unwdinding requires a consecutive (FP,LR) pair");
2637  // Windows unwind codes require consecutive registers if registers are
2638  // paired. Make the switch here, so that the code below will save (x,x+1)
2639  // and not (x+1,x).
2640  unsigned FrameIdxReg1 = RPI.FrameIdx;
2641  unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2642  if (NeedsWinCFI && RPI.isPaired()) {
2643  std::swap(Reg1, Reg2);
2644  std::swap(FrameIdxReg1, FrameIdxReg2);
2645  }
2646  MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
2647  if (!MRI.isReserved(Reg1))
2648  MBB.addLiveIn(Reg1);
2649  if (RPI.isPaired()) {
2650  if (!MRI.isReserved(Reg2))
2651  MBB.addLiveIn(Reg2);
2652  MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
2654  MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2655  MachineMemOperand::MOStore, Size, Alignment));
2656  }
2657  MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
2658  .addReg(AArch64::SP)
2659  .addImm(RPI.Offset) // [sp, #offset*scale],
2660  // where factor*scale is implicit
2663  MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2664  MachineMemOperand::MOStore, Size, Alignment));
2665  if (NeedsWinCFI)
2667 
2668  // Update the StackIDs of the SVE stack slots.
2669  MachineFrameInfo &MFI = MF.getFrameInfo();
2670  if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR)
2672 
2673  }
2674  return true;
2675 }
2676 
2680  MachineFunction &MF = *MBB.getParent();
2681  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2682  DebugLoc DL;
2683  SmallVector<RegPairInfo, 8> RegPairs;
2684  bool NeedsWinCFI = needsWinCFI(MF);
2685 
2686  if (MBBI != MBB.end())
2687  DL = MBBI->getDebugLoc();
2688 
2689  computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF));
2690 
2691  auto EmitMI = [&](const RegPairInfo &RPI) -> MachineBasicBlock::iterator {
2692  unsigned Reg1 = RPI.Reg1;
2693  unsigned Reg2 = RPI.Reg2;
2694 
2695  // Issue sequence of restores for cs regs. The last restore may be converted
2696  // to a post-increment load later by emitEpilogue if the callee-save stack
2697  // area allocation can't be combined with the local stack area allocation.
2698  // For example:
2699  // ldp fp, lr, [sp, #32] // addImm(+4)
2700  // ldp x20, x19, [sp, #16] // addImm(+2)
2701  // ldp x22, x21, [sp, #0] // addImm(+0)
2702  // Note: see comment in spillCalleeSavedRegisters()
2703  unsigned LdrOpc;
2704  unsigned Size;
2705  Align Alignment;
2706  switch (RPI.Type) {
2707  case RegPairInfo::GPR:
2708  LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
2709  Size = 8;
2710  Alignment = Align(8);
2711  break;
2712  case RegPairInfo::FPR64:
2713  LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
2714  Size = 8;
2715  Alignment = Align(8);
2716  break;
2717  case RegPairInfo::FPR128:
2718  LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
2719  Size = 16;
2720  Alignment = Align(16);
2721  break;
2722  case RegPairInfo::ZPR:
2723  LdrOpc = AArch64::LDR_ZXI;
2724  Size = 16;
2725  Alignment = Align(16);
2726  break;
2727  case RegPairInfo::PPR:
2728  LdrOpc = AArch64::LDR_PXI;
2729  Size = 2;
2730  Alignment = Align(2);
2731  break;
2732  }
2733  LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
2734  if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2735  dbgs() << ") -> fi#(" << RPI.FrameIdx;
2736  if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2737  dbgs() << ")\n");
2738 
2739  // Windows unwind codes require consecutive registers if registers are
2740  // paired. Make the switch here, so that the code below will save (x,x+1)
2741  // and not (x+1,x).
2742  unsigned FrameIdxReg1 = RPI.FrameIdx;
2743  unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2744  if (NeedsWinCFI && RPI.isPaired()) {
2745  std::swap(Reg1, Reg2);
2746  std::swap(FrameIdxReg1, FrameIdxReg2);
2747  }
2748  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
2749  if (RPI.isPaired()) {
2750  MIB.addReg(Reg2, getDefRegState(true));
2752  MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2753  MachineMemOperand::MOLoad, Size, Alignment));
2754  }
2755  MIB.addReg(Reg1, getDefRegState(true))
2756  .addReg(AArch64::SP)
2757  .addImm(RPI.Offset) // [sp, #offset*scale]
2758  // where factor*scale is implicit
2761  MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2762  MachineMemOperand::MOLoad, Size, Alignment));
2763  if (NeedsWinCFI)
2765 
2766  return MIB->getIterator();
2767  };
2768 
2769  // SVE objects are always restored in reverse order.
2770  for (const RegPairInfo &RPI : reverse(RegPairs))
2771  if (RPI.isScalable())
2772  EmitMI(RPI);
2773 
2774  if (homogeneousPrologEpilog(MF, &MBB)) {
2775  auto MIB = BuildMI(MBB, MBBI, DL, TII.get(AArch64::HOM_Epilog))
2777  for (auto &RPI : RegPairs) {
2778  MIB.addReg(RPI.Reg1, RegState::Define);
2779  MIB.addReg(RPI.Reg2, RegState::Define);
2780  }
2781  return true;
2782  }
2783 
2784  if (ReverseCSRRestoreSeq) {
2786  for (const RegPairInfo &RPI : reverse(RegPairs)) {
2787  if (RPI.isScalable())
2788  continue;
2789  MachineBasicBlock::iterator It = EmitMI(RPI);
2790  if (First == MBB.end())
2791  First = It;
2792  }
2793  if (First != MBB.end())
2794  MBB.splice(MBBI, &MBB, First);
2795  } else {
2796  for (const RegPairInfo &RPI : RegPairs) {
2797  if (RPI.isScalable())
2798  continue;
2799  (void)EmitMI(RPI);
2800  }
2801  }
2802 
2803  return true;
2804 }
2805 
2807  BitVector &SavedRegs,
2808  RegScavenger *RS) const {
2809  // All calls are tail calls in GHC calling conv, and functions have no
2810  // prologue/epilogue.
2812  return;
2813 
2814  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2815  const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
2816  MF.getSubtarget().getRegisterInfo());
2817  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2819  unsigned UnspilledCSGPR = AArch64::NoRegister;
2820  unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
2821 
2822  MachineFrameInfo &MFI = MF.getFrameInfo();
2823  const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
2824 
2825  unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
2826  ? RegInfo->getBaseRegister()
2827  : (unsigned)AArch64::NoRegister;
2828 
2829  unsigned ExtraCSSpill = 0;
2830  // Figure out which callee-saved registers to save/restore.
2831  for (unsigned i = 0; CSRegs[i]; ++i) {
2832  const unsigned Reg = CSRegs[i];
2833 
2834  // Add the base pointer register to SavedRegs if it is callee-save.
2835  if (Reg == BasePointerReg)
2836  SavedRegs.set(Reg);
2837 
2838  bool RegUsed = SavedRegs.test(Reg);
2839  unsigned PairedReg = AArch64::NoRegister;
2840  if (AArch64::GPR64RegClass.contains(Reg) ||
2841  AArch64::FPR64RegClass.contains(Reg) ||
2842  AArch64::FPR128RegClass.contains(Reg))
2843  PairedReg = CSRegs[i ^ 1];
2844 
2845  if (!RegUsed) {
2846  if (AArch64::GPR64RegClass.contains(Reg) &&
2847  !RegInfo->isReservedReg(MF, Reg)) {
2848  UnspilledCSGPR = Reg;
2849  UnspilledCSGPRPaired = PairedReg;
2850  }
2851  continue;
2852  }
2853 
2854  // MachO's compact unwind format relies on all registers being stored in
2855  // pairs.
2856  // FIXME: the usual format is actually better if unwinding isn't needed.
2857  if (producePairRegisters(MF) && PairedReg != AArch64::NoRegister &&
2858  !SavedRegs.test(PairedReg)) {
2859  SavedRegs.set(PairedReg);
2860  if (AArch64::GPR64RegClass.contains(PairedReg) &&
2861  !RegInfo->isReservedReg(MF, PairedReg))
2862  ExtraCSSpill = PairedReg;
2863  }
2864  }
2865 
2867  !Subtarget.isTargetWindows()) {
2868  // For Windows calling convention on a non-windows OS, where X18 is treated
2869  // as reserved, back up X18 when entering non-windows code (marked with the
2870  // Windows calling convention) and restore when returning regardless of
2871  // whether the individual function uses it - it might call other functions
2872  // that clobber it.
2873  SavedRegs.set(AArch64::X18);
2874  }
2875 
2876  // Calculates the callee saved stack size.
2877  unsigned CSStackSize = 0;
2878  unsigned SVECSStackSize = 0;
2880  const MachineRegisterInfo &MRI = MF.getRegInfo();
2881  for (unsigned Reg : SavedRegs.set_bits()) {
2882  auto RegSize = TRI->getRegSizeInBits(Reg, MRI) / 8;
2883  if (AArch64::PPRRegClass.contains(Reg) ||
2884  AArch64::ZPRRegClass.contains(Reg))
2885  SVECSStackSize += RegSize;
2886  else
2887  CSStackSize += RegSize;
2888  }
2889 
2890  // Save number of saved regs, so we can easily update CSStackSize later.
2891  unsigned NumSavedRegs = SavedRegs.count();
2892 
2893  // The frame record needs to be created by saving the appropriate registers
2894  uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
2895  if (hasFP(MF) ||
2896  windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
2897  SavedRegs.set(AArch64::FP);
2898  SavedRegs.set(AArch64::LR);
2899  }
2900 
2901  LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
2902  for (unsigned Reg
2903  : SavedRegs.set_bits()) dbgs()
2904  << ' ' << printReg(Reg, RegInfo);
2905  dbgs() << "\n";);
2906 
2907  // If any callee-saved registers are used, the frame cannot be eliminated.
2908  int64_t SVEStackSize =
2909  alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
2910  bool CanEliminateFrame = (SavedRegs.count() == 0) && !SVEStackSize;
2911 
2912  // The CSR spill slots have not been allocated yet, so estimateStackSize
2913  // won't include them.
2914  unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
2915 
2916  // Conservatively always assume BigStack when there are SVE spills.
2917  bool BigStack = SVEStackSize ||
2918  (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
2919  if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
2920  AFI->setHasStackFrame(true);
2921 
2922  // Estimate if we might need to scavenge a register at some point in order
2923  // to materialize a stack offset. If so, either spill one additional
2924  // callee-saved register or reserve a special spill slot to facilitate
2925  // register scavenging. If we already spilled an extra callee-saved register
2926  // above to keep the number of spills even, we don't need to do anything else
2927  // here.
2928  if (BigStack) {
2929  if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
2930  LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
2931  << " to get a scratch register.\n");
2932  SavedRegs.set(UnspilledCSGPR);
2933  // MachO's compact unwind format relies on all registers being stored in
2934  // pairs, so if we need to spill one extra for BigStack, then we need to
2935  // store the pair.
2936  if (producePairRegisters(MF))
2937  SavedRegs.set(UnspilledCSGPRPaired);
2938  ExtraCSSpill = UnspilledCSGPR;
2939  }
2940 
2941  // If we didn't find an extra callee-saved register to spill, create
2942  // an emergency spill slot.
2943  if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
2945  const TargetRegisterClass &RC = AArch64::GPR64RegClass;
2946  unsigned Size = TRI->getSpillSize(RC);
2947  Align Alignment = TRI->getSpillAlign(RC);
2948  int FI = MFI.CreateStackObject(Size, Alignment, false);
2949  RS->addScavengingFrameIndex(FI);
2950  LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
2951  << " as the emergency spill slot.\n");
2952  }
2953  }
2954 
2955  // Adding the size of additional 64bit GPR saves.
2956  CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
2957 
2958  // A Swift asynchronous context extends the frame record with a pointer
2959  // directly before FP.
2960  if (hasFP(MF) && AFI->hasSwiftAsyncContext())
2961  CSStackSize += 8;
2962 
2963  uint64_t AlignedCSStackSize = alignTo(CSStackSize, 16);
2964  LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
2965  << EstimatedStackSize + AlignedCSStackSize
2966  << " bytes.\n");
2967 
2968  assert((!MFI.isCalleeSavedInfoValid() ||
2969  AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
2970  "Should not invalidate callee saved info");
2971 
2972  // Round up to register pair alignment to avoid additional SP adjustment
2973  // instructions.
2974  AFI->setCalleeSavedStackSize(AlignedCSStackSize);
2975  AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
2976  AFI->setSVECalleeSavedStackSize(alignTo(SVECSStackSize, 16));
2977 }
2978 
2980  MachineFunction &MF, const TargetRegisterInfo *RegInfo,
2981  std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
2982  unsigned &MaxCSFrameIndex) const {
2983  bool NeedsWinCFI = needsWinCFI(MF);
2984  // To match the canonical windows frame layout, reverse the list of
2985  // callee saved registers to get them laid out by PrologEpilogInserter
2986  // in the right order. (PrologEpilogInserter allocates stack objects top
2987  // down. Windows canonical prologs store higher numbered registers at
2988  // the top, thus have the CSI array start from the highest registers.)
2989  if (NeedsWinCFI)
2990  std::reverse(CSI.begin(), CSI.end());
2991 
2992  if (CSI.empty())
2993  return true; // Early exit if no callee saved registers are modified!
2994 
2995  // Now that we know which registers need to be saved and restored, allocate
2996  // stack slots for them.
2997  MachineFrameInfo &MFI = MF.getFrameInfo();
2998  auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2999 
3000  bool UsesWinAAPCS = isTargetWindows(MF);
3001  if (UsesWinAAPCS && hasFP(MF) && AFI->hasSwiftAsyncContext()) {
3002  int FrameIdx = MFI.CreateStackObject(8, Align(16), true);
3003  AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
3004  if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3005  if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3006  }
3007 
3008  for (auto &CS : CSI) {
3009  Register Reg = CS.getReg();
3010  const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
3011 
3012  unsigned Size = RegInfo->getSpillSize(*RC);
3013  Align Alignment(RegInfo->getSpillAlign(*RC));
3014  int FrameIdx = MFI.CreateStackObject(Size, Alignment, true);
3015  CS.setFrameIdx(FrameIdx);
3016 
3017  if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3018  if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3019 
3020  // Grab 8 bytes below FP for the extended asynchronous frame info.
3021  if (hasFP(MF) && AFI->hasSwiftAsyncContext() && !UsesWinAAPCS &&
3022  Reg == AArch64::FP) {
3023  FrameIdx = MFI.CreateStackObject(8, Alignment, true);
3024  AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
3025  if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3026  if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3027  }
3028  }
3029  return true;
3030 }
3031 
3033  const MachineFunction &MF) const {
3034  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
3035  return AFI->hasCalleeSaveStackFreeSpace();
3036 }
3037 
3038 /// returns true if there are any SVE callee saves.
3040  int &Min, int &Max) {
3043 
3044  if (!MFI.isCalleeSavedInfoValid())
3045  return false;
3046 
3047  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
3048  for (auto &CS : CSI) {
3049  if (AArch64::ZPRRegClass.contains(CS.getReg()) ||
3050  AArch64::PPRRegClass.contains(CS.getReg())) {
3052  Max + 1 == CS.getFrameIdx()) &&
3053  "SVE CalleeSaves are not consecutive");
3054 
3055  Min = std::min(Min, CS.getFrameIdx());
3056  Max = std::max(Max, CS.getFrameIdx());
3057  }
3058  }
3059  return Min != std::numeric_limits<int>::max();
3060 }
3061 
3062 // Process all the SVE stack objects and determine offsets for each
3063 // object. If AssignOffsets is true, the offsets get assigned.
3064 // Fills in the first and last callee-saved frame indices into
3065 // Min/MaxCSFrameIndex, respectively.
3066 // Returns the size of the stack.
3068  int &MinCSFrameIndex,
3069  int &MaxCSFrameIndex,
3070  bool AssignOffsets) {
3071 #ifndef NDEBUG
3072  // First process all fixed stack objects.
3073  for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
3075  "SVE vectors should never be passed on the stack by value, only by "
3076  "reference.");
3077 #endif
3078 
3079  auto Assign = [&MFI](int FI, int64_t Offset) {
3080  LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
3081  MFI.setObjectOffset(FI, Offset);
3082  };
3083 
3084  int64_t Offset = 0;
3085 
3086  // Then process all callee saved slots.
3087  if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
3088  // Assign offsets to the callee save slots.
3089  for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
3090  Offset += MFI.getObjectSize(I);
3091  Offset = alignTo(Offset, MFI.getObjectAlign(I));
3092  if (AssignOffsets)
3093  Assign(I, -Offset);
3094  }
3095  }
3096 
3097  // Ensure that the Callee-save area is aligned to 16bytes.
3098  Offset = alignTo(Offset, Align(16U));
3099 
3100  // Create a buffer of SVE objects to allocate and sort it.
3101  SmallVector<int, 8> ObjectsToAllocate;
3102  // If we have a stack protector, and we've previously decided that we have SVE
3103  // objects on the stack and thus need it to go in the SVE stack area, then it
3104  // needs to go first.
3105  int StackProtectorFI = -1;
3106  if (MFI.hasStackProtectorIndex()) {
3107  StackProtectorFI = MFI.getStackProtectorIndex();
3108  if (MFI.getStackID(StackProtectorFI) == TargetStackID::ScalableVector)
3109  ObjectsToAllocate.push_back(StackProtectorFI);
3110  }
3111  for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
3112  unsigned StackID = MFI.getStackID(I);
3113  if (StackID != TargetStackID::ScalableVector)
3114  continue;
3115  if (I == StackProtectorFI)
3116  continue;
3117  if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex)
3118  continue;
3119  if (MFI.isDeadObjectIndex(I))
3120  continue;
3121 
3122  ObjectsToAllocate.push_back(I);
3123  }
3124 
3125  // Allocate all SVE locals and spills
3126  for (unsigned FI : ObjectsToAllocate) {
3127  Align Alignment = MFI.getObjectAlign(FI);
3128  // FIXME: Given that the length of SVE vectors is not necessarily a power of
3129  // two, we'd need to align every object dynamically at runtime if the
3130  // alignment is larger than 16. This is not yet supported.
3131  if (Alignment > Align(16))
3133  "Alignment of scalable vectors > 16 bytes is not yet supported");
3134 
3135  Offset = alignTo(Offset + MFI.getObjectSize(FI), Alignment);
3136  if (AssignOffsets)
3137  Assign(FI, -Offset);
3138  }
3139 
3140  return Offset;
3141 }
3142 
3143 int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
3144  MachineFrameInfo &MFI) const {
3145  int MinCSFrameIndex, MaxCSFrameIndex;
3146  return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex, false);
3147 }
3148 
3149 int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
3150  MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex) const {
3151  return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex,
3152  true);
3153 }
3154 
3156  MachineFunction &MF, RegScavenger *RS) const {
3157  MachineFrameInfo &MFI = MF.getFrameInfo();
3158 
3160  "Upwards growing stack unsupported");
3161 
3162  int MinCSFrameIndex, MaxCSFrameIndex;
3163  int64_t SVEStackSize =
3164  assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
3165 
3167  AFI->setStackSizeSVE(alignTo(SVEStackSize, 16U));
3168  AFI->setMinMaxSVECSFrameIndex(MinCSFrameIndex, MaxCSFrameIndex);
3169 
3170  // If this function isn't doing Win64-style C++ EH, we don't need to do
3171  // anything.
3172  if (!MF.hasEHFunclets())
3173  return;
3174  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
3175  WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
3176 
3177  MachineBasicBlock &MBB = MF.front();
3178  auto MBBI = MBB.begin();
3179  while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
3180  ++MBBI;
3181 
3182  // Create an UnwindHelp object.
3183  // The UnwindHelp object is allocated at the start of the fixed object area
3184  int64_t FixedObject =
3185  getFixedObjectSize(MF, AFI, /*IsWin64*/ true, /*IsFunclet*/ false);
3186  int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8,
3187  /*SPOffset*/ -FixedObject,
3188  /*IsImmutable=*/false);
3189  EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
3190 
3191  // We need to store -2 into the UnwindHelp object at the start of the
3192  // function.
3193  DebugLoc DL;
3194  RS->enterBasicBlockEnd(MBB);
3195  RS->backward(std::prev(MBBI));
3196  Register DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
3197  assert(DstReg && "There must be a free register after frame setup");
3198  BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
3199  BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
3200  .addReg(DstReg, getKillRegState(true))
3201  .addFrameIndex(UnwindHelpFI)
3202  .addImm(0);
3203 }
3204 
3205 namespace {
3206 struct TagStoreInstr {
3207  MachineInstr *MI;
3208  int64_t Offset, Size;
3209  explicit TagStoreInstr(MachineInstr *MI, int64_t Offset, int64_t Size)
3210  : MI(MI), Offset(Offset), Size(Size) {}
3211 };
3212 
3213 class TagStoreEdit {
3214  MachineFunction *MF;
3217  // Tag store instructions that are being replaced.
3219  // Combined memref arguments of the above instructions.
3220  SmallVector<MachineMemOperand *, 8> CombinedMemRefs;
3221 
3222  // Replace allocation tags in [FrameReg + FrameRegOffset, FrameReg +
3223  // FrameRegOffset + Size) with the address tag of SP.
3224  Register FrameReg;
3225  StackOffset FrameRegOffset;
3226  int64_t Size;
3227  // If not None, move FrameReg to (FrameReg + FrameRegUpdate) at the end.
3228  Optional<int64_t> FrameRegUpdate;
3229  // MIFlags for any FrameReg updating instructions.
3230  unsigned FrameRegUpdateFlags;
3231 
3232  // Use zeroing instruction variants.
3233  bool ZeroData;
3234  DebugLoc DL;
3235 
3236  void emitUnrolled(MachineBasicBlock::iterator InsertI);
3237  void emitLoop(MachineBasicBlock::iterator InsertI);
3238 
3239 public:
3240  TagStoreEdit(MachineBasicBlock *MBB, bool ZeroData)
3241  : MBB(MBB), ZeroData(ZeroData) {
3242  MF = MBB->getParent();
3243  MRI = &MF->getRegInfo();
3244  }
3245  // Add an instruction to be replaced. Instructions must be added in the
3246  // ascending order of Offset, and have to be adjacent.
3247  void addInstruction(TagStoreInstr I) {
3248  assert((TagStores.empty() ||
3249  TagStores.back().Offset + TagStores.back().Size == I.Offset) &&
3250  "Non-adjacent tag store instructions.");
3251  TagStores.push_back(I);
3252  }
3253  void clear() { TagStores.clear(); }
3254  // Emit equivalent code at the given location, and erase the current set of
3255  // instructions. May skip if the replacement is not profitable. May invalidate
3256  // the input iterator and replace it with a valid one.
3257  void emitCode(MachineBasicBlock::iterator &InsertI,
3258  const AArch64FrameLowering *TFI, bool TryMergeSPUpdate);
3259 };
3260 
3261 void TagStoreEdit::emitUnrolled(MachineBasicBlock::iterator InsertI) {
3262  const AArch64InstrInfo *TII =
3263  MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
3264 
3265  const int64_t kMinOffset = -256 * 16;
3266  const int64_t kMaxOffset = 255 * 16;
3267 
3268  Register BaseReg = FrameReg;
3269  int64_t BaseRegOffsetBytes = FrameRegOffset.getFixed();
3270  if (BaseRegOffsetBytes < kMinOffset ||
3271  BaseRegOffsetBytes + (Size - Size % 32) > kMaxOffset) {
3272  Register ScratchReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3273  emitFrameOffset(*MBB, InsertI, DL, ScratchReg, BaseReg,
3274  StackOffset::getFixed(BaseRegOffsetBytes), TII);
3275  BaseReg = ScratchReg;
3276  BaseRegOffsetBytes = 0;
3277  }
3278 
3279  MachineInstr *LastI = nullptr;
3280  while (Size) {
3281  int64_t InstrSize = (Size > 16) ? 32 : 16;
3282  unsigned Opcode =
3283  InstrSize == 16
3284  ? (ZeroData ? AArch64::STZGOffset : AArch64::STGOffset)
3285  : (ZeroData ? AArch64::STZ2GOffset : AArch64::ST2GOffset);
3286  MachineInstr *I = BuildMI(*MBB, InsertI, DL, TII->get(Opcode))
3287  .addReg(AArch64::SP)
3288  .addReg(BaseReg)
3289  .addImm(BaseRegOffsetBytes / 16)
3290  .setMemRefs(CombinedMemRefs);
3291  // A store to [BaseReg, #0] should go last for an opportunity to fold the
3292  // final SP adjustment in the epilogue.
3293  if (BaseRegOffsetBytes == 0)
3294  LastI = I;
3295  BaseRegOffsetBytes += InstrSize;
3296  Size -= InstrSize;
3297  }
3298 
3299  if (LastI)
3300  MBB->splice(InsertI, MBB, LastI);
3301 }
3302 
3303 void TagStoreEdit::emitLoop(MachineBasicBlock::iterator InsertI) {
3304  const AArch64InstrInfo *TII =
3305  MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
3306 
3307  Register BaseReg = FrameRegUpdate
3308  ? FrameReg
3309  : MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3310  Register SizeReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3311 
3312  emitFrameOffset(*MBB, InsertI, DL, BaseReg, FrameReg, FrameRegOffset, TII);
3313 
3314  int64_t LoopSize = Size;
3315  // If the loop size is not a multiple of 32, split off one 16-byte store at
3316  // the end to fold BaseReg update into.
3317  if (FrameRegUpdate && *FrameRegUpdate)
3318  LoopSize -= LoopSize % 32;
3319  MachineInstr *LoopI = BuildMI(*MBB, InsertI, DL,
3320  TII->get(ZeroData ? AArch64::STZGloop_wback
3321  : AArch64::STGloop_wback))
3322  .addDef(SizeReg)
3323  .addDef(BaseReg)
3324  .addImm(LoopSize)
3325  .addReg(BaseReg)
3326  .setMemRefs(CombinedMemRefs);
3327  if (FrameRegUpdate)
3328  LoopI->setFlags(FrameRegUpdateFlags);
3329 
3330  int64_t ExtraBaseRegUpdate =
3331  FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.getFixed() - Size) : 0;
3332  if (LoopSize < Size) {
3333  assert(FrameRegUpdate);
3334  assert(Size - LoopSize == 16);
3335  // Tag 16 more bytes at BaseReg and update BaseReg.
3336  BuildMI(*MBB, InsertI, DL,
3337  TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
3338  .addDef(BaseReg)
3339  .addReg(BaseReg)
3340  .addReg(BaseReg)
3341  .addImm(1 + ExtraBaseRegUpdate / 16)
3342  .setMemRefs(CombinedMemRefs)
3343  .setMIFlags(FrameRegUpdateFlags);
3344  } else if (ExtraBaseRegUpdate) {
3345  // Update BaseReg.
3346  BuildMI(
3347  *MBB, InsertI, DL,
3348  TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
3349  .addDef(BaseReg)
3350  .addReg(BaseReg)
3351  .addImm(std::abs(ExtraBaseRegUpdate))
3352  .addImm(0)
3353  .setMIFlags(FrameRegUpdateFlags);
3354  }
3355 }
3356 
3357 // Check if *II is a register update that can be merged into STGloop that ends
3358 // at (Reg + Size). RemainingOffset is the required adjustment to Reg after the
3359 // end of the loop.
3360 bool canMergeRegUpdate(MachineBasicBlock::iterator II, unsigned Reg,
3361  int64_t Size, int64_t *TotalOffset) {
3362  MachineInstr &MI = *II;
3363  if ((MI.getOpcode() == AArch64::ADDXri ||
3364  MI.getOpcode() == AArch64::SUBXri) &&
3365  MI.getOperand(0).getReg() == Reg && MI.getOperand(1).getReg() == Reg) {
3366  unsigned Shift = AArch64_AM::getShiftValue(MI.getOperand(3).getImm());
3367  int64_t Offset = MI.getOperand(2).getImm() << Shift;
3368  if (MI.getOpcode() == AArch64::SUBXri)
3369  Offset = -Offset;
3370  int64_t AbsPostOffset = std::abs(Offset - Size);
3371  const int64_t kMaxOffset =
3372  0xFFF; // Max encoding for unshifted ADDXri / SUBXri
3373  if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
3374  *TotalOffset = Offset;
3375  return true;
3376  }
3377  }
3378  return false;
3379 }
3380 
3381 void mergeMemRefs(const SmallVectorImpl<TagStoreInstr> &TSE,
3383  MemRefs.clear();
3384  for (auto &TS : TSE) {
3385  MachineInstr *MI = TS.MI;
3386  // An instruction without memory operands may access anything. Be
3387  // conservative and return an empty list.
3388  if (MI->memoperands_empty()) {
3389  MemRefs.clear();
3390  return;
3391  }
3392  MemRefs.append(MI->memoperands_begin(), MI->memoperands_end());
3393  }
3394 }
3395 
3396 void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
3397  const AArch64FrameLowering *TFI,
3398  bool TryMergeSPUpdate) {
3399  if (TagStores.empty())
3400  return;
3401  TagStoreInstr &FirstTagStore = TagStores[0];
3402  TagStoreInstr &LastTagStore = TagStores[TagStores.size() - 1];
3403  Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
3404  DL = TagStores[0].MI->getDebugLoc();
3405 
3406  Register Reg;
3407  FrameRegOffset = TFI->resolveFrameOffsetReference(
3408  *MF, FirstTagStore.Offset, false /*isFixed*/, false /*isSVE*/, Reg,
3409  /*PreferFP=*/false, /*ForSimm=*/true);
3410  FrameReg = Reg;
3411  FrameRegUpdate = None;
3412 
3413  mergeMemRefs(TagStores, CombinedMemRefs);
3414 
3415  LLVM_DEBUG(dbgs() << "Replacing adjacent STG instructions:\n";
3416  for (const auto &Instr
3417  : TagStores) { dbgs() << " " << *Instr.MI; });
3418 
3419  // Size threshold where a loop becomes shorter than a linear sequence of
3420  // tagging instructions.
3421  const int kSetTagLoopThreshold = 176;
3422  if (Size < kSetTagLoopThreshold) {
3423  if (TagStores.size() < 2)
3424  return;
3425  emitUnrolled(InsertI);
3426  } else {
3427  MachineInstr *UpdateInstr = nullptr;
3428  int64_t TotalOffset = 0;
3429  if (TryMergeSPUpdate) {
3430  // See if we can merge base register update into the STGloop.
3431  // This is done in AArch64LoadStoreOptimizer for "normal" stores,
3432  // but STGloop is way too unusual for that, and also it only
3433  // realistically happens in function epilogue. Also, STGloop is expanded
3434  // before that pass.
3435  if (InsertI != MBB->end() &&
3436  canMergeRegUpdate(InsertI, FrameReg, FrameRegOffset.getFixed() + Size,
3437  &TotalOffset)) {
3438  UpdateInstr = &*InsertI++;
3439  LLVM_DEBUG(dbgs() << "Folding SP update into loop:\n "
3440  << *UpdateInstr);
3441  }
3442  }
3443 
3444  if (!UpdateInstr && TagStores.size() < 2)
3445  return;
3446 
3447  if (UpdateInstr) {
3448  FrameRegUpdate = TotalOffset;
3449  FrameRegUpdateFlags = UpdateInstr->getFlags();
3450  }
3451  emitLoop(InsertI);
3452  if (UpdateInstr)
3453  UpdateInstr->eraseFromParent();
3454  }
3455 
3456  for (auto &TS : TagStores)
3457  TS.MI->eraseFromParent();
3458 }
3459 
3460 bool isMergeableStackTaggingInstruction(MachineInstr &MI, int64_t &Offset,
3461  int64_t &Size, bool &ZeroData) {
3462  MachineFunction &MF = *MI.getParent()->getParent();
3463  const MachineFrameInfo &MFI = MF.getFrameInfo();
3464 
3465  unsigned Opcode = MI.getOpcode();
3466  ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGOffset ||
3467  Opcode == AArch64::STZ2GOffset);
3468 
3469  if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
3470  if (!MI.getOperand(0).isDead() || !MI.getOperand(1).isDead())
3471  return false;
3472  if (!MI.getOperand(2).isImm() || !MI.getOperand(3).isFI())
3473  return false;
3474  Offset = MFI.getObjectOffset(MI.getOperand(3).getIndex());
3475  Size = MI.getOperand(2).getImm();
3476  return true;
3477  }
3478 
3479  if (Opcode == AArch64::STGOffset || Opcode == AArch64::STZGOffset)
3480  Size = 16;
3481  else if (Opcode == AArch64::ST2GOffset || Opcode == AArch64::STZ2GOffset)
3482  Size = 32;
3483  else
3484  return false;
3485 
3486  if (MI.getOperand(0).getReg() != AArch64::SP || !MI.getOperand(1).isFI())
3487  return false;
3488 
3489  Offset = MFI.getObjectOffset(MI.getOperand(1).getIndex()) +
3490  16 * MI.getOperand(2).getImm();
3491  return true;
3492 }
3493 
3494 // Detect a run of memory tagging instructions for adjacent stack frame slots,
3495 // and replace them with a shorter instruction sequence:
3496 // * replace STG + STG with ST2G
3497 // * replace STGloop + STGloop with STGloop
3498 // This code needs to run when stack slot offsets are already known, but before
3499 // FrameIndex operands in STG instructions are eliminated.
3501  const AArch64FrameLowering *TFI,
3502  RegScavenger *RS) {
3503  bool FirstZeroData;
3504  int64_t Size, Offset;
3505  MachineInstr &MI = *II;
3506  MachineBasicBlock *MBB = MI.getParent();
3507  MachineBasicBlock::iterator NextI = ++II;
3508  if (&MI == &MBB->instr_back())
3509  return II;
3510  if (!isMergeableStackTaggingInstruction(MI, Offset, Size, FirstZeroData))
3511  return II;
3512 
3514  Instrs.emplace_back(&MI, Offset, Size);
3515 
3516  constexpr int kScanLimit = 10;
3517  int Count = 0;
3519  NextI != E && Count < kScanLimit; ++NextI) {
3520  MachineInstr &MI = *NextI;
3521  bool ZeroData;
3522  int64_t Size, Offset;
3523  // Collect instructions that update memory tags with a FrameIndex operand
3524  // and (when applicable) constant size, and whose output registers are dead
3525  // (the latter is almost always the case in practice). Since these
3526  // instructions effectively have no inputs or outputs, we are free to skip
3527  // any non-aliasing instructions in between without tracking used registers.
3528  if (isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData)) {
3529  if (ZeroData != FirstZeroData)
3530  break;
3531  Instrs.emplace_back(&MI, Offset, Size);
3532  continue;
3533  }
3534 
3535  // Only count non-transient, non-tagging instructions toward the scan
3536  // limit.
3537  if (!MI.isTransient())
3538  ++Count;
3539 
3540  // Just in case, stop before the epilogue code starts.
3541  if (MI.getFlag(MachineInstr::FrameSetup) ||
3542  MI.getFlag(MachineInstr::FrameDestroy))
3543  break;
3544 
3545  // Reject anything that may alias the collected instructions.
3546  if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects())
3547  break;
3548  }
3549 
3550  // New code will be inserted after the last tagging instruction we've found.
3551  MachineBasicBlock::iterator InsertI = Instrs.back().MI;
3552  InsertI++;
3553 
3554  llvm::stable_sort(Instrs,
3555  [](const TagStoreInstr &Left, const TagStoreInstr &Right) {
3556  return Left.Offset < Right.Offset;
3557  });
3558 
3559  // Make sure that we don't have any overlapping stores.
3560  int64_t CurOffset = Instrs[0].Offset;
3561  for (auto &Instr : Instrs) {
3562  if (CurOffset > Instr.Offset)
3563  return NextI;
3564  CurOffset = Instr.Offset + Instr.Size;
3565  }
3566 
3567  // Find contiguous runs of tagged memory and emit shorter instruction
3568  // sequencies for them when possible.
3569  TagStoreEdit TSE(MBB, FirstZeroData);
3570  Optional<int64_t> EndOffset;
3571  for (auto &Instr : Instrs) {
3572  if (EndOffset && *EndOffset != Instr.Offset) {
3573  // Found a gap.
3574  TSE.emitCode(InsertI, TFI, /*TryMergeSPUpdate = */ false);
3575  TSE.clear();
3576  }
3577 
3578  TSE.addInstruction(Instr);
3579  EndOffset = Instr.Offset + Instr.Size;
3580  }
3581 
3582  // Multiple FP/SP updates in a loop cannot be described by CFI instructions.
3583  TSE.emitCode(InsertI, TFI, /*TryMergeSPUpdate = */
3584  !MBB->getParent()
3587 
3588  return InsertI;
3589 }
3590 } // namespace
3591 
3593  MachineFunction &MF, RegScavenger *RS = nullptr) const {
3595  for (auto &BB : MF)
3596  for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();)
3597  II = tryMergeAdjacentSTG(II, this, RS);
3598 }
3599 
3600 /// For Win64 AArch64 EH, the offset to the Unwind object is from the SP
3601 /// before the update. This is easily retrieved as it is exactly the offset
3602 /// that is set in processFunctionBeforeFrameFinalized.
3604  const MachineFunction &MF, int FI, Register &FrameReg,
3605  bool IgnoreSPUpdates) const {
3606  const MachineFrameInfo &MFI = MF.getFrameInfo();
3607  if (IgnoreSPUpdates) {
3608  LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
3609  << MFI.getObjectOffset(FI) << "\n");
3610  FrameReg = AArch64::SP;
3611  return StackOffset::getFixed(MFI.getObjectOffset(FI));
3612  }
3613 
3614  // Go to common code if we cannot provide sp + offset.
3615  if (MFI.hasVarSizedObjects() ||
3618  return getFrameIndexReference(MF, FI, FrameReg);
3619 
3620  FrameReg = AArch64::SP;
3621  return getStackOffset(MF, MFI.getObjectOffset(FI));
3622 }
3623 
3624 /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
3625 /// the parent's frame pointer
3627  const MachineFunction &MF) const {
3628  return 0;
3629 }
3630 
3631 /// Funclets only need to account for space for the callee saved registers,
3632 /// as the locals are accounted for in the parent's stack frame.
3634  const MachineFunction &MF) const {
3635  // This is the size of the pushed CSRs.
3636  unsigned CSSize =
3637  MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
3638  // This is the amount of stack a funclet needs to allocate.
3639  return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
3640  getStackAlign());
3641 }
3642 
3643 namespace {
3644 struct FrameObject {
3645  bool IsValid = false;
3646  // Index of the object in MFI.
3647  int ObjectIndex = 0;
3648  // Group ID this object belongs to.
3649  int GroupIndex = -1;
3650  // This object should be placed first (closest to SP).
3651  bool ObjectFirst = false;
3652  // This object's group (which always contains the object with
3653  // ObjectFirst==true) should be placed first.
3654  bool GroupFirst = false;
3655 };
3656 
3657 class GroupBuilder {
3658  SmallVector<int, 8> CurrentMembers;
3659  int NextGroupIndex = 0;
3660  std::vector<FrameObject> &Objects;
3661 
3662 public:
3663  GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
3664  void AddMember(int Index) { CurrentMembers.push_back(Index); }
3665  void EndCurrentGroup() {
3666  if (CurrentMembers.size() > 1) {
3667  // Create a new group with the current member list. This might remove them
3668  // from their pre-existing groups. That's OK, dealing with overlapping
3669  // groups is too hard and unlikely to make a difference.
3670  LLVM_DEBUG(dbgs() << "group:");
3671  for (int Index : CurrentMembers) {
3672  Objects[Index].GroupIndex = NextGroupIndex;
3673  LLVM_DEBUG(dbgs() << " " << Index);
3674  }
3675  LLVM_DEBUG(dbgs() << "\n");
3676  NextGroupIndex++;
3677  }
3678  CurrentMembers.clear();
3679  }
3680 };
3681 
3682 bool FrameObjectCompare(const FrameObject &A, const FrameObject &B) {
3683  // Objects at a lower index are closer to FP; objects at a higher index are
3684  // closer to SP.
3685  //
3686  // For consistency in our comparison, all invalid objects are placed
3687  // at the end. This also allows us to stop walking when we hit the
3688  // first invalid item after it's all sorted.
3689  //
3690  // The "first" object goes first (closest to SP), followed by the members of
3691  // the "first" group.
3692  //
3693  // The rest are sorted by the group index to keep the groups together.
3694  // Higher numbered groups are more likely to be around longer (i.e. untagged
3695  // in the function epilogue and not at some earlier point). Place them closer
3696  // to SP.
3697  //
3698  // If all else equal, sort by the object index to keep the objects in the
3699  // original order.
3700  return std::make_tuple(!A.IsValid, A.ObjectFirst, A.GroupFirst, A.GroupIndex,
3701  A.ObjectIndex) <
3702  std::make_tuple(!B.IsValid, B.ObjectFirst, B.GroupFirst, B.GroupIndex,
3703  B.ObjectIndex);
3704 }
3705 } // namespace
3706 
3708  const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
3709  if (!OrderFrameObjects || ObjectsToAllocate.empty())
3710  return;
3711 
3712  const MachineFrameInfo &MFI = MF.getFrameInfo();
3713  std::vector<FrameObject> FrameObjects(MFI.getObjectIndexEnd());
3714  for (auto &Obj : ObjectsToAllocate) {
3715  FrameObjects[Obj].IsValid = true;
3716  FrameObjects[Obj].ObjectIndex = Obj;
3717  }
3718 
3719  // Identify stack slots that are tagged at the same time.
3720  GroupBuilder GB(FrameObjects);
3721  for (auto &MBB : MF) {
3722  for (auto &MI : MBB) {
3723  if (MI.isDebugInstr())
3724  continue;
3725  int OpIndex;
3726  switch (MI.getOpcode()) {
3727  case AArch64::STGloop:
3728  case AArch64::STZGloop:
3729  OpIndex = 3;
3730  break;
3731  case AArch64::STGOffset:
3732  case AArch64::STZGOffset:
3733  case AArch64::ST2GOffset:
3734  case AArch64::STZ2GOffset:
3735  OpIndex = 1;
3736  break;
3737  default:
3738  OpIndex = -1;
3739  }
3740 
3741  int TaggedFI = -1;
3742  if (OpIndex >= 0) {
3743  const MachineOperand &MO = MI.getOperand(OpIndex);
3744  if (MO.isFI()) {
3745  int FI = MO.getIndex();
3746  if (FI >= 0 && FI < MFI.getObjectIndexEnd() &&
3747  FrameObjects[FI].IsValid)
3748  TaggedFI = FI;
3749  }
3750  }
3751 
3752  // If this is a stack tagging instruction for a slot that is not part of a
3753  // group yet, either start a new group or add it to the current one.
3754  if (TaggedFI >= 0)
3755  GB.AddMember(TaggedFI);
3756  else
3757  GB.EndCurrentGroup();
3758  }
3759  // Groups should never span multiple basic blocks.
3760  GB.EndCurrentGroup();
3761  }
3762 
3763  // If the function's tagged base pointer is pinned to a stack slot, we want to
3764  // put that slot first when possible. This will likely place it at SP + 0,
3765  // and save one instruction when generating the base pointer because IRG does
3766  // not allow an immediate offset.
3767  const AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
3769  if (TBPI) {
3770  FrameObjects[*TBPI].ObjectFirst = true;
3771  FrameObjects[*TBPI].GroupFirst = true;
3772  int FirstGroupIndex = FrameObjects[*TBPI].GroupIndex;
3773  if (FirstGroupIndex >= 0)
3774  for (FrameObject &Object : FrameObjects)
3775  if (Object.GroupIndex == FirstGroupIndex)
3776  Object.GroupFirst = true;
3777  }
3778 
3779  llvm::stable_sort(FrameObjects, FrameObjectCompare);
3780 
3781  int i = 0;
3782  for (auto &Obj : FrameObjects) {
3783  // All invalid items are sorted at the end, so it's safe to stop.
3784  if (!Obj.IsValid)
3785  break;
3786  ObjectsToAllocate[i++] = Obj.ObjectIndex;
3787  }
3788 
3789  LLVM_DEBUG(dbgs() << "Final frame order:\n"; for (auto &Obj
3790  : FrameObjects) {
3791  if (!Obj.IsValid)
3792  break;
3793  dbgs() << " " << Obj.ObjectIndex << ": group " << Obj.GroupIndex;
3794  if (Obj.ObjectFirst)
3795  dbgs() << ", first";
3796  if (Obj.GroupFirst)
3797  dbgs() << ", group-first";
3798  dbgs() << "\n";
3799  });
3800 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:76
llvm::MachineFunction::hasWinCFI
bool hasWinCFI() const
Definition: MachineFunction.h:722
llvm::RegState::InternalRead
@ InternalRead
Register reads a value that is defined inside the same instruction or bundle.
Definition: MachineInstrBuilder.h:59
i
i
Definition: README.txt:29
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:148
llvm::isAsynchronousEHPersonality
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
Definition: EHPersonalities.h:49
llvm::AArch64ISD::LOADgot
@ LOADgot
Definition: AArch64ISelLowering.h:66
llvm::MachineFrameInfo::isMaxCallFrameSizeComputed
bool isMaxCallFrameSizeComputed() const
Definition: MachineFrameInfo.h:653
llvm::MachineFrameInfo::hasVarSizedObjects
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
Definition: MachineFrameInfo.h:354
llvm::AArch64Subtarget::isTargetWindows
bool isTargetWindows() const
Definition: AArch64Subtarget.h:245
AArch64RegisterInfo.h
Attrs
Function Attrs
Definition: README_ALTIVEC.txt:215
MCDwarf.h
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:104
MachineInstr.h
MathExtras.h
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm::MachineFrameInfo::estimateStackSize
uint64_t estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
Definition: MachineFrameInfo.cpp:137
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::MachineInstrBuilder::copyImplicitOps
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
Definition: MachineInstrBuilder.h:315
AArch64MachineFunctionInfo.h
llvm::MachineRegisterInfo::isPhysRegUsed
bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
Definition: MachineRegisterInfo.cpp:581
llvm::MCSymbol
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
llvm::AArch64Subtarget::swiftAsyncContextIsDynamicallySet
bool swiftAsyncContextIsDynamicallySet() const
Return whether FrameLowering should always set the "extended frame present" bit in FP,...
Definition: AArch64Subtarget.h:312
getFixedObjectSize
static unsigned getFixedObjectSize(const MachineFunction &MF, const AArch64FunctionInfo *AFI, bool IsWin64, bool IsFunclet)
Returns the size of the fixed object area (allocated next to sp on entry) On Win64 this may include a...
Definition: AArch64FrameLowering.cpp:381
llvm::LivePhysRegs::addReg
void addReg(MCPhysReg Reg)
Adds a physical register and all its sub-registers to the set.
Definition: LivePhysRegs.h:81
DefaultSafeSPDisplacement
static const unsigned DefaultSafeSPDisplacement
This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using ...
Definition: AArch64FrameLowering.cpp:344
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::AArch64_AM::LSL
@ LSL
Definition: AArch64AddressingModes.h:35
llvm::AArch64FrameLowering::emitCalleeSavedFrameMoves
void emitCalleeSavedFrameMoves(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const
Definition: AArch64FrameLowering.cpp:578
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:156
llvm::MachineModuleInfo::getContext
const MCContext & getContext() const
Definition: MachineModuleInfo.h:144
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
produceCompactUnwindFrame
static bool produceCompactUnwindFrame(MachineFunction &MF)
Definition: AArch64FrameLowering.cpp:2281
RegSize
unsigned RegSize
Definition: AArch64MIPeepholeOpt.cpp:126
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition: MachineInstrBuilder.h:224
llvm::Function
Definition: Function.h:60
llvm::BitVector::set
BitVector & set()
Definition: BitVector.h:344
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:93
llvm::MachineInstrBuilder::addCFIIndex
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
Definition: MachineInstrBuilder.h:247
llvm::MachineBasicBlock::isEHFuncletEntry
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
Definition: MachineBasicBlock.h:574
contains
return AArch64::GPR64RegClass contains(Reg)
llvm::CodeModel::Medium
@ Medium
Definition: CodeGen.h:28
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
Statistic.h
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:456
ErrorHandling.h
llvm::MCRegisterInfo::getDwarfRegNum
int getDwarfRegNum(MCRegister RegNum, bool isEH) const
Map a target register to an equivalent dwarf register number.
Definition: MCRegisterInfo.cpp:68
llvm::NVPTX::PTXCvtMode::RPI
@ RPI
Definition: NVPTX.h:135
llvm::getBLRCallOpcode
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
Definition: AArch64InstrInfo.cpp:8000
llvm::AArch64RegisterInfo::getBaseRegister
unsigned getBaseRegister() const
Definition: AArch64RegisterInfo.cpp:383
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::AArch64FunctionInfo::getTaggedBasePointerIndex
Optional< int > getTaggedBasePointerIndex() const
Definition: AArch64MachineFunctionInfo.h:383
llvm::BitVector::set_bits
iterator_range< const_set_bits_iterator > set_bits() const
Definition: BitVector.h:133
llvm::AArch64FunctionInfo::setSwiftAsyncContextFrameIdx
void setSwiftAsyncContextFrameIdx(int FI)
Definition: AArch64MachineFunctionInfo.h:414
MachineBasicBlock.h
llvm::LivePhysRegs
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:50
Right
Vector Shift Left Right
Definition: README_P9.txt:118
llvm::TargetSubtargetInfo::getRegisterInfo
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Definition: TargetSubtargetInfo.h:125
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:139
llvm::MachineOperand::setImm
void setImm(int64_t immVal)
Definition: MachineOperand.h:664
llvm::MachineBasicBlock::findDebugLoc
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any DBG_VALUE and DBG_LABEL instructions.
Definition: MachineBasicBlock.cpp:1369
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:232
llvm::CallingConv::CXX_FAST_TLS
@ CXX_FAST_TLS
Definition: CallingConv.h:76
Shift
bool Shift
Definition: README.txt:468
llvm::AArch64FrameLowering::enableStackSlotScavenging
bool enableStackSlotScavenging(const MachineFunction &MF) const override
Returns true if the stack slot holes in the fixed and callee-save stack area should be used when allo...
Definition: AArch64FrameLowering.cpp:3032
llvm::AArch64Subtarget::getInstrInfo
const AArch64InstrInfo * getInstrInfo() const override
Definition: AArch64Subtarget.h:171
llvm::CallingConv::Win64
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
Definition: CallingConv.h:169
llvm::StackOffset::getFixed
ScalarTy getFixed() const
Definition: TypeSize.h:149
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::reverse
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Definition: STLExtras.h:380
llvm::AttributeList
Definition: Attributes.h:408
TargetInstrInfo.h
llvm::CallingConv::GHC
@ GHC
Definition: CallingConv.h:51
fixupSEHOpcode
static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, unsigned LocalStackSize)
Definition: AArch64FrameLowering.cpp:960
getPrologueDeath
static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg)
Definition: AArch64FrameLowering.cpp:2271
llvm::CallingConv::PreserveMost
@ PreserveMost
Definition: CallingConv.h:66
llvm::Optional< int >
llvm::AArch64FrameLowering::assignCalleeSavedSpillSlots
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI, unsigned &MinCSFrameIndex, unsigned &MaxCSFrameIndex) const override
assignCalleeSavedSpillSlots - Allows target to override spill slot assignment logic.
Definition: AArch64FrameLowering.cpp:2979
llvm::AArch64FrameLowering::hasFP
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register.
Definition: AArch64FrameLowering.cpp:425
llvm::MachineFrameInfo::getObjectIndexEnd
int getObjectIndexEnd() const
Return one past the maximum frame object index.
Definition: MachineFrameInfo.h:409
determineSVEStackObjectOffsets
static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex, bool AssignOffsets)
Definition: AArch64FrameLowering.cpp:3067
llvm::CodeModel::Kernel
@ Kernel
Definition: CodeGen.h:28
llvm::MachineOperand::isFI
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
Definition: MachineOperand.h:330
llvm::AArch64FunctionInfo::setTaggedBasePointerOffset
void setTaggedBasePointerOffset(unsigned Offset)
Definition: AArch64MachineFunctionInfo.h:391
llvm::MCCFIInstruction::createSameValue
static MCCFIInstruction createSameValue(MCSymbol *L, unsigned Register)
.cfi_same_value Current value of Register is the same as in the previous frame.
Definition: MCDwarf.h:609
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1618
llvm::RegScavenger::FindUnusedReg
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
Definition: RegisterScavenging.cpp:266
llvm::MachineFrameInfo::getMaxCallFrameSize
unsigned getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
Definition: MachineFrameInfo.h:646
OpIndex
unsigned OpIndex
Definition: SPIRVModuleAnalysis.cpp:41
llvm::ARCISD::BL
@ BL
Definition: ARCISelLowering.h:34
llvm::ArrayRef::empty
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:159
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::AArch64Subtarget::getTargetLowering
const AArch64TargetLowering * getTargetLowering() const override
Definition: AArch64Subtarget.h:168
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::MCCFIInstruction::cfiDefCfaOffset
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int Offset)
.cfi_def_cfa_offset modifies a rule for computing CFA.
Definition: MCDwarf.h:540
MachineRegisterInfo.h
llvm::MachineInstr::FrameDestroy
@ FrameDestroy
Definition: MachineInstr.h:86
isTargetWindows
static bool isTargetWindows(const MachineFunction &MF)
Definition: AArch64FrameLowering.cpp:1145
llvm::MachineBasicBlock::erase
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
Definition: MachineBasicBlock.cpp:1295
llvm::SwiftAsyncFramePointerMode::Never
@ Never
Never set the bit.
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::AArch64RegisterInfo::cannotEliminateFrame
bool cannotEliminateFrame(const MachineFunction &MF) const
Definition: AArch64RegisterInfo.cpp:461
clear
static void clear(coro::Shape &Shape)
Definition: Coroutines.cpp:149
llvm::classifyEHPersonality
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Definition: EHPersonalities.cpp:21
llvm::AArch64FrameLowering
Definition: AArch64FrameLowering.h:23
llvm::AArch64FrameOffsetCannotUpdate
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
Definition: AArch64InstrInfo.h:426
llvm::MachineInstr::getFlags
uint16_t getFlags() const
Return the MI flags bitvector.
Definition: MachineInstr.h:327
llvm::AlignStyle::Left
@ Left
ReverseCSRRestoreSeq
static cl::opt< bool > ReverseCSRRestoreSeq("reverse-csr-restore-seq", cl::desc("reverse the CSR restore sequence"), cl::init(false), cl::Hidden)
llvm::VFISAKind::SVE
@ SVE
CommandLine.h
llvm::AArch64FrameLowering::processFunctionBeforeFrameFinalized
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
Definition: AArch64FrameLowering.cpp:3155
llvm::MachineInstrBuilder::addDef
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Definition: MachineInstrBuilder.h:116
llvm::getDefRegState
unsigned getDefRegState(bool B)
Definition: MachineInstrBuilder.h:502
llvm::MachineFunction::front
const MachineBasicBlock & front() const
Definition: MachineFunction.h:834
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:650
OrderFrameObjects
static cl::opt< bool > OrderFrameObjects("aarch64-order-frame-objects", cl::desc("sort stack allocations"), cl::init(true), cl::Hidden)
llvm::MachineBasicBlock::insertAfter
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
Definition: MachineBasicBlock.h:893
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
AArch64TargetMachine.h
AArch64InstrInfo.h
llvm::AArch64FrameLowering::resolveFrameOffsetReference
StackOffset resolveFrameOffsetReference(const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE, Register &FrameReg, bool PreferFP, bool ForSimm) const
Definition: AArch64FrameLowering.cpp:2137
llvm::TargetFrameLowering::getOffsetOfLocalArea
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
Definition: TargetFrameLowering.h:140
TargetMachine.h
llvm::MutableArrayRef
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:306
llvm::AArch64FunctionInfo::isStackRealigned
bool isStackRealigned() const
Definition: AArch64MachineFunctionInfo.h:214
llvm::MachineOperand::CreateImm
static MachineOperand CreateImm(int64_t Val)
Definition: MachineOperand.h:782
llvm::AArch64InstrInfo
Definition: AArch64InstrInfo.h:37
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::SmallVectorImpl::append
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:667
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:546
llvm::AArch64FrameLowering::eliminateCallFramePseudoInstr
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
Definition: AArch64FrameLowering.cpp:465
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:738
invalidateWindowsRegisterPairing
static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2, bool NeedsWinCFI, bool IsFirst)
Definition: AArch64FrameLowering.cpp:2290
llvm::LivePhysRegs::addLiveIns
void addLiveIns(const MachineBasicBlock &MBB)
Adds all live-in registers of basic block MBB.
Definition: LivePhysRegs.cpp:238
llvm::ISD::CATCHRET
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:1037
llvm::AArch64Subtarget::isTargetILP32
bool isTargetILP32() const
Definition: AArch64Subtarget.h:253
llvm::TargetFrameLowering::getStackAlign
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Definition: TargetFrameLowering.h:100
llvm::ARM_PROC::A
@ A
Definition: ARMBaseInfo.h:34
llvm::MachineRegisterInfo::isReserved
bool isReserved(MCRegister PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
Definition: MachineRegisterInfo.h:925
llvm::BitVector::count
size_type count() const
count - Returns the number of bits which are set.
Definition: BitVector.h:155
llvm::AArch64TargetLowering::supportSwiftError
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
Definition: AArch64ISelLowering.h:798
llvm::Log2
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition: Alignment.h:207
int
Clang compiles this i1 i64 store i64 i64 store i64 i64 store i64 i64 store i64 align Which gets codegen d xmm0 movaps rbp movaps rbp movaps rbp movaps rbp rbp rbp rbp rbp It would be better to have movq s of instead of the movaps s LLVM produces ret int
Definition: README.txt:536
llvm::AArch64FunctionInfo::setHasRedZone
void setHasRedZone(bool s)
Definition: AArch64MachineFunctionInfo.h:313
getStackOffset
static StackOffset getStackOffset(const MachineFunction &MF, int64_t ObjectOffset)
Definition: AArch64FrameLowering.cpp:2109
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:45
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:127
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::TypeSize::Fixed
static TypeSize Fixed(ScalarTy MinVal)
Definition: TypeSize.h:427
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:197
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
First
into llvm powi allowing the code generator to produce balanced multiplication trees First
Definition: README.txt:54
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:48
llvm::CodeModel::Small
@ Small
Definition: CodeGen.h:28
llvm::MachineInstr::FrameSetup
@ FrameSetup
Definition: MachineInstr.h:84
llvm::make_scope_exit
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:147
llvm::MachineModuleInfo
This class contains meta information specific to a module.
Definition: MachineModuleInfo.h:75
getSVEStackSize
static StackOffset getSVEStackSize(const MachineFunction &MF)
Returns the size of the entire SVE stackframe (calleesaves + spills).
Definition: AArch64FrameLowering.cpp:398
findScratchNonCalleeSaveRegister
static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB)
Definition: AArch64FrameLowering.cpp:689
llvm::AArch64_AM::getShifterImm
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
Definition: AArch64AddressingModes.h:99
llvm::RegScavenger::backward
void backward()
Update internal register state and move MBB iterator backwards.
Definition: RegisterScavenging.cpp:239
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:143
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::MachineFrameInfo::getStackID
uint8_t getStackID(int ObjectIdx) const
Definition: MachineFrameInfo.h:723
llvm::RegScavenger::enterBasicBlockEnd
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
Definition: RegisterScavenging.cpp:87
llvm::MachineBasicBlock::instr_back
MachineInstr & instr_back()
Definition: MachineBasicBlock.h:252
llvm::AArch64FunctionInfo::needsAsyncDwarfUnwindInfo
bool needsAsyncDwarfUnwindInfo() const
Definition: AArch64MachineFunctionInfo.cpp:129
llvm::AArch64_AM::getShiftValue
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
Definition: AArch64AddressingModes.h:86
llvm::MachineFunction::setHasWinCFI
void setHasWinCFI(bool v)
Definition: MachineFunction.h:725
llvm::MachineFrameInfo::getStackSize
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
Definition: MachineFrameInfo.h:577
DebugLoc.h
emitShadowCallStackPrologue
static void emitShadowCallStackPrologue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool NeedsWinCFI, bool NeedsUnwindInfo)
Definition: AArch64FrameLowering.cpp:1176
llvm::MachineFrameInfo::getObjectOffset
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
Definition: MachineFrameInfo.h:518
llvm::AArch64FrameLowering::getFrameIndexReference
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - Provide a base+offset reference to an FI slot for debug info.
Definition: AArch64FrameLowering.cpp:2080
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
llvm::BitVector
Definition: BitVector.h:75
Align
uint64_t Align
Definition: ELFObjHandler.cpp:81
llvm::StackOffset::getScalable
ScalarTy getScalable() const
Definition: TypeSize.h:150
llvm::MCCFIInstruction::createEscape
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
Definition: MCDwarf.h:625
llvm::AArch64FunctionInfo::hasStackFrame
bool hasStackFrame() const
Definition: AArch64MachineFunctionInfo.h:211
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::MachineFrameInfo::getObjectIndexBegin
int getObjectIndexBegin() const
Return the minimum frame object index.
Definition: MachineFrameInfo.h:406
llvm::MachineInstrBuilder::addExternalSymbol
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:184
convertCalleeSaveRestoreToSPPrePostIncDec
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI, MachineInstr::MIFlag FrameFlag=MachineInstr::FrameSetup, int CFAOffset=0)
Definition: AArch64FrameLowering.cpp:982
llvm::None
const NoneType None
Definition: None.h:24
llvm::MachineFrameInfo::isCalleeSavedInfoValid
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Definition: MachineFrameInfo.h:792
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
fixupCalleeSaveRestoreStackOffset
static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, uint64_t LocalStackSize, bool NeedsWinCFI, bool *HasWinCFI)
Definition: AArch64FrameLowering.cpp:1096
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::MachineFrameInfo::isDeadObjectIndex
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
Definition: MachineFrameInfo.h:737
llvm::Function::getAttributes
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:314
computeCalleeSaveRegisterPairs
static void computeCalleeSaveRegisterPairs(MachineFunction &MF, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI, SmallVectorImpl< RegPairInfo > &RegPairs, bool NeedsFrameRecord)
Definition: AArch64FrameLowering.cpp:2366
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:50
AArch64AddressingModes.h
llvm::OutputFileType::Object
@ Object
llvm::cl::ZeroOrMore
@ ZeroOrMore
Definition: CommandLine.h:116
llvm::RegState::Implicit
@ Implicit
Not emitted register (e.g. carry, or temporary result).
Definition: MachineInstrBuilder.h:46
StackTaggingMergeSetTag
static cl::opt< bool > StackTaggingMergeSetTag("stack-tagging-merge-settag", cl::desc("merge settag instruction in function epilog"), cl::init(true), cl::Hidden)
llvm::TargetOptions::DisableFramePointerElim
bool DisableFramePointerElim(const MachineFunction &MF) const
DisableFramePointerElim - This returns true if frame pointer elimination optimization should be disab...
Definition: TargetOptionsImpl.cpp:23
llvm::MachineFunction::getMMI
MachineModuleInfo & getMMI() const
Definition: MachineFunction.h:591
llvm::TargetRegisterInfo::getSpillAlign
Align getSpillAlign(const TargetRegisterClass &RC) const
Return the minimum required alignment in bytes for a spill slot for a register of this class.
Definition: TargetRegisterInfo.h:287
llvm::AArch64FrameLowering::resolveFrameIndexReference
StackOffset resolveFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP, bool ForSimm) const
Definition: AArch64FrameLowering.cpp:2126
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:640
llvm::MachineInstrBuilder::addFrameIndex
const MachineInstrBuilder & addFrameIndex(int Idx) const
Definition: MachineInstrBuilder.h:152
llvm::MachineInstrBuilder::setMIFlag
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
Definition: MachineInstrBuilder.h:278
llvm::isAArch64FrameOffsetLegal
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
Definition: AArch64InstrInfo.cpp:4555
llvm::AArch64FunctionInfo::getCalleeSaveBaseToFrameRecordOffset
int getCalleeSaveBaseToFrameRecordOffset() const
Definition: AArch64MachineFunctionInfo.h:395
llvm::Function::hasFnAttribute
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:625
llvm::cl::opt< bool >
llvm::TargetRegisterInfo::getSpillSize
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
Definition: TargetRegisterInfo.h:281
llvm::WinEHFuncInfo
Definition: WinEHFuncInfo.h:90
llvm::AMDGPU::Hwreg::Offset
Offset
Definition: SIDefines.h:394
llvm::AArch64FrameLowering::resetCFIToInitialState
void resetCFIToInitialState(MachineBasicBlock &MBB) const override
Emit CFI instructions that recreate the state of the unwind information upon fucntion entry.
Definition: AArch64FrameLowering.cpp:593
getSVECalleeSaveSlotRange
static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI, int &Min, int &Max)
returns true if there are any SVE callee saves.
Definition: AArch64FrameLowering.cpp:3039
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
uint64_t
llvm::MachineFrameInfo::getObjectSize
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Definition: MachineFrameInfo.h:469
AArch64FrameLowering.h
llvm::Function::getCallingConv
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:238
llvm::MachineFrameInfo::hasStackProtectorIndex
bool hasStackProtectorIndex() const
Definition: MachineFrameInfo.h:359
llvm::AArch64FrameLowering::spillCalleeSavedRegisters
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
Definition: AArch64FrameLowering.cpp:2557
llvm::TargetOptions::SwiftAsyncFramePointer
SwiftAsyncFramePointerMode SwiftAsyncFramePointer
Control when and how the Swift async frame pointer bit should be set.
Definition: TargetOptions.h:243
llvm::LivePhysRegs::available
bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const
Returns true if register Reg and no aliasing register is in the set.
Definition: LivePhysRegs.cpp:141
llvm::MCCFIInstruction::createNegateRAState
static MCCFIInstruction createNegateRAState(MCSymbol *L)
.cfi_negate_ra_state AArch64 negate RA state.
Definition: MCDwarf.h:590
llvm::AArch64FunctionInfo::needsDwarfUnwindInfo
bool needsDwarfUnwindInfo() const
Definition: AArch64MachineFunctionInfo.cpp:121
llvm::MachineRegisterInfo::getCalleeSavedRegs
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
Definition: MachineRegisterInfo.cpp:617
llvm::AArch64FunctionInfo::setCalleeSaveBaseToFrameRecordOffset
void setCalleeSaveBaseToFrameRecordOffset(int Offset)
Definition: AArch64MachineFunctionInfo.h:398
llvm::ISD::CLEANUPRET
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:1041
llvm::AArch64FunctionInfo
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
Definition: AArch64MachineFunctionInfo.h:38
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::MCPhysReg
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:21
llvm::RegScavenger
Definition: RegisterScavenging.h:34
llvm::MachineFrameInfo::getObjectAlign
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:483
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
llvm::MachineFrameInfo::setObjectOffset
void setObjectOffset(int ObjectIdx, int64_t SPOffset)
Set the stack frame offset of the specified object.
Definition: MachineFrameInfo.h:552
llvm::TargetStackID::ScalableVector
@ ScalableVector
Definition: TargetFrameLowering.h:30
llvm::StackOffset::getScalable
static StackOffset getScalable(ScalarTy Scalable)
Definition: TypeSize.h:144
windowsRequiresStackProbe
static bool windowsRequiresStackProbe(MachineFunction &MF, uint64_t StackSizeInBytes)
Definition: AArch64FrameLowering.cpp:733
llvm::MachineBasicBlock::getLastNonDebugInstr
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
Definition: MachineBasicBlock.cpp:263
llvm::AArch64FrameLowering::getStackIDForScalableVectors
TargetStackID::Value getStackIDForScalableVectors() const override
Returns the StackID that scalable vectors should be associated with.
Definition: AArch64FrameLowering.cpp:375
llvm::TargetMachine::Options
TargetOptions Options
Definition: TargetMachine.h:118
llvm::MachineInstr::setFlags
void setFlags(unsigned flags)
Definition: MachineInstr.h:341
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::AArch64FunctionInfo::getLocalStackSize
uint64_t getLocalStackSize() const
Definition: AArch64MachineFunctionInfo.h:227
InsertSEH
static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI, const TargetInstrInfo &TII, MachineInstr::MIFlag Flag)
Definition: AArch64FrameLowering.cpp:840
llvm::MachineFrameInfo::CreateFixedObject
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
Definition: MachineFrameInfo.cpp:83
llvm::AArch64FunctionInfo::setStackRealigned
void setStackRealigned(bool s)
Definition: AArch64MachineFunctionInfo.h:215
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:656
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:234
llvm::RegScavenger::addScavengingFrameIndex
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
Definition: RegisterScavenging.h:143
llvm::MachineInstrBuilder::addMemOperand
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Definition: MachineInstrBuilder.h:202
llvm::TargetFrameLowering::getStackGrowthDirection
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
Definition: TargetFrameLowering.h:89
llvm::AArch64FunctionInfo::getVarArgsGPRSize
unsigned getVarArgsGPRSize() const
Definition: AArch64MachineFunctionInfo.h:321
llvm::AArch64Subtarget::isCallingConvWin64
bool isCallingConvWin64(CallingConv::ID CC) const
Definition: AArch64Subtarget.h:297
llvm::MCCFIInstruction::cfiDefCfa
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int Offset)
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
Definition: MCDwarf.h:526
llvm::emitFrameOffset
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
Definition: AArch64InstrInfo.cpp:4325
MachineModuleInfo.h
llvm::WinEHFuncInfo::UnwindHelpFrameIdx
int UnwindHelpFrameIdx
Definition: WinEHFuncInfo.h:99
llvm::AArch64FrameLowering::getNonLocalFrameIndexReference
StackOffset getNonLocalFrameIndexReference(const MachineFunction &MF, int FI) const override
getNonLocalFrameIndexReference - This method returns the offset used to reference a frame index locat...
Definition: AArch64FrameLowering.cpp:2090
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Definition: MachineInstrBuilder.h:123
llvm::AArch64FrameLowering::canUseRedZone
bool canUseRedZone(const MachineFunction &MF) const
Can this function use the red zone for local allocations.
Definition: AArch64FrameLowering.cpp:403
llvm::MachineInstr::MIFlag
MIFlag
Definition: MachineInstr.h:82
Cleanup
static const HTTPClientCleanup Cleanup
Definition: HTTPClient.cpp:42
llvm::MachineFunction
Definition: MachineFunction.h:241
TargetOptions.h
llvm::MachineFrameInfo::getCalleeSavedInfo
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
Definition: MachineFrameInfo.h:779
llvm::MachineBasicBlock::getFirstTerminator
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Definition: MachineBasicBlock.cpp:238
llvm::TargetMachine::getMCAsmInfo
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
Definition: TargetMachine.h:205
EnableRedZone
static cl::opt< bool > EnableRedZone("aarch64-redzone", cl::desc("enable use of redzone on AArch64"), cl::init(false), cl::Hidden)
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::AArch64Subtarget::isXRegisterReserved
bool isXRegisterReserved(size_t i) const
Definition: AArch64Subtarget.h:198
llvm::MachineFrameInfo::setStackID
void setStackID(int ObjectIdx, uint8_t ID)
Definition: MachineFrameInfo.h:728
llvm::MachineFrameInfo::hasPatchPoint
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
Definition: MachineFrameInfo.h:388
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
MCAsmInfo.h
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1612
DataLayout.h
llvm::MachineFrameInfo::CreateStackObject
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Definition: MachineFrameInfo.cpp:51
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:972
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: AArch64SLSHardening.cpp:75
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
if
if(llvm_vc STREQUAL "") set(fake_version_inc "$
Definition: CMakeLists.txt:14
uint32_t
llvm::StackOffset
StackOffset is a class to represent an offset with 2 dimensions, named fixed and scalable,...
Definition: TypeSize.h:134
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
TargetSubtargetInfo.h
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::SwiftAsyncFramePointerMode::DeploymentBased
@ DeploymentBased
Determine whether to set the bit statically or dynamically based on the deployment target.
llvm::AArch64FunctionInfo::getTailCallReservedStack
unsigned getTailCallReservedStack() const
Definition: AArch64MachineFunctionInfo.h:197
llvm::AArch64RegisterInfo
Definition: AArch64RegisterInfo.h:26
llvm::AArch64FunctionInfo::setMinMaxSVECSFrameIndex
void setMinMaxSVECSFrameIndex(int Min, int Max)
Definition: AArch64MachineFunctionInfo.h:299
LLVM_FALLTHROUGH
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:280
llvm::MCContext::createTempSymbol
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Definition: MCContext.cpp:321
llvm::CodeModel::Tiny
@ Tiny
Definition: CodeGen.h:28
llvm::EHPersonality
EHPersonality
Definition: EHPersonalities.h:21
llvm::StackOffset::getFixed
static StackOffset getFixed(ScalarTy Fixed)
Definition: TypeSize.h:143
llvm::TargetSubtargetInfo
TargetSubtargetInfo - Generic base class for all target subtargets.
Definition: TargetSubtargetInfo.h:60
Prolog
@ Prolog
Definition: AArch64LowerHomogeneousPrologEpilog.cpp:125
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:133
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::MachineFrameInfo::getMaxAlign
Align getMaxAlign() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
Definition: MachineFrameInfo.h:593
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::Function::hasOptSize
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
Definition: Function.h:663
llvm::MachineBasicBlock::addLiveIn
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
Definition: MachineBasicBlock.h:376
llvm::MachineFrameInfo::isFrameAddressTaken
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
Definition: MachineFrameInfo.h:370
llvm::AArch64_AM::getArithExtendImm
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
Definition: AArch64AddressingModes.h:171
llvm::MachineFrameInfo::hasCalls
bool hasCalls() const
Return true if the current function has any function calls.
Definition: MachineFrameInfo.h:605
llvm::AArch64FrameLowering::hasReservedCallFrame
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
Definition: AArch64FrameLowering.cpp:461
emitShadowCallStackEpilogue
static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL)
Definition: AArch64FrameLowering.cpp:1215
CallingConv.h
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::AArch64Subtarget::getRegisterInfo
const AArch64RegisterInfo * getRegisterInfo() const override
Definition: AArch64Subtarget.h:172
Attributes.h
emitCalleeSavedRestores
static void emitCalleeSavedRestores(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool SVE)
Definition: AArch64FrameLowering.cpp:635
isFuncletReturnInstr
static bool isFuncletReturnInstr(const MachineInstr &MI)
Definition: AArch64FrameLowering.cpp:1744
llvm::BitVector::test
bool test(unsigned Idx) const
Definition: BitVector.h:454
llvm::stable_sort
void stable_sort(R &&Range)
Definition: STLExtras.h:1749
llvm::AArch64_AM::UXTX
@ UXTX
Definition: AArch64AddressingModes.h:44
llvm::AArch64FrameLowering::getWinEHParentFrameOffset
unsigned getWinEHParentFrameOffset(const MachineFunction &MF) const override
The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve the parent's frame pointer...
Definition: AArch64FrameLowering.cpp:3626
llvm::MachineRegisterInfo::isLiveIn
bool isLiveIn(Register Reg) const
Definition: MachineRegisterInfo.cpp:432
kSetTagLoopThreshold
static const int kSetTagLoopThreshold
Definition: AArch64SelectionDAGInfo.cpp:151
llvm::AArch64FunctionInfo::hasCalleeSaveStackFreeSpace
bool hasCalleeSaveStackFreeSpace() const
Definition: AArch64MachineFunctionInfo.h:217
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:606
llvm::TargetRegisterInfo::getRegSizeInBits
unsigned getRegSizeInBits(const TargetRegisterClass &RC) const
Return the size in bits of a register from class RC.
Definition: TargetRegisterInfo.h:275
uint16_t
llvm::AArch64FunctionInfo::getStackSizeSVE
uint64_t getStackSizeSVE() const
Definition: AArch64MachineFunctionInfo.h:209
llvm::MachineFunction::getTarget
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Definition: MachineFunction.h:636
MachineFrameInfo.h
llvm::MachineFunction::getWinEHFuncInfo
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
Definition: MachineFunction.h:684
llvm::AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced
void processFunctionBeforeFrameIndicesReplaced(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameIndicesReplaced - This method is called immediately before MO_FrameIndex op...
Definition: AArch64FrameLowering.cpp:3592
llvm::MachineOperand::getIndex
int getIndex() const
Definition: MachineOperand.h:566
Success
#define Success
Definition: AArch64Disassembler.cpp:279
llvm::TypeSize
Definition: TypeSize.h:421
Function.h
needsShadowCallStackPrologueEpilogue
static bool needsShadowCallStackPrologueEpilogue(MachineFunction &MF)
Definition: AArch64FrameLowering.cpp:1163
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
insertCFISameValue
static void insertCFISameValue(const MCInstrDesc &Desc, MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, unsigned DwarfReg)
Definition: AArch64FrameLowering.cpp:584
llvm::AArch64FunctionInfo::setLocalStackSize
void setLocalStackSize(uint64_t Size)
Definition: AArch64MachineFunctionInfo.h:226
llvm::MachineInstrBuilder::setMemRefs
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
Definition: MachineInstrBuilder.h:208
llvm::TargetMachine::getCodeModel
CodeModel::Model getCodeModel() const
Returns the code model.
Definition: TargetMachine.cpp:72
llvm::AArch64FunctionInfo::getSVECalleeSavedStackSize
unsigned getSVECalleeSavedStackSize() const
Definition: AArch64MachineFunctionInfo.h:295
llvm::SmallVectorImpl::clear
void clear()
Definition: SmallVector.h:591
llvm::AArch64FrameLowering::getSEHFrameIndexOffset
int getSEHFrameIndexOffset(const MachineFunction &MF, int FI) const
Definition: AArch64FrameLowering.cpp:2116
llvm::AArch64FrameLowering::getFrameIndexReferencePreferSP
StackOffset getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI, Register &FrameReg, bool IgnoreSPUpdates) const override
For Win64 AArch64 EH, the offset to the Unwind object is from the SP before the update.
Definition: AArch64FrameLowering.cpp:3603
llvm::MachineFunction::hasEHFunclets
bool hasEHFunclets() const
Definition: MachineFunction.h:1067
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:135
llvm::AMDGPU::Hwreg::Width
Width
Definition: SIDefines.h:417
WinEHFuncInfo.h
llvm::AArch64FrameLowering::emitPrologue
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
Definition: AArch64FrameLowering.cpp:1237
llvm::TargetRegisterInfo::hasStackRealignment
bool hasStackRealignment(const MachineFunction &MF) const
True if stack realignment is required and still possible.
Definition: TargetRegisterInfo.h:945
llvm::AArch64Subtarget::isTargetMachO
bool isTargetMachO() const
Definition: AArch64Subtarget.h:251
llvm::AArch64FunctionInfo::getArgumentStackToRestore
unsigned getArgumentStackToRestore() const
Definition: AArch64MachineFunctionInfo.h:192
llvm::CodeModel::Large
@ Large
Definition: CodeGen.h:28
InsertReturnAddressAuth
static void InsertReturnAddressAuth(MachineFunction &MF, MachineBasicBlock &MBB)
Definition: AArch64FrameLowering.cpp:1706
invalidateRegisterPairing
static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2, bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord, bool IsFirst)
Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
Definition: AArch64FrameLowering.cpp:2320
llvm::getKillRegState
unsigned getKillRegState(bool B)
Definition: MachineInstrBuilder.h:508
llvm::AArch64TargetLowering::getRedZoneSize
unsigned getRedZoneSize(const Function &F) const
Definition: AArch64ISelLowering.h:840
llvm::MachineFunction::addFrameInst
LLVM_NODISCARD unsigned addFrameInst(const MCCFIInstruction &Inst)
Definition: MachineFunction.cpp:312
llvm::MachineFrameInfo
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Definition: MachineFrameInfo.h:105
AArch64Subtarget.h
SmallVector.h
estimateRSStackSizeLimit
static unsigned estimateRSStackSizeLimit(MachineFunction &MF)
Look at each instruction that references stack frames and return the stack size limit beyond which so...
Definition: AArch64FrameLowering.cpp:349
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1006
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:277
MachineInstrBuilder.h
llvm::AArch64FrameLowering::emitEpilogue
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
Definition: AArch64FrameLowering.cpp:1754
llvm::CallingConv::SwiftTail
@ SwiftTail
SwiftTail - This follows the Swift calling convention in how arguments are passed but guarantees tail...
Definition: CallingConv.h:92
llvm::MachineInstrBuilder::setMIFlags
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
Definition: MachineInstrBuilder.h:273
llvm::AArch64FunctionInfo::hasSwiftAsyncContext
bool hasSwiftAsyncContext() const
Definition: AArch64MachineFunctionInfo.h:412
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
getFPOffset
static StackOffset getFPOffset(const MachineFunction &MF, int64_t ObjectOffset)
Definition: AArch64FrameLowering.cpp:2095
llvm::AArch64FrameLowering::canUseAsPrologue
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
Definition: AArch64FrameLowering.cpp:718
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:164
llvm::AArch64FrameLowering::getWinEHFuncletFrameSize
unsigned getWinEHFuncletFrameSize(const MachineFunction &MF) const
Funclets only need to account for space for the callee saved registers, as the locals are accounted f...
Definition: AArch64FrameLowering.cpp:3633
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:340
llvm::MachineBasicBlock::empty
bool empty() const
Definition: MachineBasicBlock.h:249
ScopeExit.h
llvm::AArch64FunctionInfo::getCalleeSavedStackSize
unsigned getCalleeSavedStackSize(const MachineFrameInfo &MFI) const
Definition: AArch64MachineFunctionInfo.h:242
MachineMemOperand.h
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
MachineOperand.h
llvm::TargetFrameLowering::StackGrowsDown
@ StackGrowsDown
Definition: TargetFrameLowering.h:47
llvm::Function::hasMinSize
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:660
llvm::MCCFIInstruction::createOffset
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset)
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
Definition: MCDwarf.h:564
llvm::AArch64FrameLowering::determineCalleeSaves
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
Definition: AArch64FrameLowering.cpp:2806
llvm::AArch64FrameLowering::restoreCalleeSavedRegisters
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
Definition: AArch64FrameLowering.cpp:2677
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::MachineFrameInfo::getStackProtectorIndex
int getStackProtectorIndex() const
Return the index for the stack protector object.
Definition: MachineFrameInfo.h:357
llvm::TargetFrameLowering::determineCalleeSaves
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
Definition: TargetFrameLoweringImpl.cpp:83
llvm::MachineFrameInfo::hasStackMap
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
Definition: MachineFrameInfo.h:382
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::cl::desc
Definition: CommandLine.h:405
llvm::TargetRegisterInfo::getMinimalPhysRegClass
const TargetRegisterClass * getMinimalPhysRegClass(MCRegister Reg, MVT VT=MVT::Other) const
Returns the Register Class of a physical register of the given type, picking the most sub register cl...
Definition: TargetRegisterInfo.cpp:212
RegisterScavenging.h
llvm::RegState::Kill
@ Kill
The last use of a register.
Definition: MachineInstrBuilder.h:48
llvm::AArch64Subtarget
Definition: AArch64Subtarget.h:38
raw_ostream.h
MachineFunction.h
llvm::printReg
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Definition: TargetRegisterInfo.cpp:111
llvm::AArch64FrameLowering::orderFrameObjects
void orderFrameObjects(const MachineFunction &MF, SmallVectorImpl< int > &ObjectsToAllocate) const override
Order the symbols in the local stack frame.
Definition: AArch64FrameLowering.cpp:3707
llvm::AArch64RegisterInfo::hasBasePointer
bool hasBasePointer(const MachineFunction &MF) const
Definition: AArch64RegisterInfo.cpp:385
llvm::createCFAOffset
MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg, const StackOffset &OffsetFromDefCFA)
Definition: AArch64InstrInfo.cpp:4162
llvm::MachineInstr::eraseFromParent
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Definition: MachineInstr.cpp:650
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::MCAsmInfo::usesWindowsCFI
bool usesWindowsCFI() const
Definition: MCAsmInfo.h:793
llvm::abs
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1281
llvm::HexagonInstrInfo::copyPhysReg
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
Emit instructions to copy a pair of physical registers.
Definition: HexagonInstrInfo.cpp:853
llvm::AArch64II::MO_GOT
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
Definition: AArch64BaseInfo.h:716
TargetRegisterInfo.h
EnableHomogeneousPrologEpilog
cl::opt< bool > EnableHomogeneousPrologEpilog("homogeneous-prolog-epilog", cl::init(false), cl::ZeroOrMore, cl::Hidden, cl::desc("Emit homogeneous prologue and epilogue for the size " "optimization (default = off)"))
Debug.h
needsWinCFI
static bool needsWinCFI(const MachineFunction &MF)
Definition: AArch64FrameLowering.cpp:750
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:279
llvm::SwiftAsyncFramePointerMode::Always
@ Always
Always set the bit.
llvm::AArch64RegisterInfo::isReservedReg
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
Definition: AArch64RegisterInfo.cpp:343
llvm::AArch64InstrInfo::isSEHInstruction
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
Definition: AArch64InstrInfo.cpp:997
llvm::AArch64FunctionInfo::setStackSizeSVE
void setStackSizeSVE(uint64_t S)
Definition: AArch64MachineFunctionInfo.h:204
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:927
getArgumentStackToRestore
static int64_t getArgumentStackToRestore(MachineFunction &MF, MachineBasicBlock &MBB)
Returns how much of the incoming argument stack area (in bytes) we should clean up in an epilogue.
Definition: AArch64FrameLowering.cpp:267
IsSVECalleeSave
static bool IsSVECalleeSave(MachineBasicBlock::iterator I)
Definition: AArch64FrameLowering.cpp:1150
LivePhysRegs.h
llvm::MCCFIInstruction::createRestore
static MCCFIInstruction createRestore(MCSymbol *L, unsigned Register)
.cfi_restore says that the rule for Register is now the same as it was at the beginning of the functi...
Definition: MCDwarf.h:597
llvm::StackOffset::get
static StackOffset get(ScalarTy Fixed, ScalarTy Scalable)
Definition: TypeSize.h:145