LLVM  14.0.0git
AArch64FrameLowering.cpp
Go to the documentation of this file.
1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of TargetFrameLowering class.
10 //
11 // On AArch64, stack frames are structured as follows:
12 //
13 // The stack grows downward.
14 //
15 // All of the individual frame areas on the frame below are optional, i.e. it's
16 // possible to create a function so that the particular area isn't present
17 // in the frame.
18 //
19 // At function entry, the "frame" looks as follows:
20 //
21 // | | Higher address
22 // |-----------------------------------|
23 // | |
24 // | arguments passed on the stack |
25 // | |
26 // |-----------------------------------| <- sp
27 // | | Lower address
28 //
29 //
30 // After the prologue has run, the frame has the following general structure.
31 // Note that this doesn't depict the case where a red-zone is used. Also,
32 // technically the last frame area (VLAs) doesn't get created until in the
33 // main function body, after the prologue is run. However, it's depicted here
34 // for completeness.
35 //
36 // | | Higher address
37 // |-----------------------------------|
38 // | |
39 // | arguments passed on the stack |
40 // | |
41 // |-----------------------------------|
42 // | |
43 // | (Win64 only) varargs from reg |
44 // | |
45 // |-----------------------------------|
46 // | |
47 // | callee-saved gpr registers | <--.
48 // | | | On Darwin platforms these
49 // |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
50 // | prev_lr | | (frame record first)
51 // | prev_fp | <--'
52 // | async context if needed |
53 // | (a.k.a. "frame record") |
54 // |-----------------------------------| <- fp(=x29)
55 // | |
56 // | callee-saved fp/simd/SVE regs |
57 // | |
58 // |-----------------------------------|
59 // | |
60 // | SVE stack objects |
61 // | |
62 // |-----------------------------------|
63 // |.empty.space.to.make.part.below....|
64 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
65 // |.the.standard.16-byte.alignment....| compile time; if present)
66 // |-----------------------------------|
67 // | |
68 // | local variables of fixed size |
69 // | including spill slots |
70 // |-----------------------------------| <- bp(not defined by ABI,
71 // |.variable-sized.local.variables....| LLVM chooses X19)
72 // |.(VLAs)............................| (size of this area is unknown at
73 // |...................................| compile time)
74 // |-----------------------------------| <- sp
75 // | | Lower address
76 //
77 //
78 // To access the data in a frame, at-compile time, a constant offset must be
79 // computable from one of the pointers (fp, bp, sp) to access it. The size
80 // of the areas with a dotted background cannot be computed at compile-time
81 // if they are present, making it required to have all three of fp, bp and
82 // sp to be set up to be able to access all contents in the frame areas,
83 // assuming all of the frame areas are non-empty.
84 //
85 // For most functions, some of the frame areas are empty. For those functions,
86 // it may not be necessary to set up fp or bp:
87 // * A base pointer is definitely needed when there are both VLAs and local
88 // variables with more-than-default alignment requirements.
89 // * A frame pointer is definitely needed when there are local variables with
90 // more-than-default alignment requirements.
91 //
92 // For Darwin platforms the frame-record (fp, lr) is stored at the top of the
93 // callee-saved area, since the unwind encoding does not allow for encoding
94 // this dynamically and existing tools depend on this layout. For other
95 // platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
96 // area to allow SVE stack objects (allocated directly below the callee-saves,
97 // if available) to be accessed directly from the framepointer.
98 // The SVE spill/fill instructions have VL-scaled addressing modes such
99 // as:
100 // ldr z8, [fp, #-7 mul vl]
101 // For SVE the size of the vector length (VL) is not known at compile-time, so
102 // '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
103 // layout, we don't need to add an unscaled offset to the framepointer before
104 // accessing the SVE object in the frame.
105 //
106 // In some cases when a base pointer is not strictly needed, it is generated
107 // anyway when offsets from the frame pointer to access local variables become
108 // so large that the offset can't be encoded in the immediate fields of loads
109 // or stores.
110 //
111 // Outgoing function arguments must be at the bottom of the stack frame when
112 // calling another function. If we do not have variable-sized stack objects, we
113 // can allocate a "reserved call frame" area at the bottom of the local
114 // variable area, large enough for all outgoing calls. If we do have VLAs, then
115 // the stack pointer must be decremented and incremented around each call to
116 // make space for the arguments below the VLAs.
117 //
118 // FIXME: also explain the redzone concept.
119 //
120 //===----------------------------------------------------------------------===//
121 
122 #include "AArch64FrameLowering.h"
123 #include "AArch64InstrInfo.h"
125 #include "AArch64RegisterInfo.h"
126 #include "AArch64Subtarget.h"
127 #include "AArch64TargetMachine.h"
129 #include "llvm/ADT/ScopeExit.h"
130 #include "llvm/ADT/SmallVector.h"
131 #include "llvm/ADT/Statistic.h"
147 #include "llvm/IR/Attributes.h"
148 #include "llvm/IR/CallingConv.h"
149 #include "llvm/IR/DataLayout.h"
150 #include "llvm/IR/DebugLoc.h"
151 #include "llvm/IR/Function.h"
152 #include "llvm/MC/MCAsmInfo.h"
153 #include "llvm/MC/MCDwarf.h"
155 #include "llvm/Support/Debug.h"
157 #include "llvm/Support/LEB128.h"
158 #include "llvm/Support/MathExtras.h"
162 #include <cassert>
163 #include <cstdint>
164 #include <iterator>
165 #include <vector>
166 
167 using namespace llvm;
168 
169 #define DEBUG_TYPE "frame-info"
170 
171 static cl::opt<bool> EnableRedZone("aarch64-redzone",
172  cl::desc("enable use of redzone on AArch64"),
173  cl::init(false), cl::Hidden);
174 
175 static cl::opt<bool>
176  ReverseCSRRestoreSeq("reverse-csr-restore-seq",
177  cl::desc("reverse the CSR restore sequence"),
178  cl::init(false), cl::Hidden);
179 
181  "stack-tagging-merge-settag",
182  cl::desc("merge settag instruction in function epilog"), cl::init(true),
183  cl::Hidden);
184 
185 static cl::opt<bool> OrderFrameObjects("aarch64-order-frame-objects",
186  cl::desc("sort stack allocations"),
187  cl::init(true), cl::Hidden);
188 
190  "homogeneous-prolog-epilog", cl::init(false), cl::ZeroOrMore, cl::Hidden,
191  cl::desc("Emit homogeneous prologue and epilogue for the size "
192  "optimization (default = off)"));
193 
194 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
195 
196 /// Returns how much of the incoming argument stack area (in bytes) we should
197 /// clean up in an epilogue. For the C calling convention this will be 0, for
198 /// guaranteed tail call conventions it can be positive (a normal return or a
199 /// tail call to a function that uses less stack space for arguments) or
200 /// negative (for a tail call to a function that needs more stack space than us
201 /// for arguments).
205  bool IsTailCallReturn = false;
206  if (MBB.end() != MBBI) {
207  unsigned RetOpcode = MBBI->getOpcode();
208  IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
209  RetOpcode == AArch64::TCRETURNri ||
210  RetOpcode == AArch64::TCRETURNriBTI;
211  }
213 
214  int64_t ArgumentPopSize = 0;
215  if (IsTailCallReturn) {
216  MachineOperand &StackAdjust = MBBI->getOperand(1);
217 
218  // For a tail-call in a callee-pops-arguments environment, some or all of
219  // the stack may actually be in use for the call's arguments, this is
220  // calculated during LowerCall and consumed here...
221  ArgumentPopSize = StackAdjust.getImm();
222  } else {
223  // ... otherwise the amount to pop is *all* of the argument space,
224  // conveniently stored in the MachineFunctionInfo by
225  // LowerFormalArguments. This will, of course, be zero for the C calling
226  // convention.
227  ArgumentPopSize = AFI->getArgumentStackToRestore();
228  }
229 
230  return ArgumentPopSize;
231 }
232 
234 static bool needsWinCFI(const MachineFunction &MF);
236 
237 /// Returns true if a homogeneous prolog or epilog code can be emitted
238 /// for the size optimization. If possible, a frame helper call is injected.
239 /// When Exit block is given, this check is for epilog.
240 bool AArch64FrameLowering::homogeneousPrologEpilog(
241  MachineFunction &MF, MachineBasicBlock *Exit) const {
242  if (!MF.getFunction().hasMinSize())
243  return false;
245  return false;
247  return false;
248  if (EnableRedZone)
249  return false;
250 
251  // TODO: Window is supported yet.
252  if (needsWinCFI(MF))
253  return false;
254  // TODO: SVE is not supported yet.
255  if (getSVEStackSize(MF))
256  return false;
257 
258  // Bail on stack adjustment needed on return for simplicity.
259  const MachineFrameInfo &MFI = MF.getFrameInfo();
260  const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
261  if (MFI.hasVarSizedObjects() || RegInfo->hasStackRealignment(MF))
262  return false;
263  if (Exit && getArgumentStackToRestore(MF, *Exit))
264  return false;
265 
266  return true;
267 }
268 
269 /// Returns true if CSRs should be paired.
270 bool AArch64FrameLowering::producePairRegisters(MachineFunction &MF) const {
271  return produceCompactUnwindFrame(MF) || homogeneousPrologEpilog(MF);
272 }
273 
274 /// This is the biggest offset to the stack pointer we can encode in aarch64
275 /// instructions (without using a separate calculation and a temp register).
276 /// Note that the exception here are vector stores/loads which cannot encode any
277 /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
278 static const unsigned DefaultSafeSPDisplacement = 255;
279 
280 /// Look at each instruction that references stack frames and return the stack
281 /// size limit beyond which some of these instructions will require a scratch
282 /// register during their expansion later.
284  // FIXME: For now, just conservatively guestimate based on unscaled indexing
285  // range. We'll end up allocating an unnecessary spill slot a lot, but
286  // realistically that's not a big deal at this stage of the game.
287  for (MachineBasicBlock &MBB : MF) {
288  for (MachineInstr &MI : MBB) {
289  if (MI.isDebugInstr() || MI.isPseudo() ||
290  MI.getOpcode() == AArch64::ADDXri ||
291  MI.getOpcode() == AArch64::ADDSXri)
292  continue;
293 
294  for (const MachineOperand &MO : MI.operands()) {
295  if (!MO.isFI())
296  continue;
297 
299  if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
301  return 0;
302  }
303  }
304  }
306 }
307 
311 }
312 
313 /// Returns the size of the fixed object area (allocated next to sp on entry)
314 /// On Win64 this may include a var args area and an UnwindHelp object for EH.
315 static unsigned getFixedObjectSize(const MachineFunction &MF,
316  const AArch64FunctionInfo *AFI, bool IsWin64,
317  bool IsFunclet) {
318  if (!IsWin64 || IsFunclet) {
319  return AFI->getTailCallReservedStack();
320  } else {
321  if (AFI->getTailCallReservedStack() != 0)
322  report_fatal_error("cannot generate ABI-changing tail call for Win64");
323  // Var args are stored here in the primary function.
324  const unsigned VarArgsArea = AFI->getVarArgsGPRSize();
325  // To support EH funclets we allocate an UnwindHelp object
326  const unsigned UnwindHelpObject = (MF.hasEHFunclets() ? 8 : 0);
327  return alignTo(VarArgsArea + UnwindHelpObject, 16);
328  }
329 }
330 
331 /// Returns the size of the entire SVE stackframe (calleesaves + spills).
334  return StackOffset::getScalable((int64_t)AFI->getStackSizeSVE());
335 }
336 
338  if (!EnableRedZone)
339  return false;
340 
341  // Don't use the red zone if the function explicitly asks us not to.
342  // This is typically used for kernel code.
343  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
344  const unsigned RedZoneSize =
345  Subtarget.getTargetLowering()->getRedZoneSize(MF.getFunction());
346  if (!RedZoneSize)
347  return false;
348 
349  const MachineFrameInfo &MFI = MF.getFrameInfo();
351  uint64_t NumBytes = AFI->getLocalStackSize();
352 
353  return !(MFI.hasCalls() || hasFP(MF) || NumBytes > RedZoneSize ||
354  getSVEStackSize(MF));
355 }
356 
357 /// hasFP - Return true if the specified function should have a dedicated frame
358 /// pointer register.
360  const MachineFrameInfo &MFI = MF.getFrameInfo();
361  const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
362  // Win64 EH requires a frame pointer if funclets are present, as the locals
363  // are accessed off the frame pointer in both the parent function and the
364  // funclets.
365  if (MF.hasEHFunclets())
366  return true;
367  // Retain behavior of always omitting the FP for leaf functions when possible.
369  return true;
370  if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
371  MFI.hasStackMap() || MFI.hasPatchPoint() ||
372  RegInfo->hasStackRealignment(MF))
373  return true;
374  // With large callframes around we may need to use FP to access the scavenging
375  // emergency spillslot.
376  //
377  // Unfortunately some calls to hasFP() like machine verifier ->
378  // getReservedReg() -> hasFP in the middle of global isel are too early
379  // to know the max call frame size. Hopefully conservatively returning "true"
380  // in those cases is fine.
381  // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
382  if (!MFI.isMaxCallFrameSizeComputed() ||
384  return true;
385 
386  return false;
387 }
388 
389 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
390 /// not required, we reserve argument space for call sites in the function
391 /// immediately on entry to the current function. This eliminates the need for
392 /// add/sub sp brackets around call sites. Returns true if the call frame is
393 /// included as part of the stack frame.
394 bool
396  return !MF.getFrameInfo().hasVarSizedObjects();
397 }
398 
402  const AArch64InstrInfo *TII =
403  static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
404  DebugLoc DL = I->getDebugLoc();
405  unsigned Opc = I->getOpcode();
406  bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
407  uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
408 
409  if (!hasReservedCallFrame(MF)) {
410  int64_t Amount = I->getOperand(0).getImm();
411  Amount = alignTo(Amount, getStackAlign());
412  if (!IsDestroy)
413  Amount = -Amount;
414 
415  // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
416  // doesn't have to pop anything), then the first operand will be zero too so
417  // this adjustment is a no-op.
418  if (CalleePopAmount == 0) {
419  // FIXME: in-function stack adjustment for calls is limited to 24-bits
420  // because there's no guaranteed temporary register available.
421  //
422  // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
423  // 1) For offset <= 12-bit, we use LSL #0
424  // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
425  // LSL #0, and the other uses LSL #12.
426  //
427  // Most call frames will be allocated at the start of a function so
428  // this is OK, but it is a limitation that needs dealing with.
429  assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
430  emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
431  StackOffset::getFixed(Amount), TII);
432  }
433  } else if (CalleePopAmount != 0) {
434  // If the calling convention demands that the callee pops arguments from the
435  // stack, we want to add it back if we have a reserved call frame.
436  assert(CalleePopAmount < 0xffffff && "call frame too large");
437  emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
438  StackOffset::getFixed(-(int64_t)CalleePopAmount), TII);
439  }
440  return MBB.erase(I);
441 }
442 
443 // Convenience function to create a DWARF expression for
444 // Expr + NumBytes + NumVGScaledBytes * AArch64::VG
446  int NumBytes, int NumVGScaledBytes, unsigned VG,
447  llvm::raw_string_ostream &Comment) {
448  uint8_t buffer[16];
449 
450  if (NumBytes) {
451  Expr.push_back(dwarf::DW_OP_consts);
452  Expr.append(buffer, buffer + encodeSLEB128(NumBytes, buffer));
453  Expr.push_back((uint8_t)dwarf::DW_OP_plus);
454  Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(NumBytes);
455  }
456 
457  if (NumVGScaledBytes) {
458  Expr.push_back((uint8_t)dwarf::DW_OP_consts);
459  Expr.append(buffer, buffer + encodeSLEB128(NumVGScaledBytes, buffer));
460 
461  Expr.push_back((uint8_t)dwarf::DW_OP_bregx);
462  Expr.append(buffer, buffer + encodeULEB128(VG, buffer));
463  Expr.push_back(0);
464 
465  Expr.push_back((uint8_t)dwarf::DW_OP_mul);
466  Expr.push_back((uint8_t)dwarf::DW_OP_plus);
467 
468  Comment << (NumVGScaledBytes < 0 ? " - " : " + ")
469  << std::abs(NumVGScaledBytes) << " * VG";
470  }
471 }
472 
473 // Creates an MCCFIInstruction:
474 // { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
475 MCCFIInstruction AArch64FrameLowering::createDefCFAExpressionFromSP(
476  const TargetRegisterInfo &TRI, const StackOffset &OffsetFromSP) const {
477  int64_t NumBytes, NumVGScaledBytes;
479  NumVGScaledBytes);
480 
481  std::string CommentBuffer = "sp";
482  llvm::raw_string_ostream Comment(CommentBuffer);
483 
484  // Build up the expression (SP + NumBytes + NumVGScaledBytes * AArch64::VG)
485  SmallString<64> Expr;
486  Expr.push_back((uint8_t)(dwarf::DW_OP_breg0 + /*SP*/ 31));
487  Expr.push_back(0);
488  appendVGScaledOffsetExpr(Expr, NumBytes, NumVGScaledBytes,
489  TRI.getDwarfRegNum(AArch64::VG, true), Comment);
490 
491  // Wrap this into DW_CFA_def_cfa.
492  SmallString<64> DefCfaExpr;
493  DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression);
494  uint8_t buffer[16];
495  DefCfaExpr.append(buffer,
496  buffer + encodeULEB128(Expr.size(), buffer));
497  DefCfaExpr.append(Expr.str());
498  return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(),
499  Comment.str());
500 }
501 
502 MCCFIInstruction AArch64FrameLowering::createCfaOffset(
503  const TargetRegisterInfo &TRI, unsigned Reg,
504  const StackOffset &OffsetFromDefCFA) const {
505  int64_t NumBytes, NumVGScaledBytes;
507  OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
508 
509  unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
510 
511  // Non-scalable offsets can use DW_CFA_offset directly.
512  if (!NumVGScaledBytes)
513  return MCCFIInstruction::createOffset(nullptr, DwarfReg, NumBytes);
514 
515  std::string CommentBuffer;
516  llvm::raw_string_ostream Comment(CommentBuffer);
517  Comment << printReg(Reg, &TRI) << " @ cfa";
518 
519  // Build up expression (NumBytes + NumVGScaledBytes * AArch64::VG)
520  SmallString<64> OffsetExpr;
521  appendVGScaledOffsetExpr(OffsetExpr, NumBytes, NumVGScaledBytes,
522  TRI.getDwarfRegNum(AArch64::VG, true), Comment);
523 
524  // Wrap this into DW_CFA_expression
525  SmallString<64> CfaExpr;
526  CfaExpr.push_back(dwarf::DW_CFA_expression);
527  uint8_t buffer[16];
528  CfaExpr.append(buffer, buffer + encodeULEB128(DwarfReg, buffer));
529  CfaExpr.append(buffer, buffer + encodeULEB128(OffsetExpr.size(), buffer));
530  CfaExpr.append(OffsetExpr.str());
531 
532  return MCCFIInstruction::createEscape(nullptr, CfaExpr.str(), Comment.str());
533 }
534 
537  MachineFunction &MF = *MBB.getParent();
538  MachineFrameInfo &MFI = MF.getFrameInfo();
539  const TargetSubtargetInfo &STI = MF.getSubtarget();
540  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
541  const TargetInstrInfo *TII = STI.getInstrInfo();
543 
544  // Add callee saved registers to move list.
545  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
546  if (CSI.empty())
547  return;
548 
549  for (const auto &Info : CSI) {
550  unsigned Reg = Info.getReg();
551 
552  // Not all unwinders may know about SVE registers, so assume the lowest
553  // common demoninator.
554  unsigned NewReg;
555  if (static_cast<const AArch64RegisterInfo *>(TRI)->regNeedsCFI(Reg, NewReg))
556  Reg = NewReg;
557  else
558  continue;
559 
561  if (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector) {
563  Offset =
564  StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) -
566  } else {
567  Offset = StackOffset::getFixed(MFI.getObjectOffset(Info.getFrameIdx()) -
569  }
570  unsigned CFIIndex = MF.addFrameInst(createCfaOffset(*TRI, Reg, Offset));
571  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
572  .addCFIIndex(CFIIndex)
574  }
575 }
576 
577 // Find a scratch register that we can use at the start of the prologue to
578 // re-align the stack pointer. We avoid using callee-save registers since they
579 // may appear to be free when this is called from canUseAsPrologue (during
580 // shrink wrapping), but then no longer be free when this is called from
581 // emitPrologue.
582 //
583 // FIXME: This is a bit conservative, since in the above case we could use one
584 // of the callee-save registers as a scratch temp to re-align the stack pointer,
585 // but we would then have to make sure that we were in fact saving at least one
586 // callee-save register in the prologue, which is additional complexity that
587 // doesn't seem worth the benefit.
589  MachineFunction *MF = MBB->getParent();
590 
591  // If MBB is an entry block, use X9 as the scratch register
592  if (&MF->front() == MBB)
593  return AArch64::X9;
594 
595  const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
596  const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
597  LivePhysRegs LiveRegs(TRI);
598  LiveRegs.addLiveIns(*MBB);
599 
600  // Mark callee saved registers as used so we will not choose them.
601  const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
602  for (unsigned i = 0; CSRegs[i]; ++i)
603  LiveRegs.addReg(CSRegs[i]);
604 
605  // Prefer X9 since it was historically used for the prologue scratch reg.
606  const MachineRegisterInfo &MRI = MF->getRegInfo();
607  if (LiveRegs.available(MRI, AArch64::X9))
608  return AArch64::X9;
609 
610  for (unsigned Reg : AArch64::GPR64RegClass) {
611  if (LiveRegs.available(MRI, Reg))
612  return Reg;
613  }
614  return AArch64::NoRegister;
615 }
616 
618  const MachineBasicBlock &MBB) const {
619  const MachineFunction *MF = MBB.getParent();
620  MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
621  const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
622  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
623 
624  // Don't need a scratch register if we're not going to re-align the stack.
625  if (!RegInfo->hasStackRealignment(*MF))
626  return true;
627  // Otherwise, we can use any block as long as it has a scratch register
628  // available.
629  return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
630 }
631 
633  uint64_t StackSizeInBytes) {
634  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
635  if (!Subtarget.isTargetWindows())
636  return false;
637  const Function &F = MF.getFunction();
638  // TODO: When implementing stack protectors, take that into account
639  // for the probe threshold.
640  unsigned StackProbeSize = 4096;
641  if (F.hasFnAttribute("stack-probe-size"))
642  F.getFnAttribute("stack-probe-size")
643  .getValueAsString()
644  .getAsInteger(0, StackProbeSize);
645  return (StackSizeInBytes >= StackProbeSize) &&
646  !F.hasFnAttribute("no-stack-arg-probe");
647 }
648 
649 static bool needsWinCFI(const MachineFunction &MF) {
650  const Function &F = MF.getFunction();
651  return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
652  F.needsUnwindTableEntry();
653 }
654 
655 bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
656  MachineFunction &MF, uint64_t StackBumpBytes) const {
658  const MachineFrameInfo &MFI = MF.getFrameInfo();
659  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
660  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
661  if (homogeneousPrologEpilog(MF))
662  return false;
663 
664  if (AFI->getLocalStackSize() == 0)
665  return false;
666 
667  // For WinCFI, if optimizing for size, prefer to not combine the stack bump
668  // (to force a stp with predecrement) to match the packed unwind format,
669  // provided that there actually are any callee saved registers to merge the
670  // decrement with.
671  // This is potentially marginally slower, but allows using the packed
672  // unwind format for functions that both have a local area and callee saved
673  // registers. Using the packed unwind format notably reduces the size of
674  // the unwind info.
675  if (needsWinCFI(MF) && AFI->getCalleeSavedStackSize() > 0 &&
676  MF.getFunction().hasOptSize())
677  return false;
678 
679  // 512 is the maximum immediate for stp/ldp that will be used for
680  // callee-save save/restores
681  if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
682  return false;
683 
684  if (MFI.hasVarSizedObjects())
685  return false;
686 
687  if (RegInfo->hasStackRealignment(MF))
688  return false;
689 
690  // This isn't strictly necessary, but it simplifies things a bit since the
691  // current RedZone handling code assumes the SP is adjusted by the
692  // callee-save save/restore code.
693  if (canUseRedZone(MF))
694  return false;
695 
696  // When there is an SVE area on the stack, always allocate the
697  // callee-saves and spills/locals separately.
698  if (getSVEStackSize(MF))
699  return false;
700 
701  return true;
702 }
703 
704 bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
705  MachineBasicBlock &MBB, unsigned StackBumpBytes) const {
706  if (!shouldCombineCSRLocalStackBump(*MBB.getParent(), StackBumpBytes))
707  return false;
708 
709  if (MBB.empty())
710  return true;
711 
712  // Disable combined SP bump if the last instruction is an MTE tag store. It
713  // is almost always better to merge SP adjustment into those instructions.
716  while (LastI != Begin) {
717  --LastI;
718  if (LastI->isTransient())
719  continue;
720  if (!LastI->getFlag(MachineInstr::FrameDestroy))
721  break;
722  }
723  switch (LastI->getOpcode()) {
724  case AArch64::STGloop:
725  case AArch64::STZGloop:
726  case AArch64::STGOffset:
727  case AArch64::STZGOffset:
728  case AArch64::ST2GOffset:
729  case AArch64::STZ2GOffset:
730  return false;
731  default:
732  return true;
733  }
734  llvm_unreachable("unreachable");
735 }
736 
737 // Given a load or a store instruction, generate an appropriate unwinding SEH
738 // code on Windows.
740  const TargetInstrInfo &TII,
742  unsigned Opc = MBBI->getOpcode();
744  MachineFunction &MF = *MBB->getParent();
745  DebugLoc DL = MBBI->getDebugLoc();
746  unsigned ImmIdx = MBBI->getNumOperands() - 1;
747  int Imm = MBBI->getOperand(ImmIdx).getImm();
749  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
750  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
751 
752  switch (Opc) {
753  default:
754  llvm_unreachable("No SEH Opcode for this instruction");
755  case AArch64::LDPDpost:
756  Imm = -Imm;
758  case AArch64::STPDpre: {
759  unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
760  unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
761  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
762  .addImm(Reg0)
763  .addImm(Reg1)
764  .addImm(Imm * 8)
765  .setMIFlag(Flag);
766  break;
767  }
768  case AArch64::LDPXpost:
769  Imm = -Imm;
771  case AArch64::STPXpre: {
772  Register Reg0 = MBBI->getOperand(1).getReg();
773  Register Reg1 = MBBI->getOperand(2).getReg();
774  if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
775  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
776  .addImm(Imm * 8)
777  .setMIFlag(Flag);
778  else
779  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
780  .addImm(RegInfo->getSEHRegNum(Reg0))
781  .addImm(RegInfo->getSEHRegNum(Reg1))
782  .addImm(Imm * 8)
783  .setMIFlag(Flag);
784  break;
785  }
786  case AArch64::LDRDpost:
787  Imm = -Imm;
789  case AArch64::STRDpre: {
790  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
791  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
792  .addImm(Reg)
793  .addImm(Imm)
794  .setMIFlag(Flag);
795  break;
796  }
797  case AArch64::LDRXpost:
798  Imm = -Imm;
800  case AArch64::STRXpre: {
801  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
802  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
803  .addImm(Reg)
804  .addImm(Imm)
805  .setMIFlag(Flag);
806  break;
807  }
808  case AArch64::STPDi:
809  case AArch64::LDPDi: {
810  unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
811  unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
812  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
813  .addImm(Reg0)
814  .addImm(Reg1)
815  .addImm(Imm * 8)
816  .setMIFlag(Flag);
817  break;
818  }
819  case AArch64::STPXi:
820  case AArch64::LDPXi: {
821  Register Reg0 = MBBI->getOperand(0).getReg();
822  Register Reg1 = MBBI->getOperand(1).getReg();
823  if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
824  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
825  .addImm(Imm * 8)
826  .setMIFlag(Flag);
827  else
828  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
829  .addImm(RegInfo->getSEHRegNum(Reg0))
830  .addImm(RegInfo->getSEHRegNum(Reg1))
831  .addImm(Imm * 8)
832  .setMIFlag(Flag);
833  break;
834  }
835  case AArch64::STRXui:
836  case AArch64::LDRXui: {
837  int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
838  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
839  .addImm(Reg)
840  .addImm(Imm * 8)
841  .setMIFlag(Flag);
842  break;
843  }
844  case AArch64::STRDui:
845  case AArch64::LDRDui: {
846  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
847  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
848  .addImm(Reg)
849  .addImm(Imm * 8)
850  .setMIFlag(Flag);
851  break;
852  }
853  }
854  auto I = MBB->insertAfter(MBBI, MIB);
855  return I;
856 }
857 
858 // Fix up the SEH opcode associated with the save/restore instruction.
860  unsigned LocalStackSize) {
861  MachineOperand *ImmOpnd = nullptr;
862  unsigned ImmIdx = MBBI->getNumOperands() - 1;
863  switch (MBBI->getOpcode()) {
864  default:
865  llvm_unreachable("Fix the offset in the SEH instruction");
866  case AArch64::SEH_SaveFPLR:
867  case AArch64::SEH_SaveRegP:
868  case AArch64::SEH_SaveReg:
869  case AArch64::SEH_SaveFRegP:
870  case AArch64::SEH_SaveFReg:
871  ImmOpnd = &MBBI->getOperand(ImmIdx);
872  break;
873  }
874  if (ImmOpnd)
875  ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
876 }
877 
878 // Convert callee-save register save/restore instruction to do stack pointer
879 // decrement/increment to allocate/deallocate the callee-save stack area by
880 // converting store/load to use pre/post increment version.
883  const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
884  bool NeedsWinCFI, bool *HasWinCFI, bool InProlog = true) {
885  // Ignore instructions that do not operate on SP, i.e. shadow call stack
886  // instructions and associated CFI instruction.
887  while (MBBI->getOpcode() == AArch64::STRXpost ||
888  MBBI->getOpcode() == AArch64::LDRXpre ||
889  MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) {
890  if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION)
891  assert(MBBI->getOperand(0).getReg() != AArch64::SP);
892  ++MBBI;
893  }
894  unsigned NewOpc;
895  switch (MBBI->getOpcode()) {
896  default:
897  llvm_unreachable("Unexpected callee-save save/restore opcode!");
898  case AArch64::STPXi:
899  NewOpc = AArch64::STPXpre;
900  break;
901  case AArch64::STPDi:
902  NewOpc = AArch64::STPDpre;
903  break;
904  case AArch64::STPQi:
905  NewOpc = AArch64::STPQpre;
906  break;
907  case AArch64::STRXui:
908  NewOpc = AArch64::STRXpre;
909  break;
910  case AArch64::STRDui:
911  NewOpc = AArch64::STRDpre;
912  break;
913  case AArch64::STRQui:
914  NewOpc = AArch64::STRQpre;
915  break;
916  case AArch64::LDPXi:
917  NewOpc = AArch64::LDPXpost;
918  break;
919  case AArch64::LDPDi:
920  NewOpc = AArch64::LDPDpost;
921  break;
922  case AArch64::LDPQi:
923  NewOpc = AArch64::LDPQpost;
924  break;
925  case AArch64::LDRXui:
926  NewOpc = AArch64::LDRXpost;
927  break;
928  case AArch64::LDRDui:
929  NewOpc = AArch64::LDRDpost;
930  break;
931  case AArch64::LDRQui:
932  NewOpc = AArch64::LDRQpost;
933  break;
934  }
935  // Get rid of the SEH code associated with the old instruction.
936  if (NeedsWinCFI) {
937  auto SEH = std::next(MBBI);
939  SEH->eraseFromParent();
940  }
941 
942  TypeSize Scale = TypeSize::Fixed(1);
943  unsigned Width;
944  int64_t MinOffset, MaxOffset;
945  bool Success = static_cast<const AArch64InstrInfo *>(TII)->getMemOpInfo(
946  NewOpc, Scale, Width, MinOffset, MaxOffset);
947  (void)Success;
948  assert(Success && "unknown load/store opcode");
949 
950  // If the first store isn't right where we want SP then we can't fold the
951  // update in so create a normal arithmetic instruction instead.
952  if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0 ||
953  CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) {
954  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
955  StackOffset::getFixed(CSStackSizeInc), TII,
956  InProlog ? MachineInstr::FrameSetup
958  return std::prev(MBBI);
959  }
960 
961  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
962  MIB.addReg(AArch64::SP, RegState::Define);
963 
964  // Copy all operands other than the immediate offset.
965  unsigned OpndIdx = 0;
966  for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
967  ++OpndIdx)
968  MIB.add(MBBI->getOperand(OpndIdx));
969 
970  assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
971  "Unexpected immediate offset in first/last callee-save save/restore "
972  "instruction!");
973  assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
974  "Unexpected base register in callee-save save/restore instruction!");
975  assert(CSStackSizeInc % Scale == 0);
976  MIB.addImm(CSStackSizeInc / (int)Scale);
977 
978  MIB.setMIFlags(MBBI->getFlags());
979  MIB.setMemRefs(MBBI->memoperands());
980 
981  // Generate a new SEH code that corresponds to the new instruction.
982  if (NeedsWinCFI) {
983  *HasWinCFI = true;
984  InsertSEH(*MIB, *TII,
986  }
987 
988  return std::prev(MBB.erase(MBBI));
989 }
990 
991 // Fixup callee-save register save/restore instructions to take into account
992 // combined SP bump by adding the local stack size to the stack offsets.
994  uint64_t LocalStackSize,
995  bool NeedsWinCFI,
996  bool *HasWinCFI) {
998  return;
999 
1000  unsigned Opc = MI.getOpcode();
1001 
1002  // Ignore instructions that do not operate on SP, i.e. shadow call stack
1003  // instructions and associated CFI instruction.
1004  if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre ||
1005  Opc == AArch64::CFI_INSTRUCTION) {
1006  if (Opc != AArch64::CFI_INSTRUCTION)
1007  assert(MI.getOperand(0).getReg() != AArch64::SP);
1008  return;
1009  }
1010 
1011  unsigned Scale;
1012  switch (Opc) {
1013  case AArch64::STPXi:
1014  case AArch64::STRXui:
1015  case AArch64::STPDi:
1016  case AArch64::STRDui:
1017  case AArch64::LDPXi:
1018  case AArch64::LDRXui:
1019  case AArch64::LDPDi:
1020  case AArch64::LDRDui:
1021  Scale = 8;
1022  break;
1023  case AArch64::STPQi:
1024  case AArch64::STRQui:
1025  case AArch64::LDPQi:
1026  case AArch64::LDRQui:
1027  Scale = 16;
1028  break;
1029  default:
1030  llvm_unreachable("Unexpected callee-save save/restore opcode!");
1031  }
1032 
1033  unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
1034  assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
1035  "Unexpected base register in callee-save save/restore instruction!");
1036  // Last operand is immediate offset that needs fixing.
1037  MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
1038  // All generated opcodes have scaled offsets.
1039  assert(LocalStackSize % Scale == 0);
1040  OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
1041 
1042  if (NeedsWinCFI) {
1043  *HasWinCFI = true;
1044  auto MBBI = std::next(MachineBasicBlock::iterator(MI));
1045  assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
1047  "Expecting a SEH instruction");
1048  fixupSEHOpcode(MBBI, LocalStackSize);
1049  }
1050 }
1051 
1053  MachineBasicBlock::iterator FirstSPPopI,
1054  MachineBasicBlock::iterator LastPopI) {
1055  // Sometimes (when we restore in the same order as we save), we can end up
1056  // with code like this:
1057  //
1058  // ldp x26, x25, [sp]
1059  // ldp x24, x23, [sp, #16]
1060  // ldp x22, x21, [sp, #32]
1061  // ldp x20, x19, [sp, #48]
1062  // add sp, sp, #64
1063  //
1064  // In this case, it is always better to put the first ldp at the end, so
1065  // that the load-store optimizer can run and merge the ldp and the add into
1066  // a post-index ldp.
1067  // If we managed to grab the first pop instruction, move it to the end.
1069  MBB.splice(FirstSPPopI, &MBB, LastPopI);
1070  // We should end up with something like this now:
1071  //
1072  // ldp x24, x23, [sp, #16]
1073  // ldp x22, x21, [sp, #32]
1074  // ldp x20, x19, [sp, #48]
1075  // ldp x26, x25, [sp]
1076  // add sp, sp, #64
1077  //
1078  // and the load-store optimizer can merge the last two instructions into:
1079  //
1080  // ldp x26, x25, [sp], #64
1081  //
1082 }
1083 
1084 static bool isTargetWindows(const MachineFunction &MF) {
1086 }
1087 
1088 // Convenience function to determine whether I is an SVE callee save.
1090  switch (I->getOpcode()) {
1091  default:
1092  return false;
1093  case AArch64::STR_ZXI:
1094  case AArch64::STR_PXI:
1095  case AArch64::LDR_ZXI:
1096  case AArch64::LDR_PXI:
1097  return I->getFlag(MachineInstr::FrameSetup) ||
1098  I->getFlag(MachineInstr::FrameDestroy);
1099  }
1100 }
1101 
1103  MachineBasicBlock &MBB) const {
1105  const MachineFrameInfo &MFI = MF.getFrameInfo();
1106  const Function &F = MF.getFunction();
1107  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1108  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1109  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1110  MachineModuleInfo &MMI = MF.getMMI();
1112  bool needsFrameMoves =
1114  bool HasFP = hasFP(MF);
1115  bool NeedsWinCFI = needsWinCFI(MF);
1116  bool HasWinCFI = false;
1117  auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); });
1118 
1119  bool IsFunclet = MBB.isEHFuncletEntry();
1120 
1121  // At this point, we're going to decide whether or not the function uses a
1122  // redzone. In most cases, the function doesn't have a redzone so let's
1123  // assume that's false and set it to true in the case that there's a redzone.
1124  AFI->setHasRedZone(false);
1125 
1126  // Debug location must be unknown since the first debug location is used
1127  // to determine the end of the prologue.
1128  DebugLoc DL;
1129 
1130  const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
1131  if (MFnI.shouldSignReturnAddress()) {
1132 
1133  unsigned PACI;
1134  if (MFnI.shouldSignWithBKey()) {
1135  BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
1137  PACI = Subtarget.hasPAuth() ? AArch64::PACIB : AArch64::PACIBSP;
1138  } else {
1139  PACI = Subtarget.hasPAuth() ? AArch64::PACIA : AArch64::PACIASP;
1140  }
1141 
1142  auto MI = BuildMI(MBB, MBBI, DL, TII->get(PACI));
1143  if (Subtarget.hasPAuth())
1144  MI.addReg(AArch64::LR, RegState::Define)
1145  .addReg(AArch64::LR)
1146  .addReg(AArch64::SP, RegState::InternalRead);
1147  MI.setMIFlag(MachineInstr::FrameSetup);
1148 
1149  unsigned CFIIndex =
1151  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1152  .addCFIIndex(CFIIndex)
1154  }
1155 
1156  // We signal the presence of a Swift extended frame to external tools by
1157  // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple
1158  // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI
1159  // bits so that is still true.
1160  if (HasFP && AFI->hasSwiftAsyncContext()) {
1161  switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
1163  if (Subtarget.swiftAsyncContextIsDynamicallySet()) {
1164  // The special symbol below is absolute and has a *value* that can be
1165  // combined with the frame pointer to signal an extended frame.
1166  BuildMI(MBB, MBBI, DL, TII->get(AArch64::LOADgot), AArch64::X16)
1167  .addExternalSymbol("swift_async_extendedFramePointerFlags",
1169  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::FP)
1170  .addUse(AArch64::FP)
1171  .addUse(AArch64::X16)
1172  .addImm(Subtarget.isTargetILP32() ? 32 : 0);
1173  break;
1174  }
1176 
1178  // ORR x29, x29, #0x1000_0000_0000_0000
1179  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP)
1180  .addUse(AArch64::FP)
1181  .addImm(0x1100)
1183  break;
1184 
1186  break;
1187  }
1188  }
1189 
1190  // All calls are tail calls in GHC calling conv, and functions have no
1191  // prologue/epilogue.
1193  return;
1194 
1195  // Set tagged base pointer to the requested stack slot.
1196  // Ideally it should match SP value after prologue.
1198  if (TBPI)
1199  AFI->setTaggedBasePointerOffset(-MFI.getObjectOffset(*TBPI));
1200  else
1202 
1203  const StackOffset &SVEStackSize = getSVEStackSize(MF);
1204 
1205  // getStackSize() includes all the locals in its size calculation. We don't
1206  // include these locals when computing the stack size of a funclet, as they
1207  // are allocated in the parent's stack frame and accessed via the frame
1208  // pointer from the funclet. We only save the callee saved registers in the
1209  // funclet, which are really the callee saved registers of the parent
1210  // function, including the funclet.
1211  int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1212  : MFI.getStackSize();
1213  if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
1214  assert(!HasFP && "unexpected function without stack frame but with FP");
1215  assert(!SVEStackSize &&
1216  "unexpected function without stack frame but with SVE objects");
1217  // All of the stack allocation is for locals.
1218  AFI->setLocalStackSize(NumBytes);
1219  if (!NumBytes)
1220  return;
1221  // REDZONE: If the stack size is less than 128 bytes, we don't need
1222  // to actually allocate.
1223  if (canUseRedZone(MF)) {
1224  AFI->setHasRedZone(true);
1225  ++NumRedZoneFunctions;
1226  } else {
1227  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1228  StackOffset::getFixed(-NumBytes), TII,
1229  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
1230  if (needsFrameMoves) {
1231  // Label used to tie together the PROLOG_LABEL and the MachineMoves.
1232  MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
1233  // Encode the stack size of the leaf function.
1234  unsigned CFIIndex = MF.addFrameInst(
1235  MCCFIInstruction::cfiDefCfaOffset(FrameLabel, NumBytes));
1236  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1237  .addCFIIndex(CFIIndex)
1239  }
1240  }
1241 
1242  if (NeedsWinCFI) {
1243  HasWinCFI = true;
1244  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1246  }
1247 
1248  return;
1249  }
1250 
1251  bool IsWin64 =
1252  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1253  unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1254 
1255  auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1256  // All of the remaining stack allocations are for locals.
1257  AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1258  bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1259  bool HomPrologEpilog = homogeneousPrologEpilog(MF);
1260  if (CombineSPBump) {
1261  assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1262  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1263  StackOffset::getFixed(-NumBytes), TII,
1264  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
1265  NumBytes = 0;
1266  } else if (HomPrologEpilog) {
1267  // Stack has been already adjusted.
1268  NumBytes -= PrologueSaveSize;
1269  } else if (PrologueSaveSize != 0) {
1271  MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI);
1272  NumBytes -= PrologueSaveSize;
1273  }
1274  assert(NumBytes >= 0 && "Negative stack allocation size!?");
1275 
1276  // Move past the saves of the callee-saved registers, fixing up the offsets
1277  // and pre-inc if we decided to combine the callee-save and local stack
1278  // pointer bump above.
1280  while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) &&
1281  !IsSVECalleeSave(MBBI)) {
1282  if (CombineSPBump)
1284  NeedsWinCFI, &HasWinCFI);
1285  ++MBBI;
1286  }
1287 
1288  // For funclets the FP belongs to the containing function.
1289  if (!IsFunclet && HasFP) {
1290  // Only set up FP if we actually need to.
1291  int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset();
1292 
1293  if (CombineSPBump)
1294  FPOffset += AFI->getLocalStackSize();
1295 
1296  if (AFI->hasSwiftAsyncContext()) {
1297  // Before we update the live FP we have to ensure there's a valid (or
1298  // null) asynchronous context in its slot just before FP in the frame
1299  // record, so store it now.
1300  const auto &Attrs = MF.getFunction().getAttributes();
1301  bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync);
1302  if (HaveInitialContext)
1303  MBB.addLiveIn(AArch64::X22);
1304  BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext))
1305  .addUse(HaveInitialContext ? AArch64::X22 : AArch64::XZR)
1306  .addUse(AArch64::SP)
1307  .addImm(FPOffset - 8)
1309  }
1310 
1311  if (HomPrologEpilog) {
1312  auto Prolog = MBBI;
1313  --Prolog;
1314  assert(Prolog->getOpcode() == AArch64::HOM_Prolog);
1315  Prolog->addOperand(MachineOperand::CreateImm(FPOffset));
1316  } else {
1317  // Issue sub fp, sp, FPOffset or
1318  // mov fp,sp when FPOffset is zero.
1319  // Note: All stores of callee-saved registers are marked as "FrameSetup".
1320  // This code marks the instruction(s) that set the FP also.
1321  emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP,
1322  StackOffset::getFixed(FPOffset), TII,
1323  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
1324  }
1325  }
1326 
1327  if (windowsRequiresStackProbe(MF, NumBytes)) {
1328  uint64_t NumWords = NumBytes >> 4;
1329  if (NeedsWinCFI) {
1330  HasWinCFI = true;
1331  // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
1332  // exceed this amount. We need to move at most 2^24 - 1 into x15.
1333  // This is at most two instructions, MOVZ follwed by MOVK.
1334  // TODO: Fix to use multiple stack alloc unwind codes for stacks
1335  // exceeding 256MB in size.
1336  if (NumBytes >= (1 << 28))
1337  report_fatal_error("Stack size cannot exceed 256MB for stack "
1338  "unwinding purposes");
1339 
1340  uint32_t LowNumWords = NumWords & 0xFFFF;
1341  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
1342  .addImm(LowNumWords)
1345  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1347  if ((NumWords & 0xFFFF0000) != 0) {
1348  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
1349  .addReg(AArch64::X15)
1350  .addImm((NumWords & 0xFFFF0000) >> 16) // High half
1353  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1355  }
1356  } else {
1357  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
1358  .addImm(NumWords)
1360  }
1361 
1362  switch (MF.getTarget().getCodeModel()) {
1363  case CodeModel::Tiny:
1364  case CodeModel::Small:
1365  case CodeModel::Medium:
1366  case CodeModel::Kernel:
1367  BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
1368  .addExternalSymbol("__chkstk")
1369  .addReg(AArch64::X15, RegState::Implicit)
1374  if (NeedsWinCFI) {
1375  HasWinCFI = true;
1376  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1378  }
1379  break;
1380  case CodeModel::Large:
1381  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
1382  .addReg(AArch64::X16, RegState::Define)
1383  .addExternalSymbol("__chkstk")
1384  .addExternalSymbol("__chkstk")
1386  if (NeedsWinCFI) {
1387  HasWinCFI = true;
1388  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1390  }
1391 
1392  BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF)))
1393  .addReg(AArch64::X16, RegState::Kill)
1394  .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
1399  if (NeedsWinCFI) {
1400  HasWinCFI = true;
1401  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1403  }
1404  break;
1405  }
1406 
1407  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
1408  .addReg(AArch64::SP, RegState::Kill)
1409  .addReg(AArch64::X15, RegState::Kill)
1412  if (NeedsWinCFI) {
1413  HasWinCFI = true;
1414  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1415  .addImm(NumBytes)
1417  }
1418  NumBytes = 0;
1419  }
1420 
1421  StackOffset AllocateBefore = SVEStackSize, AllocateAfter = {};
1422  MachineBasicBlock::iterator CalleeSavesBegin = MBBI, CalleeSavesEnd = MBBI;
1423 
1424  // Process the SVE callee-saves to determine what space needs to be
1425  // allocated.
1426  if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
1427  // Find callee save instructions in frame.
1428  CalleeSavesBegin = MBBI;
1429  assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
1431  ++MBBI;
1432  CalleeSavesEnd = MBBI;
1433 
1434  AllocateBefore = StackOffset::getScalable(CalleeSavedSize);
1435  AllocateAfter = SVEStackSize - AllocateBefore;
1436  }
1437 
1438  // Allocate space for the callee saves (if any).
1439  emitFrameOffset(MBB, CalleeSavesBegin, DL, AArch64::SP, AArch64::SP,
1440  -AllocateBefore, TII,
1442 
1443  // Finally allocate remaining SVE stack space.
1444  emitFrameOffset(MBB, CalleeSavesEnd, DL, AArch64::SP, AArch64::SP,
1445  -AllocateAfter, TII,
1447 
1448  // Allocate space for the rest of the frame.
1449  if (NumBytes) {
1450  // Alignment is required for the parent frame, not the funclet
1451  const bool NeedsRealignment =
1452  !IsFunclet && RegInfo->hasStackRealignment(MF);
1453  unsigned scratchSPReg = AArch64::SP;
1454 
1455  if (NeedsRealignment) {
1456  scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
1457  assert(scratchSPReg != AArch64::NoRegister);
1458  }
1459 
1460  // If we're a leaf function, try using the red zone.
1461  if (!canUseRedZone(MF))
1462  // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
1463  // the correct value here, as NumBytes also includes padding bytes,
1464  // which shouldn't be counted here.
1465  emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP,
1466  StackOffset::getFixed(-NumBytes), TII,
1467  MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
1468 
1469  if (NeedsRealignment) {
1470  const unsigned NrBitsToZero = Log2(MFI.getMaxAlign());
1471  assert(NrBitsToZero > 1);
1472  assert(scratchSPReg != AArch64::SP);
1473 
1474  // SUB X9, SP, NumBytes
1475  // -- X9 is temporary register, so shouldn't contain any live data here,
1476  // -- free to use. This is already produced by emitFrameOffset above.
1477  // AND SP, X9, 0b11111...0000
1478  // The logical immediates have a non-trivial encoding. The following
1479  // formula computes the encoded immediate with all ones but
1480  // NrBitsToZero zero bits as least significant bits.
1481  uint32_t andMaskEncoded = (1 << 12) // = N
1482  | ((64 - NrBitsToZero) << 6) // immr
1483  | ((64 - NrBitsToZero - 1) << 0); // imms
1484 
1485  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
1486  .addReg(scratchSPReg, RegState::Kill)
1487  .addImm(andMaskEncoded);
1488  AFI->setStackRealigned(true);
1489  if (NeedsWinCFI) {
1490  HasWinCFI = true;
1491  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1492  .addImm(NumBytes & andMaskEncoded)
1494  }
1495  }
1496  }
1497 
1498  // If we need a base pointer, set it up here. It's whatever the value of the
1499  // stack pointer is at this point. Any variable size objects will be allocated
1500  // after this, so we can still use the base pointer to reference locals.
1501  //
1502  // FIXME: Clarify FrameSetup flags here.
1503  // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
1504  // needed.
1505  // For funclets the BP belongs to the containing function.
1506  if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
1507  TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
1508  false);
1509  if (NeedsWinCFI) {
1510  HasWinCFI = true;
1511  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1513  }
1514  }
1515 
1516  // The very last FrameSetup instruction indicates the end of prologue. Emit a
1517  // SEH opcode indicating the prologue end.
1518  if (NeedsWinCFI && HasWinCFI) {
1519  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1521  }
1522 
1523  // SEH funclets are passed the frame pointer in X1. If the parent
1524  // function uses the base register, then the base register is used
1525  // directly, and is not retrieved from X1.
1526  if (IsFunclet && F.hasPersonalityFn()) {
1527  EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
1528  if (isAsynchronousEHPersonality(Per)) {
1529  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
1530  .addReg(AArch64::X1)
1532  MBB.addLiveIn(AArch64::X1);
1533  }
1534  }
1535 
1536  if (needsFrameMoves) {
1537  // An example of the prologue:
1538  //
1539  // .globl __foo
1540  // .align 2
1541  // __foo:
1542  // Ltmp0:
1543  // .cfi_startproc
1544  // .cfi_personality 155, ___gxx_personality_v0
1545  // Leh_func_begin:
1546  // .cfi_lsda 16, Lexception33
1547  //
1548  // stp xa,bx, [sp, -#offset]!
1549  // ...
1550  // stp x28, x27, [sp, #offset-32]
1551  // stp fp, lr, [sp, #offset-16]
1552  // add fp, sp, #offset - 16
1553  // sub sp, sp, #1360
1554  //
1555  // The Stack:
1556  // +-------------------------------------------+
1557  // 10000 | ........ | ........ | ........ | ........ |
1558  // 10004 | ........ | ........ | ........ | ........ |
1559  // +-------------------------------------------+
1560  // 10008 | ........ | ........ | ........ | ........ |
1561  // 1000c | ........ | ........ | ........ | ........ |
1562  // +===========================================+
1563  // 10010 | X28 Register |
1564  // 10014 | X28 Register |
1565  // +-------------------------------------------+
1566  // 10018 | X27 Register |
1567  // 1001c | X27 Register |
1568  // +===========================================+
1569  // 10020 | Frame Pointer |
1570  // 10024 | Frame Pointer |
1571  // +-------------------------------------------+
1572  // 10028 | Link Register |
1573  // 1002c | Link Register |
1574  // +===========================================+
1575  // 10030 | ........ | ........ | ........ | ........ |
1576  // 10034 | ........ | ........ | ........ | ........ |
1577  // +-------------------------------------------+
1578  // 10038 | ........ | ........ | ........ | ........ |
1579  // 1003c | ........ | ........ | ........ | ........ |
1580  // +-------------------------------------------+
1581  //
1582  // [sp] = 10030 :: >>initial value<<
1583  // sp = 10020 :: stp fp, lr, [sp, #-16]!
1584  // fp = sp == 10020 :: mov fp, sp
1585  // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
1586  // sp == 10010 :: >>final value<<
1587  //
1588  // The frame pointer (w29) points to address 10020. If we use an offset of
1589  // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
1590  // for w27, and -32 for w28:
1591  //
1592  // Ltmp1:
1593  // .cfi_def_cfa w29, 16
1594  // Ltmp2:
1595  // .cfi_offset w30, -8
1596  // Ltmp3:
1597  // .cfi_offset w29, -16
1598  // Ltmp4:
1599  // .cfi_offset w27, -24
1600  // Ltmp5:
1601  // .cfi_offset w28, -32
1602 
1603  if (HasFP) {
1604  const int OffsetToFirstCalleeSaveFromFP =
1606  AFI->getCalleeSavedStackSize();
1607  Register FramePtr = RegInfo->getFrameRegister(MF);
1608 
1609  // Define the current CFA rule to use the provided FP.
1610  unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
1611  unsigned CFIIndex = MF.addFrameInst(
1612  MCCFIInstruction::cfiDefCfa(nullptr, Reg, FixedObject - OffsetToFirstCalleeSaveFromFP));
1613  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1614  .addCFIIndex(CFIIndex)
1616  } else {
1617  unsigned CFIIndex;
1618  if (SVEStackSize) {
1619  const TargetSubtargetInfo &STI = MF.getSubtarget();
1620  const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
1621  StackOffset TotalSize =
1622  SVEStackSize + StackOffset::getFixed((int64_t)MFI.getStackSize());
1623  CFIIndex = MF.addFrameInst(createDefCFAExpressionFromSP(TRI, TotalSize));
1624  } else {
1625  // Encode the stack size of the leaf function.
1626  CFIIndex = MF.addFrameInst(
1628  }
1629  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1630  .addCFIIndex(CFIIndex)
1632  }
1633 
1634  // Now emit the moves for whatever callee saved regs we have (including FP,
1635  // LR if those are saved).
1637  }
1638 }
1639 
1642  const auto &MFI = *MF.getInfo<AArch64FunctionInfo>();
1643  if (!MFI.shouldSignReturnAddress())
1644  return;
1645  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1646  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1647 
1649  DebugLoc DL;
1650  if (MBBI != MBB.end())
1651  DL = MBBI->getDebugLoc();
1652 
1653  // The AUTIASP instruction assembles to a hint instruction before v8.3a so
1654  // this instruction can safely used for any v8a architecture.
1655  // From v8.3a onwards there are optimised authenticate LR and return
1656  // instructions, namely RETA{A,B}, that can be used instead.
1657  if (Subtarget.hasPAuth() && MBBI != MBB.end() &&
1658  MBBI->getOpcode() == AArch64::RET_ReallyLR) {
1659  BuildMI(MBB, MBBI, DL,
1660  TII->get(MFI.shouldSignWithBKey() ? AArch64::RETAB : AArch64::RETAA))
1661  .copyImplicitOps(*MBBI);
1662  MBB.erase(MBBI);
1663  } else {
1664  BuildMI(
1665  MBB, MBBI, DL,
1666  TII->get(MFI.shouldSignWithBKey() ? AArch64::AUTIBSP : AArch64::AUTIASP))
1668  }
1669 }
1670 
1671 static bool isFuncletReturnInstr(const MachineInstr &MI) {
1672  switch (MI.getOpcode()) {
1673  default:
1674  return false;
1675  case AArch64::CATCHRET:
1676  case AArch64::CLEANUPRET:
1677  return true;
1678  }
1679 }
1680 
1682  MachineBasicBlock &MBB) const {
1684  MachineFrameInfo &MFI = MF.getFrameInfo();
1685  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1686  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1687  DebugLoc DL;
1688  bool NeedsWinCFI = needsWinCFI(MF);
1689  bool HasWinCFI = false;
1690  bool IsFunclet = false;
1691  auto WinCFI = make_scope_exit([&]() { assert(HasWinCFI == MF.hasWinCFI()); });
1692 
1693  if (MBB.end() != MBBI) {
1694  DL = MBBI->getDebugLoc();
1695  IsFunclet = isFuncletReturnInstr(*MBBI);
1696  }
1697 
1698  int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1699  : MFI.getStackSize();
1701 
1702  // All calls are tail calls in GHC calling conv, and functions have no
1703  // prologue/epilogue.
1705  return;
1706 
1707  // How much of the stack used by incoming arguments this function is expected
1708  // to restore in this particular epilogue.
1709  int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB);
1710 
1711  // The stack frame should be like below,
1712  //
1713  // ---------------------- ---
1714  // | | |
1715  // | BytesInStackArgArea| CalleeArgStackSize
1716  // | (NumReusableBytes) | (of tail call)
1717  // | | ---
1718  // | | |
1719  // ---------------------| --- |
1720  // | | | |
1721  // | CalleeSavedReg | | |
1722  // | (CalleeSavedStackSize)| | |
1723  // | | | |
1724  // ---------------------| | NumBytes
1725  // | | StackSize (StackAdjustUp)
1726  // | LocalStackSize | | |
1727  // | (covering callee | | |
1728  // | args) | | |
1729  // | | | |
1730  // ---------------------- --- ---
1731  //
1732  // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
1733  // = StackSize + ArgumentPopSize
1734  //
1735  // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
1736  // it as the 2nd argument of AArch64ISD::TC_RETURN.
1737 
1738  auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
1739 
1740  bool IsWin64 =
1741  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1742  unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1743 
1744  int64_t AfterCSRPopSize = ArgumentStackToRestore;
1745  auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1746  // We cannot rely on the local stack size set in emitPrologue if the function
1747  // has funclets, as funclets have different local stack size requirements, and
1748  // the current value set in emitPrologue may be that of the containing
1749  // function.
1750  if (MF.hasEHFunclets())
1751  AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1752  if (homogeneousPrologEpilog(MF, &MBB)) {
1753  assert(!NeedsWinCFI);
1754  auto LastPopI = MBB.getFirstTerminator();
1755  if (LastPopI != MBB.begin()) {
1756  auto HomogeneousEpilog = std::prev(LastPopI);
1757  if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog)
1758  LastPopI = HomogeneousEpilog;
1759  }
1760 
1761  // Adjust local stack
1762  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1764  MachineInstr::FrameDestroy, false, NeedsWinCFI);
1765 
1766  // SP has been already adjusted while restoring callee save regs.
1767  // We've bailed-out the case with adjusting SP for arguments.
1768  assert(AfterCSRPopSize == 0);
1769  return;
1770  }
1771  bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
1772  // Assume we can't combine the last pop with the sp restore.
1773 
1774  if (!CombineSPBump && PrologueSaveSize != 0) {
1777  Pop = std::prev(Pop);
1778  // Converting the last ldp to a post-index ldp is valid only if the last
1779  // ldp's offset is 0.
1780  const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
1781  // If the offset is 0 and the AfterCSR pop is not actually trying to
1782  // allocate more stack for arguments (in space that an untimely interrupt
1783  // may clobber), convert it to a post-index ldp.
1784  if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0)
1786  MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, false);
1787  else {
1788  // If not, make sure to emit an add after the last ldp.
1789  // We're doing this by transfering the size to be restored from the
1790  // adjustment *before* the CSR pops to the adjustment *after* the CSR
1791  // pops.
1792  AfterCSRPopSize += PrologueSaveSize;
1793  }
1794  }
1795 
1796  // Move past the restores of the callee-saved registers.
1797  // If we plan on combining the sp bump of the local stack size and the callee
1798  // save stack size, we might need to adjust the CSR save and restore offsets.
1801  while (LastPopI != Begin) {
1802  --LastPopI;
1803  if (!LastPopI->getFlag(MachineInstr::FrameDestroy) ||
1804  IsSVECalleeSave(LastPopI)) {
1805  ++LastPopI;
1806  break;
1807  } else if (CombineSPBump)
1809  NeedsWinCFI, &HasWinCFI);
1810  }
1811 
1812  if (MF.hasWinCFI()) {
1813  // If the prologue didn't contain any SEH opcodes and didn't set the
1814  // MF.hasWinCFI() flag, assume the epilogue won't either, and skip the
1815  // EpilogStart - to avoid generating CFI for functions that don't need it.
1816  // (And as we didn't generate any prologue at all, it would be asymmetrical
1817  // to the epilogue.) By the end of the function, we assert that
1818  // HasWinCFI is equal to MF.hasWinCFI(), to verify this assumption.
1819  HasWinCFI = true;
1820  BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
1822  }
1823 
1824  if (hasFP(MF) && AFI->hasSwiftAsyncContext()) {
1825  // We need to reset FP to its untagged state on return. Bit 60 is currently
1826  // used to show the presence of an extended frame.
1827 
1828  // BIC x29, x29, #0x1000_0000_0000_0000
1829  BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::ANDXri),
1830  AArch64::FP)
1831  .addUse(AArch64::FP)
1832  .addImm(0x10fe)
1834  }
1835 
1836  const StackOffset &SVEStackSize = getSVEStackSize(MF);
1837 
1838  // If there is a single SP update, insert it before the ret and we're done.
1839  if (CombineSPBump) {
1840  assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1841  emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
1842  StackOffset::getFixed(NumBytes + (int64_t)AfterCSRPopSize),
1843  TII, MachineInstr::FrameDestroy, false, NeedsWinCFI,
1844  &HasWinCFI);
1845  if (HasWinCFI)
1847  TII->get(AArch64::SEH_EpilogEnd))
1849  return;
1850  }
1851 
1852  NumBytes -= PrologueSaveSize;
1853  assert(NumBytes >= 0 && "Negative stack allocation size!?");
1854 
1855  // Process the SVE callee-saves to determine what space needs to be
1856  // deallocated.
1857  StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
1858  MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
1859  if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
1860  RestoreBegin = std::prev(RestoreEnd);
1861  while (RestoreBegin != MBB.begin() &&
1862  IsSVECalleeSave(std::prev(RestoreBegin)))
1863  --RestoreBegin;
1864 
1865  assert(IsSVECalleeSave(RestoreBegin) &&
1866  IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
1867 
1868  StackOffset CalleeSavedSizeAsOffset =
1869  StackOffset::getScalable(CalleeSavedSize);
1870  DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
1871  DeallocateAfter = CalleeSavedSizeAsOffset;
1872  }
1873 
1874  // Deallocate the SVE area.
1875  if (SVEStackSize) {
1876  if (AFI->isStackRealigned()) {
1877  if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize())
1878  // Set SP to start of SVE callee-save area from which they can
1879  // be reloaded. The code below will deallocate the stack space
1880  // space by moving FP -> SP.
1881  emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
1882  StackOffset::getScalable(-CalleeSavedSize), TII,
1884  } else {
1885  if (AFI->getSVECalleeSavedStackSize()) {
1886  // Deallocate the non-SVE locals first before we can deallocate (and
1887  // restore callee saves) from the SVE area.
1888  emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
1889  StackOffset::getFixed(NumBytes), TII,
1891  NumBytes = 0;
1892  }
1893 
1894  emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
1895  DeallocateBefore, TII, MachineInstr::FrameDestroy);
1896 
1897  emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
1898  DeallocateAfter, TII, MachineInstr::FrameDestroy);
1899  }
1900  }
1901 
1902  if (!hasFP(MF)) {
1903  bool RedZone = canUseRedZone(MF);
1904  // If this was a redzone leaf function, we don't need to restore the
1905  // stack pointer (but we may need to pop stack args for fastcc).
1906  if (RedZone && AfterCSRPopSize == 0)
1907  return;
1908 
1909  bool NoCalleeSaveRestore = PrologueSaveSize == 0;
1910  int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
1911  if (NoCalleeSaveRestore)
1912  StackRestoreBytes += AfterCSRPopSize;
1913 
1914  // If we were able to combine the local stack pop with the argument pop,
1915  // then we're done.
1916  bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
1917 
1918  // If we're done after this, make sure to help the load store optimizer.
1919  if (Done)
1920  adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
1921 
1922  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1923  StackOffset::getFixed(StackRestoreBytes), TII,
1924  MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1925  if (Done) {
1926  if (HasWinCFI) {
1928  TII->get(AArch64::SEH_EpilogEnd))
1930  }
1931  return;
1932  }
1933 
1934  NumBytes = 0;
1935  }
1936 
1937  // Restore the original stack pointer.
1938  // FIXME: Rather than doing the math here, we should instead just use
1939  // non-post-indexed loads for the restores if we aren't actually going to
1940  // be able to save any instructions.
1941  if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
1943  MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
1945  TII, MachineInstr::FrameDestroy, false, NeedsWinCFI);
1946  } else if (NumBytes)
1947  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1948  StackOffset::getFixed(NumBytes), TII,
1949  MachineInstr::FrameDestroy, false, NeedsWinCFI);
1950 
1951  // This must be placed after the callee-save restore code because that code
1952  // assumes the SP is at the same location as it was after the callee-save save
1953  // code in the prologue.
1954  if (AfterCSRPopSize) {
1955  assert(AfterCSRPopSize > 0 && "attempting to reallocate arg stack that an "
1956  "interrupt may have clobbered");
1957  // Find an insertion point for the first ldp so that it goes before the
1958  // shadow call stack epilog instruction. This ensures that the restore of
1959  // lr from x18 is placed after the restore from sp.
1960  auto FirstSPPopI = MBB.getFirstTerminator();
1961  while (FirstSPPopI != Begin) {
1962  auto Prev = std::prev(FirstSPPopI);
1963  if (Prev->getOpcode() != AArch64::LDRXpre ||
1964  Prev->getOperand(0).getReg() == AArch64::SP)
1965  break;
1966  FirstSPPopI = Prev;
1967  }
1968 
1969  adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
1970 
1971  emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
1972  StackOffset::getFixed(AfterCSRPopSize), TII,
1973  MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1974  }
1975  if (HasWinCFI)
1976  BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
1978 }
1979 
1980 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1981 /// debug info. It's the same as what we use for resolving the code-gen
1982 /// references for now. FIXME: This can go wrong when references are
1983 /// SP-relative and simple call frames aren't used.
1986  Register &FrameReg) const {
1988  MF, FI, FrameReg,
1989  /*PreferFP=*/
1990  MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress),
1991  /*ForSimm=*/false);
1992 }
1993 
1996  int FI) const {
1998 }
1999 
2001  int64_t ObjectOffset) {
2002  const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2003  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2004  bool IsWin64 =
2005  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
2006  unsigned FixedObject =
2007  getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
2008  int64_t CalleeSaveSize = AFI->getCalleeSavedStackSize(MF.getFrameInfo());
2009  int64_t FPAdjust =
2010  CalleeSaveSize - AFI->getCalleeSaveBaseToFrameRecordOffset();
2011  return StackOffset::getFixed(ObjectOffset + FixedObject + FPAdjust);
2012 }
2013 
2015  int64_t ObjectOffset) {
2016  const auto &MFI = MF.getFrameInfo();
2017  return StackOffset::getFixed(ObjectOffset + (int64_t)MFI.getStackSize());
2018 }
2019 
2020  // TODO: This function currently does not work for scalable vectors.
2022  int FI) const {
2023  const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
2024  MF.getSubtarget().getRegisterInfo());
2025  int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI);
2026  return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
2027  ? getFPOffset(MF, ObjectOffset).getFixed()
2028  : getStackOffset(MF, ObjectOffset).getFixed();
2029 }
2030 
2032  const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP,
2033  bool ForSimm) const {
2034  const auto &MFI = MF.getFrameInfo();
2035  int64_t ObjectOffset = MFI.getObjectOffset(FI);
2036  bool isFixed = MFI.isFixedObjectIndex(FI);
2037  bool isSVE = MFI.getStackID(FI) == TargetStackID::ScalableVector;
2038  return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
2039  PreferFP, ForSimm);
2040 }
2041 
2043  const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
2044  Register &FrameReg, bool PreferFP, bool ForSimm) const {
2045  const auto &MFI = MF.getFrameInfo();
2046  const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
2047  MF.getSubtarget().getRegisterInfo());
2048  const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2049  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2050 
2051  int64_t FPOffset = getFPOffset(MF, ObjectOffset).getFixed();
2052  int64_t Offset = getStackOffset(MF, ObjectOffset).getFixed();
2053  bool isCSR =
2054  !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
2055 
2056  const StackOffset &SVEStackSize = getSVEStackSize(MF);
2057 
2058  // Use frame pointer to reference fixed objects. Use it for locals if
2059  // there are VLAs or a dynamically realigned SP (and thus the SP isn't
2060  // reliable as a base). Make sure useFPForScavengingIndex() does the
2061  // right thing for the emergency spill slot.
2062  bool UseFP = false;
2063  if (AFI->hasStackFrame() && !isSVE) {
2064  // We shouldn't prefer using the FP when there is an SVE area
2065  // in between the FP and the non-SVE locals/spills.
2066  PreferFP &= !SVEStackSize;
2067 
2068  // Note: Keeping the following as multiple 'if' statements rather than
2069  // merging to a single expression for readability.
2070  //
2071  // Argument access should always use the FP.
2072  if (isFixed) {
2073  UseFP = hasFP(MF);
2074  } else if (isCSR && RegInfo->hasStackRealignment(MF)) {
2075  // References to the CSR area must use FP if we're re-aligning the stack
2076  // since the dynamically-sized alignment padding is between the SP/BP and
2077  // the CSR area.
2078  assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
2079  UseFP = true;
2080  } else if (hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
2081  // If the FPOffset is negative and we're producing a signed immediate, we
2082  // have to keep in mind that the available offset range for negative
2083  // offsets is smaller than for positive ones. If an offset is available
2084  // via the FP and the SP, use whichever is closest.
2085  bool FPOffsetFits = !ForSimm || FPOffset >= -256;
2086  PreferFP |= Offset > -FPOffset;
2087 
2088  if (MFI.hasVarSizedObjects()) {
2089  // If we have variable sized objects, we can use either FP or BP, as the
2090  // SP offset is unknown. We can use the base pointer if we have one and
2091  // FP is not preferred. If not, we're stuck with using FP.
2092  bool CanUseBP = RegInfo->hasBasePointer(MF);
2093  if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
2094  UseFP = PreferFP;
2095  else if (!CanUseBP) // Can't use BP. Forced to use FP.
2096  UseFP = true;
2097  // else we can use BP and FP, but the offset from FP won't fit.
2098  // That will make us scavenge registers which we can probably avoid by
2099  // using BP. If it won't fit for BP either, we'll scavenge anyway.
2100  } else if (FPOffset >= 0) {
2101  // Use SP or FP, whichever gives us the best chance of the offset
2102  // being in range for direct access. If the FPOffset is positive,
2103  // that'll always be best, as the SP will be even further away.
2104  UseFP = true;
2105  } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
2106  // Funclets access the locals contained in the parent's stack frame
2107  // via the frame pointer, so we have to use the FP in the parent
2108  // function.
2109  (void) Subtarget;
2110  assert(
2111  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
2112  "Funclets should only be present on Win64");
2113  UseFP = true;
2114  } else {
2115  // We have the choice between FP and (SP or BP).
2116  if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
2117  UseFP = true;
2118  }
2119  }
2120  }
2121 
2122  assert(
2123  ((isFixed || isCSR) || !RegInfo->hasStackRealignment(MF) || !UseFP) &&
2124  "In the presence of dynamic stack pointer realignment, "
2125  "non-argument/CSR objects cannot be accessed through the frame pointer");
2126 
2127  if (isSVE) {
2128  StackOffset FPOffset =
2130  StackOffset SPOffset =
2131  SVEStackSize +
2132  StackOffset::get(MFI.getStackSize() - AFI->getCalleeSavedStackSize(),
2133  ObjectOffset);
2134  // Always use the FP for SVE spills if available and beneficial.
2135  if (hasFP(MF) && (SPOffset.getFixed() ||
2136  FPOffset.getScalable() < SPOffset.getScalable() ||
2137  RegInfo->hasStackRealignment(MF))) {
2138  FrameReg = RegInfo->getFrameRegister(MF);
2139  return FPOffset;
2140  }
2141 
2142  FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
2143  : (unsigned)AArch64::SP;
2144  return SPOffset;
2145  }
2146 
2147  StackOffset ScalableOffset = {};
2148  if (UseFP && !(isFixed || isCSR))
2149  ScalableOffset = -SVEStackSize;
2150  if (!UseFP && (isFixed || isCSR))
2151  ScalableOffset = SVEStackSize;
2152 
2153  if (UseFP) {
2154  FrameReg = RegInfo->getFrameRegister(MF);
2155  return StackOffset::getFixed(FPOffset) + ScalableOffset;
2156  }
2157 
2158  // Use the base pointer if we have one.
2159  if (RegInfo->hasBasePointer(MF))
2160  FrameReg = RegInfo->getBaseRegister();
2161  else {
2162  assert(!MFI.hasVarSizedObjects() &&
2163  "Can't use SP when we have var sized objects.");
2164  FrameReg = AArch64::SP;
2165  // If we're using the red zone for this function, the SP won't actually
2166  // be adjusted, so the offsets will be negative. They're also all
2167  // within range of the signed 9-bit immediate instructions.
2168  if (canUseRedZone(MF))
2169  Offset -= AFI->getLocalStackSize();
2170  }
2171 
2172  return StackOffset::getFixed(Offset) + ScalableOffset;
2173 }
2174 
2175 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
2176  // Do not set a kill flag on values that are also marked as live-in. This
2177  // happens with the @llvm-returnaddress intrinsic and with arguments passed in
2178  // callee saved registers.
2179  // Omitting the kill flags is conservatively correct even if the live-in
2180  // is not used after all.
2181  bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
2182  return getKillRegState(!IsLiveIn);
2183 }
2184 
2186  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2188  return Subtarget.isTargetMachO() &&
2189  !(Subtarget.getTargetLowering()->supportSwiftError() &&
2190  Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
2192 }
2193 
2194 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
2195  bool NeedsWinCFI, bool IsFirst) {
2196  // If we are generating register pairs for a Windows function that requires
2197  // EH support, then pair consecutive registers only. There are no unwind
2198  // opcodes for saves/restores of non-consectuve register pairs.
2199  // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x,
2200  // save_lrpair.
2201  // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
2202 
2203  if (Reg2 == AArch64::FP)
2204  return true;
2205  if (!NeedsWinCFI)
2206  return false;
2207  if (Reg2 == Reg1 + 1)
2208  return false;
2209  // If pairing a GPR with LR, the pair can be described by the save_lrpair
2210  // opcode. If this is the first register pair, it would end up with a
2211  // predecrement, but there's no save_lrpair_x opcode, so we can only do this
2212  // if LR is paired with something else than the first register.
2213  // The save_lrpair opcode requires the first register to be an odd one.
2214  if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
2215  (Reg1 - AArch64::X19) % 2 == 0 && Reg2 == AArch64::LR && !IsFirst)
2216  return false;
2217  return true;
2218 }
2219 
2220 /// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
2221 /// WindowsCFI requires that only consecutive registers can be paired.
2222 /// LR and FP need to be allocated together when the frame needs to save
2223 /// the frame-record. This means any other register pairing with LR is invalid.
2224 static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2,
2225  bool UsesWinAAPCS, bool NeedsWinCFI,
2226  bool NeedsFrameRecord, bool IsFirst) {
2227  if (UsesWinAAPCS)
2228  return invalidateWindowsRegisterPairing(Reg1, Reg2, NeedsWinCFI, IsFirst);
2229 
2230  // If we need to store the frame record, don't pair any register
2231  // with LR other than FP.
2232  if (NeedsFrameRecord)
2233  return Reg2 == AArch64::LR;
2234 
2235  return false;
2236 }
2237 
2238 namespace {
2239 
2240 struct RegPairInfo {
2241  unsigned Reg1 = AArch64::NoRegister;
2242  unsigned Reg2 = AArch64::NoRegister;
2243  int FrameIdx;
2244  int Offset;
2245  enum RegType { GPR, FPR64, FPR128, PPR, ZPR } Type;
2246 
2247  RegPairInfo() = default;
2248 
2249  bool isPaired() const { return Reg2 != AArch64::NoRegister; }
2250 
2251  unsigned getScale() const {
2252  switch (Type) {
2253  case PPR:
2254  return 2;
2255  case GPR:
2256  case FPR64:
2257  return 8;
2258  case ZPR:
2259  case FPR128:
2260  return 16;
2261  }
2262  llvm_unreachable("Unsupported type");
2263  }
2264 
2265  bool isScalable() const { return Type == PPR || Type == ZPR; }
2266 };
2267 
2268 } // end anonymous namespace
2269 
2273  bool &NeedShadowCallStackProlog, bool NeedsFrameRecord) {
2274 
2275  if (CSI.empty())
2276  return;
2277 
2278  bool IsWindows = isTargetWindows(MF);
2279  bool NeedsWinCFI = needsWinCFI(MF);
2281  MachineFrameInfo &MFI = MF.getFrameInfo();
2283  unsigned Count = CSI.size();
2284  (void)CC;
2285  // MachO's compact unwind format relies on all registers being stored in
2286  // pairs.
2288  CC == CallingConv::PreserveMost ||
2289  (Count & 1) == 0) &&
2290  "Odd number of callee-saved regs to spill!");
2291  int ByteOffset = AFI->getCalleeSavedStackSize();
2292  int StackFillDir = -1;
2293  int RegInc = 1;
2294  unsigned FirstReg = 0;
2295  if (NeedsWinCFI) {
2296  // For WinCFI, fill the stack from the bottom up.
2297  ByteOffset = 0;
2298  StackFillDir = 1;
2299  // As the CSI array is reversed to match PrologEpilogInserter, iterate
2300  // backwards, to pair up registers starting from lower numbered registers.
2301  RegInc = -1;
2302  FirstReg = Count - 1;
2303  }
2304  int ScalableByteOffset = AFI->getSVECalleeSavedStackSize();
2305  bool NeedGapToAlignStack = AFI->hasCalleeSaveStackFreeSpace();
2306 
2307  // When iterating backwards, the loop condition relies on unsigned wraparound.
2308  for (unsigned i = FirstReg; i < Count; i += RegInc) {
2309  RegPairInfo RPI;
2310  RPI.Reg1 = CSI[i].getReg();
2311 
2312  if (AArch64::GPR64RegClass.contains(RPI.Reg1))
2313  RPI.Type = RegPairInfo::GPR;
2314  else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
2315  RPI.Type = RegPairInfo::FPR64;
2316  else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
2317  RPI.Type = RegPairInfo::FPR128;
2318  else if (AArch64::ZPRRegClass.contains(RPI.Reg1))
2319  RPI.Type = RegPairInfo::ZPR;
2320  else if (AArch64::PPRRegClass.contains(RPI.Reg1))
2321  RPI.Type = RegPairInfo::PPR;
2322  else
2323  llvm_unreachable("Unsupported register class.");
2324 
2325  // Add the next reg to the pair if it is in the same register class.
2326  if (unsigned(i + RegInc) < Count) {
2327  unsigned NextReg = CSI[i + RegInc].getReg();
2328  bool IsFirst = i == FirstReg;
2329  switch (RPI.Type) {
2330  case RegPairInfo::GPR:
2331  if (AArch64::GPR64RegClass.contains(NextReg) &&
2332  !invalidateRegisterPairing(RPI.Reg1, NextReg, IsWindows,
2333  NeedsWinCFI, NeedsFrameRecord, IsFirst))
2334  RPI.Reg2 = NextReg;
2335  break;
2336  case RegPairInfo::FPR64:
2337  if (AArch64::FPR64RegClass.contains(NextReg) &&
2338  !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI,
2339  IsFirst))
2340  RPI.Reg2 = NextReg;
2341  break;
2342  case RegPairInfo::FPR128:
2343  if (AArch64::FPR128RegClass.contains(NextReg))
2344  RPI.Reg2 = NextReg;
2345  break;
2346  case RegPairInfo::PPR:
2347  case RegPairInfo::ZPR:
2348  break;
2349  }
2350  }
2351 
2352  // If either of the registers to be saved is the lr register, it means that
2353  // we also need to save lr in the shadow call stack.
2354  if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
2355  MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
2357  report_fatal_error("Must reserve x18 to use shadow call stack");
2358  NeedShadowCallStackProlog = true;
2359  }
2360 
2361  // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
2362  // list to come in sorted by frame index so that we can issue the store
2363  // pair instructions directly. Assert if we see anything otherwise.
2364  //
2365  // The order of the registers in the list is controlled by
2366  // getCalleeSavedRegs(), so they will always be in-order, as well.
2367  assert((!RPI.isPaired() ||
2368  (CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
2369  "Out of order callee saved regs!");
2370 
2371  assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
2372  RPI.Reg1 == AArch64::LR) &&
2373  "FrameRecord must be allocated together with LR");
2374 
2375  // Windows AAPCS has FP and LR reversed.
2376  assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
2377  RPI.Reg2 == AArch64::LR) &&
2378  "FrameRecord must be allocated together with LR");
2379 
2380  // MachO's compact unwind format relies on all registers being stored in
2381  // adjacent register pairs.
2383  CC == CallingConv::PreserveMost ||
2384  (RPI.isPaired() &&
2385  ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
2386  RPI.Reg1 + 1 == RPI.Reg2))) &&
2387  "Callee-save registers not saved as adjacent register pair!");
2388 
2389  RPI.FrameIdx = CSI[i].getFrameIdx();
2390  if (NeedsWinCFI &&
2391  RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
2392  RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
2393 
2394  int Scale = RPI.getScale();
2395 
2396  int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2397  assert(OffsetPre % Scale == 0);
2398 
2399  if (RPI.isScalable())
2400  ScalableByteOffset += StackFillDir * Scale;
2401  else
2402  ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
2403 
2404  // Swift's async context is directly before FP, so allocate an extra
2405  // 8 bytes for it.
2406  if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
2407  RPI.Reg2 == AArch64::FP)
2408  ByteOffset += StackFillDir * 8;
2409 
2410  assert(!(RPI.isScalable() && RPI.isPaired()) &&
2411  "Paired spill/fill instructions don't exist for SVE vectors");
2412 
2413  // Round up size of non-pair to pair size if we need to pad the
2414  // callee-save area to ensure 16-byte alignment.
2415  if (NeedGapToAlignStack && !NeedsWinCFI &&
2416  !RPI.isScalable() && RPI.Type != RegPairInfo::FPR128 &&
2417  !RPI.isPaired() && ByteOffset % 16 != 0) {
2418  ByteOffset += 8 * StackFillDir;
2419  assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
2420  // A stack frame with a gap looks like this, bottom up:
2421  // d9, d8. x21, gap, x20, x19.
2422  // Set extra alignment on the x21 object to create the gap above it.
2423  MFI.setObjectAlignment(RPI.FrameIdx, Align(16));
2424  NeedGapToAlignStack = false;
2425  }
2426 
2427  int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2428  assert(OffsetPost % Scale == 0);
2429  // If filling top down (default), we want the offset after incrementing it.
2430  // If fillibg bootom up (WinCFI) we need the original offset.
2431  int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
2432 
2433  // The FP, LR pair goes 8 bytes into our expanded 24-byte slot so that the
2434  // Swift context can directly precede FP.
2435  if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
2436  RPI.Reg2 == AArch64::FP)
2437  Offset += 8;
2438  RPI.Offset = Offset / Scale;
2439 
2440  assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
2441  (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
2442  "Offset out of bounds for LDP/STP immediate");
2443 
2444  // Save the offset to frame record so that the FP register can point to the
2445  // innermost frame record (spilled FP and LR registers).
2446  if (NeedsFrameRecord && ((!IsWindows && RPI.Reg1 == AArch64::LR &&
2447  RPI.Reg2 == AArch64::FP) ||
2448  (IsWindows && RPI.Reg1 == AArch64::FP &&
2449  RPI.Reg2 == AArch64::LR)))
2451 
2452  RegPairs.push_back(RPI);
2453  if (RPI.isPaired())
2454  i += RegInc;
2455  }
2456  if (NeedsWinCFI) {
2457  // If we need an alignment gap in the stack, align the topmost stack
2458  // object. A stack frame with a gap looks like this, bottom up:
2459  // x19, d8. d9, gap.
2460  // Set extra alignment on the topmost stack object (the first element in
2461  // CSI, which goes top down), to create the gap above it.
2462  if (AFI->hasCalleeSaveStackFreeSpace())
2463  MFI.setObjectAlignment(CSI[0].getFrameIdx(), Align(16));
2464  // We iterated bottom up over the registers; flip RegPairs back to top
2465  // down order.
2466  std::reverse(RegPairs.begin(), RegPairs.end());
2467  }
2468 }
2469 
2472  ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2473  MachineFunction &MF = *MBB.getParent();
2474  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2475  bool NeedsWinCFI = needsWinCFI(MF);
2476  DebugLoc DL;
2477  SmallVector<RegPairInfo, 8> RegPairs;
2478 
2479  bool NeedShadowCallStackProlog = false;
2480  computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
2481  NeedShadowCallStackProlog, hasFP(MF));
2482  const MachineRegisterInfo &MRI = MF.getRegInfo();
2483 
2484  if (NeedShadowCallStackProlog) {
2485  // Shadow call stack prolog: str x30, [x18], #8
2486  BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
2487  .addReg(AArch64::X18, RegState::Define)
2488  .addReg(AArch64::LR)
2489  .addReg(AArch64::X18)
2490  .addImm(8)
2492 
2493  if (NeedsWinCFI)
2494  BuildMI(MBB, MI, DL, TII.get(AArch64::SEH_Nop))
2496 
2497  if (!MF.getFunction().hasFnAttribute(Attribute::NoUnwind)) {
2498  // Emit a CFI instruction that causes 8 to be subtracted from the value of
2499  // x18 when unwinding past this frame.
2500  static const char CFIInst[] = {
2501  dwarf::DW_CFA_val_expression,
2502  18, // register
2503  2, // length
2504  static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
2505  static_cast<char>(-8) & 0x7f, // addend (sleb128)
2506  };
2507  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
2508  nullptr, StringRef(CFIInst, sizeof(CFIInst))));
2509  BuildMI(MBB, MI, DL, TII.get(AArch64::CFI_INSTRUCTION))
2510  .addCFIIndex(CFIIndex)
2512  }
2513 
2514  // This instruction also makes x18 live-in to the entry block.
2515  MBB.addLiveIn(AArch64::X18);
2516  }
2517 
2518  if (homogeneousPrologEpilog(MF)) {
2519  auto MIB = BuildMI(MBB, MI, DL, TII.get(AArch64::HOM_Prolog))
2521 
2522  for (auto &RPI : RegPairs) {
2523  MIB.addReg(RPI.Reg1);
2524  MIB.addReg(RPI.Reg2);
2525 
2526  // Update register live in.
2527  if (!MRI.isReserved(RPI.Reg1))
2528  MBB.addLiveIn(RPI.Reg1);
2529  if (!MRI.isReserved(RPI.Reg2))
2530  MBB.addLiveIn(RPI.Reg2);
2531  }
2532  return true;
2533  }
2534  for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
2535  ++RPII) {
2536  RegPairInfo RPI = *RPII;
2537  unsigned Reg1 = RPI.Reg1;
2538  unsigned Reg2 = RPI.Reg2;
2539  unsigned StrOpc;
2540 
2541  // Issue sequence of spills for cs regs. The first spill may be converted
2542  // to a pre-decrement store later by emitPrologue if the callee-save stack
2543  // area allocation can't be combined with the local stack area allocation.
2544  // For example:
2545  // stp x22, x21, [sp, #0] // addImm(+0)
2546  // stp x20, x19, [sp, #16] // addImm(+2)
2547  // stp fp, lr, [sp, #32] // addImm(+4)
2548  // Rationale: This sequence saves uop updates compared to a sequence of
2549  // pre-increment spills like stp xi,xj,[sp,#-16]!
2550  // Note: Similar rationale and sequence for restores in epilog.
2551  unsigned Size;
2552  Align Alignment;
2553  switch (RPI.Type) {
2554  case RegPairInfo::GPR:
2555  StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
2556  Size = 8;
2557  Alignment = Align(8);
2558  break;
2559  case RegPairInfo::FPR64:
2560  StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
2561  Size = 8;
2562  Alignment = Align(8);
2563  break;
2564  case RegPairInfo::FPR128:
2565  StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
2566  Size = 16;
2567  Alignment = Align(16);
2568  break;
2569  case RegPairInfo::ZPR:
2570  StrOpc = AArch64::STR_ZXI;
2571  Size = 16;
2572  Alignment = Align(16);
2573  break;
2574  case RegPairInfo::PPR:
2575  StrOpc = AArch64::STR_PXI;
2576  Size = 2;
2577  Alignment = Align(2);
2578  break;
2579  }
2580  LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
2581  if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2582  dbgs() << ") -> fi#(" << RPI.FrameIdx;
2583  if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2584  dbgs() << ")\n");
2585 
2586  assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
2587  "Windows unwdinding requires a consecutive (FP,LR) pair");
2588  // Windows unwind codes require consecutive registers if registers are
2589  // paired. Make the switch here, so that the code below will save (x,x+1)
2590  // and not (x+1,x).
2591  unsigned FrameIdxReg1 = RPI.FrameIdx;
2592  unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2593  if (NeedsWinCFI && RPI.isPaired()) {
2594  std::swap(Reg1, Reg2);
2595  std::swap(FrameIdxReg1, FrameIdxReg2);
2596  }
2597  MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
2598  if (!MRI.isReserved(Reg1))
2599  MBB.addLiveIn(Reg1);
2600  if (RPI.isPaired()) {
2601  if (!MRI.isReserved(Reg2))
2602  MBB.addLiveIn(Reg2);
2603  MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
2605  MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2606  MachineMemOperand::MOStore, Size, Alignment));
2607  }
2608  MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
2609  .addReg(AArch64::SP)
2610  .addImm(RPI.Offset) // [sp, #offset*scale],
2611  // where factor*scale is implicit
2614  MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2615  MachineMemOperand::MOStore, Size, Alignment));
2616  if (NeedsWinCFI)
2618 
2619  // Update the StackIDs of the SVE stack slots.
2620  MachineFrameInfo &MFI = MF.getFrameInfo();
2621  if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR)
2623 
2624  }
2625  return true;
2626 }
2627 
2631  MachineFunction &MF = *MBB.getParent();
2632  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2633  DebugLoc DL;
2634  SmallVector<RegPairInfo, 8> RegPairs;
2635  bool NeedsWinCFI = needsWinCFI(MF);
2636 
2637  if (MI != MBB.end())
2638  DL = MI->getDebugLoc();
2639 
2640  bool NeedShadowCallStackProlog = false;
2641  computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
2642  NeedShadowCallStackProlog, hasFP(MF));
2643 
2644  auto EmitMI = [&](const RegPairInfo &RPI) {
2645  unsigned Reg1 = RPI.Reg1;
2646  unsigned Reg2 = RPI.Reg2;
2647 
2648  // Issue sequence of restores for cs regs. The last restore may be converted
2649  // to a post-increment load later by emitEpilogue if the callee-save stack
2650  // area allocation can't be combined with the local stack area allocation.
2651  // For example:
2652  // ldp fp, lr, [sp, #32] // addImm(+4)
2653  // ldp x20, x19, [sp, #16] // addImm(+2)
2654  // ldp x22, x21, [sp, #0] // addImm(+0)
2655  // Note: see comment in spillCalleeSavedRegisters()
2656  unsigned LdrOpc;
2657  unsigned Size;
2658  Align Alignment;
2659  switch (RPI.Type) {
2660  case RegPairInfo::GPR:
2661  LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
2662  Size = 8;
2663  Alignment = Align(8);
2664  break;
2665  case RegPairInfo::FPR64:
2666  LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
2667  Size = 8;
2668  Alignment = Align(8);
2669  break;
2670  case RegPairInfo::FPR128:
2671  LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
2672  Size = 16;
2673  Alignment = Align(16);
2674  break;
2675  case RegPairInfo::ZPR:
2676  LdrOpc = AArch64::LDR_ZXI;
2677  Size = 16;
2678  Alignment = Align(16);
2679  break;
2680  case RegPairInfo::PPR:
2681  LdrOpc = AArch64::LDR_PXI;
2682  Size = 2;
2683  Alignment = Align(2);
2684  break;
2685  }
2686  LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
2687  if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2688  dbgs() << ") -> fi#(" << RPI.FrameIdx;
2689  if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2690  dbgs() << ")\n");
2691 
2692  // Windows unwind codes require consecutive registers if registers are
2693  // paired. Make the switch here, so that the code below will save (x,x+1)
2694  // and not (x+1,x).
2695  unsigned FrameIdxReg1 = RPI.FrameIdx;
2696  unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2697  if (NeedsWinCFI && RPI.isPaired()) {
2698  std::swap(Reg1, Reg2);
2699  std::swap(FrameIdxReg1, FrameIdxReg2);
2700  }
2701  MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
2702  if (RPI.isPaired()) {
2703  MIB.addReg(Reg2, getDefRegState(true));
2705  MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2706  MachineMemOperand::MOLoad, Size, Alignment));
2707  }
2708  MIB.addReg(Reg1, getDefRegState(true))
2709  .addReg(AArch64::SP)
2710  .addImm(RPI.Offset) // [sp, #offset*scale]
2711  // where factor*scale is implicit
2714  MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2715  MachineMemOperand::MOLoad, Size, Alignment));
2716  if (NeedsWinCFI)
2718  };
2719 
2720  // SVE objects are always restored in reverse order.
2721  for (const RegPairInfo &RPI : reverse(RegPairs))
2722  if (RPI.isScalable())
2723  EmitMI(RPI);
2724 
2725  if (ReverseCSRRestoreSeq) {
2726  for (const RegPairInfo &RPI : reverse(RegPairs))
2727  if (!RPI.isScalable())
2728  EmitMI(RPI);
2729  } else if (homogeneousPrologEpilog(MF, &MBB)) {
2730  auto MIB = BuildMI(MBB, MI, DL, TII.get(AArch64::HOM_Epilog))
2732  for (auto &RPI : RegPairs) {
2733  MIB.addReg(RPI.Reg1, RegState::Define);
2734  MIB.addReg(RPI.Reg2, RegState::Define);
2735  }
2736  return true;
2737  } else
2738  for (const RegPairInfo &RPI : RegPairs)
2739  if (!RPI.isScalable())
2740  EmitMI(RPI);
2741 
2742  if (NeedShadowCallStackProlog) {
2743  // Shadow call stack epilog: ldr x30, [x18, #-8]!
2744  BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
2745  .addReg(AArch64::X18, RegState::Define)
2746  .addReg(AArch64::LR, RegState::Define)
2747  .addReg(AArch64::X18)
2748  .addImm(-8)
2750  }
2751 
2752  return true;
2753 }
2754 
2756  BitVector &SavedRegs,
2757  RegScavenger *RS) const {
2758  // All calls are tail calls in GHC calling conv, and functions have no
2759  // prologue/epilogue.
2761  return;
2762 
2763  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2764  const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
2765  MF.getSubtarget().getRegisterInfo());
2766  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2768  unsigned UnspilledCSGPR = AArch64::NoRegister;
2769  unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
2770 
2771  MachineFrameInfo &MFI = MF.getFrameInfo();
2772  const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
2773 
2774  unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
2775  ? RegInfo->getBaseRegister()
2776  : (unsigned)AArch64::NoRegister;
2777 
2778  unsigned ExtraCSSpill = 0;
2779  // Figure out which callee-saved registers to save/restore.
2780  for (unsigned i = 0; CSRegs[i]; ++i) {
2781  const unsigned Reg = CSRegs[i];
2782 
2783  // Add the base pointer register to SavedRegs if it is callee-save.
2784  if (Reg == BasePointerReg)
2785  SavedRegs.set(Reg);
2786 
2787  bool RegUsed = SavedRegs.test(Reg);
2788  unsigned PairedReg = AArch64::NoRegister;
2789  if (AArch64::GPR64RegClass.contains(Reg) ||
2790  AArch64::FPR64RegClass.contains(Reg) ||
2791  AArch64::FPR128RegClass.contains(Reg))
2792  PairedReg = CSRegs[i ^ 1];
2793 
2794  if (!RegUsed) {
2795  if (AArch64::GPR64RegClass.contains(Reg) &&
2796  !RegInfo->isReservedReg(MF, Reg)) {
2797  UnspilledCSGPR = Reg;
2798  UnspilledCSGPRPaired = PairedReg;
2799  }
2800  continue;
2801  }
2802 
2803  // MachO's compact unwind format relies on all registers being stored in
2804  // pairs.
2805  // FIXME: the usual format is actually better if unwinding isn't needed.
2806  if (producePairRegisters(MF) && PairedReg != AArch64::NoRegister &&
2807  !SavedRegs.test(PairedReg)) {
2808  SavedRegs.set(PairedReg);
2809  if (AArch64::GPR64RegClass.contains(PairedReg) &&
2810  !RegInfo->isReservedReg(MF, PairedReg))
2811  ExtraCSSpill = PairedReg;
2812  }
2813  }
2814 
2816  !Subtarget.isTargetWindows()) {
2817  // For Windows calling convention on a non-windows OS, where X18 is treated
2818  // as reserved, back up X18 when entering non-windows code (marked with the
2819  // Windows calling convention) and restore when returning regardless of
2820  // whether the individual function uses it - it might call other functions
2821  // that clobber it.
2822  SavedRegs.set(AArch64::X18);
2823  }
2824 
2825  // Calculates the callee saved stack size.
2826  unsigned CSStackSize = 0;
2827  unsigned SVECSStackSize = 0;
2829  const MachineRegisterInfo &MRI = MF.getRegInfo();
2830  for (unsigned Reg : SavedRegs.set_bits()) {
2831  auto RegSize = TRI->getRegSizeInBits(Reg, MRI) / 8;
2832  if (AArch64::PPRRegClass.contains(Reg) ||
2833  AArch64::ZPRRegClass.contains(Reg))
2834  SVECSStackSize += RegSize;
2835  else
2836  CSStackSize += RegSize;
2837  }
2838 
2839  // Save number of saved regs, so we can easily update CSStackSize later.
2840  unsigned NumSavedRegs = SavedRegs.count();
2841 
2842  // The frame record needs to be created by saving the appropriate registers
2843  uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
2844  if (hasFP(MF) ||
2845  windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
2846  SavedRegs.set(AArch64::FP);
2847  SavedRegs.set(AArch64::LR);
2848  }
2849 
2850  LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
2851  for (unsigned Reg
2852  : SavedRegs.set_bits()) dbgs()
2853  << ' ' << printReg(Reg, RegInfo);
2854  dbgs() << "\n";);
2855 
2856  // If any callee-saved registers are used, the frame cannot be eliminated.
2857  int64_t SVEStackSize =
2858  alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
2859  bool CanEliminateFrame = (SavedRegs.count() == 0) && !SVEStackSize;
2860 
2861  // The CSR spill slots have not been allocated yet, so estimateStackSize
2862  // won't include them.
2863  unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
2864 
2865  // Conservatively always assume BigStack when there are SVE spills.
2866  bool BigStack = SVEStackSize ||
2867  (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
2868  if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
2869  AFI->setHasStackFrame(true);
2870 
2871  // Estimate if we might need to scavenge a register at some point in order
2872  // to materialize a stack offset. If so, either spill one additional
2873  // callee-saved register or reserve a special spill slot to facilitate
2874  // register scavenging. If we already spilled an extra callee-saved register
2875  // above to keep the number of spills even, we don't need to do anything else
2876  // here.
2877  if (BigStack) {
2878  if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
2879  LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
2880  << " to get a scratch register.\n");
2881  SavedRegs.set(UnspilledCSGPR);
2882  // MachO's compact unwind format relies on all registers being stored in
2883  // pairs, so if we need to spill one extra for BigStack, then we need to
2884  // store the pair.
2885  if (producePairRegisters(MF))
2886  SavedRegs.set(UnspilledCSGPRPaired);
2887  ExtraCSSpill = UnspilledCSGPR;
2888  }
2889 
2890  // If we didn't find an extra callee-saved register to spill, create
2891  // an emergency spill slot.
2892  if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
2894  const TargetRegisterClass &RC = AArch64::GPR64RegClass;
2895  unsigned Size = TRI->getSpillSize(RC);
2896  Align Alignment = TRI->getSpillAlign(RC);
2897  int FI = MFI.CreateStackObject(Size, Alignment, false);
2898  RS->addScavengingFrameIndex(FI);
2899  LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
2900  << " as the emergency spill slot.\n");
2901  }
2902  }
2903 
2904  // Adding the size of additional 64bit GPR saves.
2905  CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
2906 
2907  // A Swift asynchronous context extends the frame record with a pointer
2908  // directly before FP.
2909  if (hasFP(MF) && AFI->hasSwiftAsyncContext())
2910  CSStackSize += 8;
2911 
2912  uint64_t AlignedCSStackSize = alignTo(CSStackSize, 16);
2913  LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
2914  << EstimatedStackSize + AlignedCSStackSize
2915  << " bytes.\n");
2916 
2917  assert((!MFI.isCalleeSavedInfoValid() ||
2918  AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
2919  "Should not invalidate callee saved info");
2920 
2921  // Round up to register pair alignment to avoid additional SP adjustment
2922  // instructions.
2923  AFI->setCalleeSavedStackSize(AlignedCSStackSize);
2924  AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
2925  AFI->setSVECalleeSavedStackSize(alignTo(SVECSStackSize, 16));
2926 }
2927 
2929  MachineFunction &MF, const TargetRegisterInfo *RegInfo,
2930  std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
2931  unsigned &MaxCSFrameIndex) const {
2932  bool NeedsWinCFI = needsWinCFI(MF);
2933  // To match the canonical windows frame layout, reverse the list of
2934  // callee saved registers to get them laid out by PrologEpilogInserter
2935  // in the right order. (PrologEpilogInserter allocates stack objects top
2936  // down. Windows canonical prologs store higher numbered registers at
2937  // the top, thus have the CSI array start from the highest registers.)
2938  if (NeedsWinCFI)
2939  std::reverse(CSI.begin(), CSI.end());
2940 
2941  if (CSI.empty())
2942  return true; // Early exit if no callee saved registers are modified!
2943 
2944  // Now that we know which registers need to be saved and restored, allocate
2945  // stack slots for them.
2946  MachineFrameInfo &MFI = MF.getFrameInfo();
2947  auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2948  for (auto &CS : CSI) {
2949  Register Reg = CS.getReg();
2950  const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
2951 
2952  unsigned Size = RegInfo->getSpillSize(*RC);
2953  Align Alignment(RegInfo->getSpillAlign(*RC));
2954  int FrameIdx = MFI.CreateStackObject(Size, Alignment, true);
2955  CS.setFrameIdx(FrameIdx);
2956 
2957  if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
2958  if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
2959 
2960  // Grab 8 bytes below FP for the extended asynchronous frame info.
2961  if (hasFP(MF) && AFI->hasSwiftAsyncContext() && Reg == AArch64::FP) {
2962  FrameIdx = MFI.CreateStackObject(8, Alignment, true);
2963  AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
2964  if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
2965  if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
2966  }
2967  }
2968  return true;
2969 }
2970 
2972  const MachineFunction &MF) const {
2973  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2974  return AFI->hasCalleeSaveStackFreeSpace();
2975 }
2976 
2977 /// returns true if there are any SVE callee saves.
2979  int &Min, int &Max) {
2982 
2983  if (!MFI.isCalleeSavedInfoValid())
2984  return false;
2985 
2986  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
2987  for (auto &CS : CSI) {
2988  if (AArch64::ZPRRegClass.contains(CS.getReg()) ||
2989  AArch64::PPRRegClass.contains(CS.getReg())) {
2991  Max + 1 == CS.getFrameIdx()) &&
2992  "SVE CalleeSaves are not consecutive");
2993 
2994  Min = std::min(Min, CS.getFrameIdx());
2995  Max = std::max(Max, CS.getFrameIdx());
2996  }
2997  }
2998  return Min != std::numeric_limits<int>::max();
2999 }
3000 
3001 // Process all the SVE stack objects and determine offsets for each
3002 // object. If AssignOffsets is true, the offsets get assigned.
3003 // Fills in the first and last callee-saved frame indices into
3004 // Min/MaxCSFrameIndex, respectively.
3005 // Returns the size of the stack.
3007  int &MinCSFrameIndex,
3008  int &MaxCSFrameIndex,
3009  bool AssignOffsets) {
3010 #ifndef NDEBUG
3011  // First process all fixed stack objects.
3012  for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
3014  "SVE vectors should never be passed on the stack by value, only by "
3015  "reference.");
3016 #endif
3017 
3018  auto Assign = [&MFI](int FI, int64_t Offset) {
3019  LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
3020  MFI.setObjectOffset(FI, Offset);
3021  };
3022 
3023  int64_t Offset = 0;
3024 
3025  // Then process all callee saved slots.
3026  if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
3027  // Assign offsets to the callee save slots.
3028  for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
3029  Offset += MFI.getObjectSize(I);
3031  if (AssignOffsets)
3032  Assign(I, -Offset);
3033  }
3034  }
3035 
3036  // Ensure that the Callee-save area is aligned to 16bytes.
3037  Offset = alignTo(Offset, Align(16U));
3038 
3039  // Create a buffer of SVE objects to allocate and sort it.
3040  SmallVector<int, 8> ObjectsToAllocate;
3041  for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
3042  unsigned StackID = MFI.getStackID(I);
3043  if (StackID != TargetStackID::ScalableVector)
3044  continue;
3045  if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex)
3046  continue;
3047  if (MFI.isDeadObjectIndex(I))
3048  continue;
3049 
3050  ObjectsToAllocate.push_back(I);
3051  }
3052 
3053  // Allocate all SVE locals and spills
3054  for (unsigned FI : ObjectsToAllocate) {
3055  Align Alignment = MFI.getObjectAlign(FI);
3056  // FIXME: Given that the length of SVE vectors is not necessarily a power of
3057  // two, we'd need to align every object dynamically at runtime if the
3058  // alignment is larger than 16. This is not yet supported.
3059  if (Alignment > Align(16))
3061  "Alignment of scalable vectors > 16 bytes is not yet supported");
3062 
3063  Offset = alignTo(Offset + MFI.getObjectSize(FI), Alignment);
3064  if (AssignOffsets)
3065  Assign(FI, -Offset);
3066  }
3067 
3068  return Offset;
3069 }
3070 
3071 int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
3072  MachineFrameInfo &MFI) const {
3073  int MinCSFrameIndex, MaxCSFrameIndex;
3074  return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex, false);
3075 }
3076 
3077 int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
3078  MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex) const {
3079  return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex,
3080  true);
3081 }
3082 
3084  MachineFunction &MF, RegScavenger *RS) const {
3085  MachineFrameInfo &MFI = MF.getFrameInfo();
3086 
3088  "Upwards growing stack unsupported");
3089 
3090  int MinCSFrameIndex, MaxCSFrameIndex;
3091  int64_t SVEStackSize =
3092  assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
3093 
3095  AFI->setStackSizeSVE(alignTo(SVEStackSize, 16U));
3096  AFI->setMinMaxSVECSFrameIndex(MinCSFrameIndex, MaxCSFrameIndex);
3097 
3098  // If this function isn't doing Win64-style C++ EH, we don't need to do
3099  // anything.
3100  if (!MF.hasEHFunclets())
3101  return;
3102  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
3103  WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
3104 
3105  MachineBasicBlock &MBB = MF.front();
3106  auto MBBI = MBB.begin();
3107  while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
3108  ++MBBI;
3109 
3110  // Create an UnwindHelp object.
3111  // The UnwindHelp object is allocated at the start of the fixed object area
3112  int64_t FixedObject =
3113  getFixedObjectSize(MF, AFI, /*IsWin64*/ true, /*IsFunclet*/ false);
3114  int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8,
3115  /*SPOffset*/ -FixedObject,
3116  /*IsImmutable=*/false);
3117  EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
3118 
3119  // We need to store -2 into the UnwindHelp object at the start of the
3120  // function.
3121  DebugLoc DL;
3122  RS->enterBasicBlockEnd(MBB);
3123  RS->backward(std::prev(MBBI));
3124  unsigned DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
3125  assert(DstReg && "There must be a free register after frame setup");
3126  BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
3127  BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
3128  .addReg(DstReg, getKillRegState(true))
3129  .addFrameIndex(UnwindHelpFI)
3130  .addImm(0);
3131 }
3132 
3133 namespace {
3134 struct TagStoreInstr {
3135  MachineInstr *MI;
3136  int64_t Offset, Size;
3137  explicit TagStoreInstr(MachineInstr *MI, int64_t Offset, int64_t Size)
3138  : MI(MI), Offset(Offset), Size(Size) {}
3139 };
3140 
3141 class TagStoreEdit {
3142  MachineFunction *MF;
3145  // Tag store instructions that are being replaced.
3147  // Combined memref arguments of the above instructions.
3148  SmallVector<MachineMemOperand *, 8> CombinedMemRefs;
3149 
3150  // Replace allocation tags in [FrameReg + FrameRegOffset, FrameReg +
3151  // FrameRegOffset + Size) with the address tag of SP.
3152  Register FrameReg;
3153  StackOffset FrameRegOffset;
3154  int64_t Size;
3155  // If not None, move FrameReg to (FrameReg + FrameRegUpdate) at the end.
3156  Optional<int64_t> FrameRegUpdate;
3157  // MIFlags for any FrameReg updating instructions.
3158  unsigned FrameRegUpdateFlags;
3159 
3160  // Use zeroing instruction variants.
3161  bool ZeroData;
3162  DebugLoc DL;
3163 
3164  void emitUnrolled(MachineBasicBlock::iterator InsertI);
3165  void emitLoop(MachineBasicBlock::iterator InsertI);
3166 
3167 public:
3168  TagStoreEdit(MachineBasicBlock *MBB, bool ZeroData)
3169  : MBB(MBB), ZeroData(ZeroData) {
3170  MF = MBB->getParent();
3171  MRI = &MF->getRegInfo();
3172  }
3173  // Add an instruction to be replaced. Instructions must be added in the
3174  // ascending order of Offset, and have to be adjacent.
3175  void addInstruction(TagStoreInstr I) {
3176  assert((TagStores.empty() ||
3177  TagStores.back().Offset + TagStores.back().Size == I.Offset) &&
3178  "Non-adjacent tag store instructions.");
3179  TagStores.push_back(I);
3180  }
3181  void clear() { TagStores.clear(); }
3182  // Emit equivalent code at the given location, and erase the current set of
3183  // instructions. May skip if the replacement is not profitable. May invalidate
3184  // the input iterator and replace it with a valid one.
3185  void emitCode(MachineBasicBlock::iterator &InsertI,
3186  const AArch64FrameLowering *TFI, bool IsLast);
3187 };
3188 
3189 void TagStoreEdit::emitUnrolled(MachineBasicBlock::iterator InsertI) {
3190  const AArch64InstrInfo *TII =
3191  MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
3192 
3193  const int64_t kMinOffset = -256 * 16;
3194  const int64_t kMaxOffset = 255 * 16;
3195 
3196  Register BaseReg = FrameReg;
3197  int64_t BaseRegOffsetBytes = FrameRegOffset.getFixed();
3198  if (BaseRegOffsetBytes < kMinOffset ||
3199  BaseRegOffsetBytes + (Size - Size % 32) > kMaxOffset) {
3200  Register ScratchReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3201  emitFrameOffset(*MBB, InsertI, DL, ScratchReg, BaseReg,
3202  StackOffset::getFixed(BaseRegOffsetBytes), TII);
3203  BaseReg = ScratchReg;
3204  BaseRegOffsetBytes = 0;
3205  }
3206 
3207  MachineInstr *LastI = nullptr;
3208  while (Size) {
3209  int64_t InstrSize = (Size > 16) ? 32 : 16;
3210  unsigned Opcode =
3211  InstrSize == 16
3212  ? (ZeroData ? AArch64::STZGOffset : AArch64::STGOffset)
3213  : (ZeroData ? AArch64::STZ2GOffset : AArch64::ST2GOffset);
3214  MachineInstr *I = BuildMI(*MBB, InsertI, DL, TII->get(Opcode))
3215  .addReg(AArch64::SP)
3216  .addReg(BaseReg)
3217  .addImm(BaseRegOffsetBytes / 16)
3218  .setMemRefs(CombinedMemRefs);
3219  // A store to [BaseReg, #0] should go last for an opportunity to fold the
3220  // final SP adjustment in the epilogue.
3221  if (BaseRegOffsetBytes == 0)
3222  LastI = I;
3223  BaseRegOffsetBytes += InstrSize;
3224  Size -= InstrSize;
3225  }
3226 
3227  if (LastI)
3228  MBB->splice(InsertI, MBB, LastI);
3229 }
3230 
3231 void TagStoreEdit::emitLoop(MachineBasicBlock::iterator InsertI) {
3232  const AArch64InstrInfo *TII =
3233  MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
3234 
3235  Register BaseReg = FrameRegUpdate
3236  ? FrameReg
3237  : MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3238  Register SizeReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
3239 
3240  emitFrameOffset(*MBB, InsertI, DL, BaseReg, FrameReg, FrameRegOffset, TII);
3241 
3242  int64_t LoopSize = Size;
3243  // If the loop size is not a multiple of 32, split off one 16-byte store at
3244  // the end to fold BaseReg update into.
3245  if (FrameRegUpdate && *FrameRegUpdate)
3246  LoopSize -= LoopSize % 32;
3247  MachineInstr *LoopI = BuildMI(*MBB, InsertI, DL,
3248  TII->get(ZeroData ? AArch64::STZGloop_wback
3249  : AArch64::STGloop_wback))
3250  .addDef(SizeReg)
3251  .addDef(BaseReg)
3252  .addImm(LoopSize)
3253  .addReg(BaseReg)
3254  .setMemRefs(CombinedMemRefs);
3255  if (FrameRegUpdate)
3256  LoopI->setFlags(FrameRegUpdateFlags);
3257 
3258  int64_t ExtraBaseRegUpdate =
3259  FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.getFixed() - Size) : 0;
3260  if (LoopSize < Size) {
3261  assert(FrameRegUpdate);
3262  assert(Size - LoopSize == 16);
3263  // Tag 16 more bytes at BaseReg and update BaseReg.
3264  BuildMI(*MBB, InsertI, DL,
3265  TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
3266  .addDef(BaseReg)
3267  .addReg(BaseReg)
3268  .addReg(BaseReg)
3269  .addImm(1 + ExtraBaseRegUpdate / 16)
3270  .setMemRefs(CombinedMemRefs)
3271  .setMIFlags(FrameRegUpdateFlags);
3272  } else if (ExtraBaseRegUpdate) {
3273  // Update BaseReg.
3274  BuildMI(
3275  *MBB, InsertI, DL,
3276  TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
3277  .addDef(BaseReg)
3278  .addReg(BaseReg)
3279  .addImm(std::abs(ExtraBaseRegUpdate))
3280  .addImm(0)
3281  .setMIFlags(FrameRegUpdateFlags);
3282  }
3283 }
3284 
3285 // Check if *II is a register update that can be merged into STGloop that ends
3286 // at (Reg + Size). RemainingOffset is the required adjustment to Reg after the
3287 // end of the loop.
3288 bool canMergeRegUpdate(MachineBasicBlock::iterator II, unsigned Reg,
3289  int64_t Size, int64_t *TotalOffset) {
3290  MachineInstr &MI = *II;
3291  if ((MI.getOpcode() == AArch64::ADDXri ||
3292  MI.getOpcode() == AArch64::SUBXri) &&
3293  MI.getOperand(0).getReg() == Reg && MI.getOperand(1).getReg() == Reg) {
3294  unsigned Shift = AArch64_AM::getShiftValue(MI.getOperand(3).getImm());
3295  int64_t Offset = MI.getOperand(2).getImm() << Shift;
3296  if (MI.getOpcode() == AArch64::SUBXri)
3297  Offset = -Offset;
3298  int64_t AbsPostOffset = std::abs(Offset - Size);
3299  const int64_t kMaxOffset =
3300  0xFFF; // Max encoding for unshifted ADDXri / SUBXri
3301  if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
3302  *TotalOffset = Offset;
3303  return true;
3304  }
3305  }
3306  return false;
3307 }
3308 
3309 void mergeMemRefs(const SmallVectorImpl<TagStoreInstr> &TSE,
3311  MemRefs.clear();
3312  for (auto &TS : TSE) {
3313  MachineInstr *MI = TS.MI;
3314  // An instruction without memory operands may access anything. Be
3315  // conservative and return an empty list.
3316  if (MI->memoperands_empty()) {
3317  MemRefs.clear();
3318  return;
3319  }
3320  MemRefs.append(MI->memoperands_begin(), MI->memoperands_end());
3321  }
3322 }
3323 
3324 void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
3325  const AArch64FrameLowering *TFI, bool IsLast) {
3326  if (TagStores.empty())
3327  return;
3328  TagStoreInstr &FirstTagStore = TagStores[0];
3329  TagStoreInstr &LastTagStore = TagStores[TagStores.size() - 1];
3330  Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
3331  DL = TagStores[0].MI->getDebugLoc();
3332 
3333  Register Reg;
3334  FrameRegOffset = TFI->resolveFrameOffsetReference(
3335  *MF, FirstTagStore.Offset, false /*isFixed*/, false /*isSVE*/, Reg,
3336  /*PreferFP=*/false, /*ForSimm=*/true);
3337  FrameReg = Reg;
3338  FrameRegUpdate = None;
3339 
3340  mergeMemRefs(TagStores, CombinedMemRefs);
3341 
3342  LLVM_DEBUG(dbgs() << "Replacing adjacent STG instructions:\n";
3343  for (const auto &Instr
3344  : TagStores) { dbgs() << " " << *Instr.MI; });
3345 
3346  // Size threshold where a loop becomes shorter than a linear sequence of
3347  // tagging instructions.
3348  const int kSetTagLoopThreshold = 176;
3349  if (Size < kSetTagLoopThreshold) {
3350  if (TagStores.size() < 2)
3351  return;
3352  emitUnrolled(InsertI);
3353  } else {
3354  MachineInstr *UpdateInstr = nullptr;
3355  int64_t TotalOffset;
3356  if (IsLast) {
3357  // See if we can merge base register update into the STGloop.
3358  // This is done in AArch64LoadStoreOptimizer for "normal" stores,
3359  // but STGloop is way too unusual for that, and also it only
3360  // realistically happens in function epilogue. Also, STGloop is expanded
3361  // before that pass.
3362  if (InsertI != MBB->end() &&
3363  canMergeRegUpdate(InsertI, FrameReg, FrameRegOffset.getFixed() + Size,
3364  &TotalOffset)) {
3365  UpdateInstr = &*InsertI++;
3366  LLVM_DEBUG(dbgs() << "Folding SP update into loop:\n "
3367  << *UpdateInstr);
3368  }
3369  }
3370 
3371  if (!UpdateInstr && TagStores.size() < 2)
3372  return;
3373 
3374  if (UpdateInstr) {
3375  FrameRegUpdate = TotalOffset;
3376  FrameRegUpdateFlags = UpdateInstr->getFlags();
3377  }
3378  emitLoop(InsertI);
3379  if (UpdateInstr)
3380  UpdateInstr->eraseFromParent();
3381  }
3382 
3383  for (auto &TS : TagStores)
3384  TS.MI->eraseFromParent();
3385 }
3386 
3387 bool isMergeableStackTaggingInstruction(MachineInstr &MI, int64_t &Offset,
3388  int64_t &Size, bool &ZeroData) {
3389  MachineFunction &MF = *MI.getParent()->getParent();
3390  const MachineFrameInfo &MFI = MF.getFrameInfo();
3391 
3392  unsigned Opcode = MI.getOpcode();
3393  ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGOffset ||
3394  Opcode == AArch64::STZ2GOffset);
3395 
3396  if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
3397  if (!MI.getOperand(0).isDead() || !MI.getOperand(1).isDead())
3398  return false;
3399  if (!MI.getOperand(2).isImm() || !MI.getOperand(3).isFI())
3400  return false;
3401  Offset = MFI.getObjectOffset(MI.getOperand(3).getIndex());
3402  Size = MI.getOperand(2).getImm();
3403  return true;
3404  }
3405 
3406  if (Opcode == AArch64::STGOffset || Opcode == AArch64::STZGOffset)
3407  Size = 16;
3408  else if (Opcode == AArch64::ST2GOffset || Opcode == AArch64::STZ2GOffset)
3409  Size = 32;
3410  else
3411  return false;
3412 
3413  if (MI.getOperand(0).getReg() != AArch64::SP || !MI.getOperand(1).isFI())
3414  return false;
3415 
3416  Offset = MFI.getObjectOffset(MI.getOperand(1).getIndex()) +
3417  16 * MI.getOperand(2).getImm();
3418  return true;
3419 }
3420 
3421 // Detect a run of memory tagging instructions for adjacent stack frame slots,
3422 // and replace them with a shorter instruction sequence:
3423 // * replace STG + STG with ST2G
3424 // * replace STGloop + STGloop with STGloop
3425 // This code needs to run when stack slot offsets are already known, but before
3426 // FrameIndex operands in STG instructions are eliminated.
3428  const AArch64FrameLowering *TFI,
3429  RegScavenger *RS) {
3430  bool FirstZeroData;
3431  int64_t Size, Offset;
3432  MachineInstr &MI = *II;
3433  MachineBasicBlock *MBB = MI.getParent();
3434  MachineBasicBlock::iterator NextI = ++II;
3435  if (&MI == &MBB->instr_back())
3436  return II;
3437  if (!isMergeableStackTaggingInstruction(MI, Offset, Size, FirstZeroData))
3438  return II;
3439 
3441  Instrs.emplace_back(&MI, Offset, Size);
3442 
3443  constexpr int kScanLimit = 10;
3444  int Count = 0;
3446  NextI != E && Count < kScanLimit; ++NextI) {
3447  MachineInstr &MI = *NextI;
3448  bool ZeroData;
3449  int64_t Size, Offset;
3450  // Collect instructions that update memory tags with a FrameIndex operand
3451  // and (when applicable) constant size, and whose output registers are dead
3452  // (the latter is almost always the case in practice). Since these
3453  // instructions effectively have no inputs or outputs, we are free to skip
3454  // any non-aliasing instructions in between without tracking used registers.
3455  if (isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData)) {
3456  if (ZeroData != FirstZeroData)
3457  break;
3458  Instrs.emplace_back(&MI, Offset, Size);
3459  continue;
3460  }
3461 
3462  // Only count non-transient, non-tagging instructions toward the scan
3463  // limit.
3464  if (!MI.isTransient())
3465  ++Count;
3466 
3467  // Just in case, stop before the epilogue code starts.
3468  if (MI.getFlag(MachineInstr::FrameSetup) ||
3469  MI.getFlag(MachineInstr::FrameDestroy))
3470  break;
3471 
3472  // Reject anything that may alias the collected instructions.
3473  if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects())
3474  break;
3475  }
3476 
3477  // New code will be inserted after the last tagging instruction we've found.
3478  MachineBasicBlock::iterator InsertI = Instrs.back().MI;
3479  InsertI++;
3480 
3481  llvm::stable_sort(Instrs,
3482  [](const TagStoreInstr &Left, const TagStoreInstr &Right) {
3483  return Left.Offset < Right.Offset;
3484  });
3485 
3486  // Make sure that we don't have any overlapping stores.
3487  int64_t CurOffset = Instrs[0].Offset;
3488  for (auto &Instr : Instrs) {
3489  if (CurOffset > Instr.Offset)
3490  return NextI;
3491  CurOffset = Instr.Offset + Instr.Size;
3492  }
3493 
3494  // Find contiguous runs of tagged memory and emit shorter instruction
3495  // sequencies for them when possible.
3496  TagStoreEdit TSE(MBB, FirstZeroData);
3497  Optional<int64_t> EndOffset;
3498  for (auto &Instr : Instrs) {
3499  if (EndOffset && *EndOffset != Instr.Offset) {
3500  // Found a gap.
3501  TSE.emitCode(InsertI, TFI, /*IsLast = */ false);
3502  TSE.clear();
3503  }
3504 
3505  TSE.addInstruction(Instr);
3506  EndOffset = Instr.Offset + Instr.Size;
3507  }
3508 
3509  TSE.emitCode(InsertI, TFI, /*IsLast = */ true);
3510 
3511  return InsertI;
3512 }
3513 } // namespace
3514 
3516  MachineFunction &MF, RegScavenger *RS = nullptr) const {
3518  for (auto &BB : MF)
3519  for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();)
3520  II = tryMergeAdjacentSTG(II, this, RS);
3521 }
3522 
3523 /// For Win64 AArch64 EH, the offset to the Unwind object is from the SP
3524 /// before the update. This is easily retrieved as it is exactly the offset
3525 /// that is set in processFunctionBeforeFrameFinalized.
3527  const MachineFunction &MF, int FI, Register &FrameReg,
3528  bool IgnoreSPUpdates) const {
3529  const MachineFrameInfo &MFI = MF.getFrameInfo();
3530  if (IgnoreSPUpdates) {
3531  LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
3532  << MFI.getObjectOffset(FI) << "\n");
3533  FrameReg = AArch64::SP;
3534  return StackOffset::getFixed(MFI.getObjectOffset(FI));
3535  }
3536 
3537  return getFrameIndexReference(MF, FI, FrameReg);
3538 }
3539 
3540 /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
3541 /// the parent's frame pointer
3543  const MachineFunction &MF) const {
3544  return 0;
3545 }
3546 
3547 /// Funclets only need to account for space for the callee saved registers,
3548 /// as the locals are accounted for in the parent's stack frame.
3550  const MachineFunction &MF) const {
3551  // This is the size of the pushed CSRs.
3552  unsigned CSSize =
3553  MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
3554  // This is the amount of stack a funclet needs to allocate.
3555  return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
3556  getStackAlign());
3557 }
3558 
3559 namespace {
3560 struct FrameObject {
3561  bool IsValid = false;
3562  // Index of the object in MFI.
3563  int ObjectIndex = 0;
3564  // Group ID this object belongs to.
3565  int GroupIndex = -1;
3566  // This object should be placed first (closest to SP).
3567  bool ObjectFirst = false;
3568  // This object's group (which always contains the object with
3569  // ObjectFirst==true) should be placed first.
3570  bool GroupFirst = false;
3571 };
3572 
3573 class GroupBuilder {
3574  SmallVector<int, 8> CurrentMembers;
3575  int NextGroupIndex = 0;
3576  std::vector<FrameObject> &Objects;
3577 
3578 public:
3579  GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
3580  void AddMember(int Index) { CurrentMembers.push_back(Index); }
3581  void EndCurrentGroup() {
3582  if (CurrentMembers.size() > 1) {
3583  // Create a new group with the current member list. This might remove them
3584  // from their pre-existing groups. That's OK, dealing with overlapping
3585  // groups is too hard and unlikely to make a difference.
3586  LLVM_DEBUG(dbgs() << "group:");
3587  for (int Index : CurrentMembers) {
3588  Objects[Index].GroupIndex = NextGroupIndex;
3589  LLVM_DEBUG(dbgs() << " " << Index);
3590  }
3591  LLVM_DEBUG(dbgs() << "\n");
3592  NextGroupIndex++;
3593  }
3594  CurrentMembers.clear();
3595  }
3596 };
3597 
3598 bool FrameObjectCompare(const FrameObject &A, const FrameObject &B) {
3599  // Objects at a lower index are closer to FP; objects at a higher index are
3600  // closer to SP.
3601  //
3602  // For consistency in our comparison, all invalid objects are placed
3603  // at the end. This also allows us to stop walking when we hit the
3604  // first invalid item after it's all sorted.
3605  //
3606  // The "first" object goes first (closest to SP), followed by the members of
3607  // the "first" group.
3608  //
3609  // The rest are sorted by the group index to keep the groups together.
3610  // Higher numbered groups are more likely to be around longer (i.e. untagged
3611  // in the function epilogue and not at some earlier point). Place them closer
3612  // to SP.
3613  //
3614  // If all else equal, sort by the object index to keep the objects in the
3615  // original order.
3616  return std::make_tuple(!A.IsValid, A.ObjectFirst, A.GroupFirst, A.GroupIndex,
3617  A.ObjectIndex) <
3618  std::make_tuple(!B.IsValid, B.ObjectFirst, B.GroupFirst, B.GroupIndex,
3619  B.ObjectIndex);
3620 }
3621 } // namespace
3622 
3624  const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
3625  if (!OrderFrameObjects || ObjectsToAllocate.empty())
3626  return;
3627 
3628  const MachineFrameInfo &MFI = MF.getFrameInfo();
3629  std::vector<FrameObject> FrameObjects(MFI.getObjectIndexEnd());
3630  for (auto &Obj : ObjectsToAllocate) {
3631  FrameObjects[Obj].IsValid = true;
3632  FrameObjects[Obj].ObjectIndex = Obj;
3633  }
3634 
3635  // Identify stack slots that are tagged at the same time.
3636  GroupBuilder GB(FrameObjects);
3637  for (auto &MBB : MF) {
3638  for (auto &MI : MBB) {
3639  if (MI.isDebugInstr())
3640  continue;
3641  int OpIndex;
3642  switch (MI.getOpcode()) {
3643  case AArch64::STGloop:
3644  case AArch64::STZGloop:
3645  OpIndex = 3;
3646  break;
3647  case AArch64::STGOffset:
3648  case AArch64::STZGOffset:
3649  case AArch64::ST2GOffset:
3650  case AArch64::STZ2GOffset:
3651  OpIndex = 1;
3652  break;
3653  default:
3654  OpIndex = -1;
3655  }
3656 
3657  int TaggedFI = -1;
3658  if (OpIndex >= 0) {
3659  const MachineOperand &MO = MI.getOperand(OpIndex);
3660  if (MO.isFI()) {
3661  int FI = MO.getIndex();
3662  if (FI >= 0 && FI < MFI.getObjectIndexEnd() &&
3663  FrameObjects[FI].IsValid)
3664  TaggedFI = FI;
3665  }
3666  }
3667 
3668  // If this is a stack tagging instruction for a slot that is not part of a
3669  // group yet, either start a new group or add it to the current one.
3670  if (TaggedFI >= 0)
3671  GB.AddMember(TaggedFI);
3672  else
3673  GB.EndCurrentGroup();
3674  }
3675  // Groups should never span multiple basic blocks.
3676  GB.EndCurrentGroup();
3677  }
3678 
3679  // If the function's tagged base pointer is pinned to a stack slot, we want to
3680  // put that slot first when possible. This will likely place it at SP + 0,
3681  // and save one instruction when generating the base pointer because IRG does
3682  // not allow an immediate offset.
3683  const AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
3685  if (TBPI) {
3686  FrameObjects[*TBPI].ObjectFirst = true;
3687  FrameObjects[*TBPI].GroupFirst = true;
3688  int FirstGroupIndex = FrameObjects[*TBPI].GroupIndex;
3689  if (FirstGroupIndex >= 0)
3690  for (FrameObject &Object : FrameObjects)
3691  if (Object.GroupIndex == FirstGroupIndex)
3692  Object.GroupFirst = true;
3693  }
3694 
3695  llvm::stable_sort(FrameObjects, FrameObjectCompare);
3696 
3697  int i = 0;
3698  for (auto &Obj : FrameObjects) {
3699  // All invalid items are sorted at the end, so it's safe to stop.
3700  if (!Obj.IsValid)
3701  break;
3702  ObjectsToAllocate[i++] = Obj.ObjectIndex;
3703  }
3704 
3705  LLVM_DEBUG(dbgs() << "Final frame order:\n"; for (auto &Obj
3706  : FrameObjects) {
3707  if (!Obj.IsValid)
3708  break;
3709  dbgs() << " " << Obj.ObjectIndex << ": group " << Obj.GroupIndex;
3710  if (Obj.ObjectFirst)
3711  dbgs() << ", first";
3712  if (Obj.GroupFirst)
3713  dbgs() << ", group-first";
3714  dbgs() << "\n";
3715  });
3716 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::MachineFunction::hasWinCFI
bool hasWinCFI() const
Definition: MachineFunction.h:708
i
i
Definition: README.txt:29
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:148
llvm::isAsynchronousEHPersonality
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
Definition: EHPersonalities.h:50
llvm::AArch64ISD::LOADgot
@ LOADgot
Definition: AArch64ISelLowering.h:64
llvm::MachineFrameInfo::isMaxCallFrameSizeComputed
bool isMaxCallFrameSizeComputed() const
Definition: MachineFrameInfo.h:629
llvm::MachineFrameInfo::hasVarSizedObjects
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
Definition: MachineFrameInfo.h:353
llvm::AArch64Subtarget::isTargetWindows
bool isTargetWindows() const
Definition: AArch64Subtarget.h:506
AArch64RegisterInfo.h
Attrs
Function Attrs
Definition: README_ALTIVEC.txt:215
llvm::CallingConv::SwiftTail
@ SwiftTail
SwiftTail - This follows the Swift calling convention in how arguments are passed but guarantees tail...
Definition: CallingConv.h:92
MCDwarf.h
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:103
MachineInstr.h
MathExtras.h
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm::MachineFrameInfo::estimateStackSize
uint64_t estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
Definition: MachineFrameInfo.cpp:137
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
llvm::MachineInstrBuilder::copyImplicitOps
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
Definition: MachineInstrBuilder.h:315
AArch64MachineFunctionInfo.h
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
llvm::MachineRegisterInfo::isPhysRegUsed
bool isPhysRegUsed(MCRegister PhysReg, bool SkipRegMaskTest=false) const
Return true if the specified register is modified or read in this function.
Definition: MachineRegisterInfo.cpp:585
llvm::MCSymbol
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
llvm::AArch64Subtarget::swiftAsyncContextIsDynamicallySet
bool swiftAsyncContextIsDynamicallySet() const
Return whether FrameLowering should always set the "extended frame present" bit in FP,...
Definition: AArch64Subtarget.h:605
getFixedObjectSize
static unsigned getFixedObjectSize(const MachineFunction &MF, const AArch64FunctionInfo *AFI, bool IsWin64, bool IsFunclet)
Returns the size of the fixed object area (allocated next to sp on entry) On Win64 this may include a...
Definition: AArch64FrameLowering.cpp:315
llvm::LivePhysRegs::addReg
void addReg(MCPhysReg Reg)
Adds a physical register and all its sub-registers to the set.
Definition: LivePhysRegs.h:79
DefaultSafeSPDisplacement
static const unsigned DefaultSafeSPDisplacement
This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using ...
Definition: AArch64FrameLowering.cpp:278
llvm::AArch64_AM::LSL
@ LSL
Definition: AArch64AddressingModes.h:34
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:158
llvm::MachineModuleInfo::getContext
const MCContext & getContext() const
Definition: MachineModuleInfo.h:167
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
produceCompactUnwindFrame
static bool produceCompactUnwindFrame(MachineFunction &MF)
Definition: AArch64FrameLowering.cpp:2185
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition: MachineInstrBuilder.h:224
llvm::Function
Definition: Function.h:61
llvm::BitVector::set
BitVector & set()
Definition: BitVector.h:343
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:92
llvm::MachineInstrBuilder::addCFIIndex
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
Definition: MachineInstrBuilder.h:247
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:625
llvm::MachineBasicBlock::isEHFuncletEntry
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
Definition: MachineBasicBlock.h:552
contains
return AArch64::GPR64RegClass contains(Reg)
llvm::CodeModel::Medium
@ Medium
Definition: CodeGen.h:28
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
Statistic.h
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:430
ErrorHandling.h
llvm::MCRegisterInfo::getDwarfRegNum
int getDwarfRegNum(MCRegister RegNum, bool isEH) const
Map a target register to an equivalent dwarf register number.
Definition: MCRegisterInfo.cpp:68
llvm::NVPTX::PTXCvtMode::RPI
@ RPI
Definition: NVPTX.h:135
llvm::getBLRCallOpcode
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
Definition: AArch64InstrInfo.cpp:7557
llvm::AArch64RegisterInfo::getBaseRegister
unsigned getBaseRegister() const
Definition: AArch64RegisterInfo.cpp:382
llvm::AArch64FunctionInfo::getTaggedBasePointerIndex
Optional< int > getTaggedBasePointerIndex() const
Definition: AArch64MachineFunctionInfo.h:375
llvm::BitVector::set_bits
iterator_range< const_set_bits_iterator > set_bits() const
Definition: BitVector.h:132
llvm::AArch64FunctionInfo::setSwiftAsyncContextFrameIdx
void setSwiftAsyncContextFrameIdx(int FI)
Definition: AArch64MachineFunctionInfo.h:406
MachineBasicBlock.h
llvm::LivePhysRegs
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:48
Right
Vector Shift Left Right
Definition: README_P9.txt:118
llvm::TargetSubtargetInfo::getRegisterInfo
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Definition: TargetSubtargetInfo.h:124
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
llvm::MachineOperand::setImm
void setImm(int64_t immVal)
Definition: MachineOperand.h:655
llvm::MachineBasicBlock::findDebugLoc
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any DBG_VALUE and DBG_LABEL instructions.
Definition: MachineBasicBlock.cpp:1373
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:233
Shift
bool Shift
Definition: README.txt:468
llvm::AArch64FrameLowering::enableStackSlotScavenging
bool enableStackSlotScavenging(const MachineFunction &MF) const override
Returns true if the stack slot holes in the fixed and callee-save stack area should be used when allo...
Definition: AArch64FrameLowering.cpp:2971
llvm::AArch64Subtarget::getInstrInfo
const AArch64InstrInfo * getInstrInfo() const override
Definition: AArch64Subtarget.h:319
llvm::StackOffset::getFixed
ScalarTy getFixed() const
Definition: TypeSize.h:149
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::reverse
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Definition: STLExtras.h:329
llvm::AttributeList
Definition: Attributes.h:398
TargetInstrInfo.h
llvm::RegState::Implicit
@ Implicit
Not emitted register (e.g. carry, or temporary result).
Definition: MachineInstrBuilder.h:46
fixupSEHOpcode
static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, unsigned LocalStackSize)
Definition: AArch64FrameLowering.cpp:859
getPrologueDeath
static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg)
Definition: AArch64FrameLowering.cpp:2175
llvm::Optional< int >
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
llvm::AArch64FrameLowering::assignCalleeSavedSpillSlots
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI, unsigned &MinCSFrameIndex, unsigned &MaxCSFrameIndex) const override
assignCalleeSavedSpillSlots - Allows target to override spill slot assignment logic.
Definition: AArch64FrameLowering.cpp:2928
llvm::AArch64FrameLowering::hasFP
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register.
Definition: AArch64FrameLowering.cpp:359
llvm::MachineFrameInfo::getObjectIndexEnd
int getObjectIndexEnd() const
Return one past the maximum frame object index.
Definition: MachineFrameInfo.h:393
determineSVEStackObjectOffsets
static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex, bool AssignOffsets)
Definition: AArch64FrameLowering.cpp:3006
llvm::CodeModel::Kernel
@ Kernel
Definition: CodeGen.h:28
llvm::MachineOperand::isFI
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
Definition: MachineOperand.h:331
llvm::AArch64FunctionInfo::setTaggedBasePointerOffset
void setTaggedBasePointerOffset(unsigned Offset)
Definition: AArch64MachineFunctionInfo.h:383
llvm::AArch64FrameLowering::emitCalleeSavedFrameMoves
void emitCalleeSavedFrameMoves(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const override
With basic block sections, emit callee saved frame moves for basic blocks that are in a different sec...
Definition: AArch64FrameLowering.cpp:535
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1567
llvm::RegScavenger::FindUnusedReg
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
Definition: RegisterScavenging.cpp:268
llvm::MachineFrameInfo::getMaxCallFrameSize
unsigned getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
Definition: MachineFrameInfo.h:622
llvm::ARCISD::BL
@ BL
Definition: ARCISelLowering.h:34
llvm::ArrayRef::empty
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::AArch64Subtarget::getTargetLowering
const AArch64TargetLowering * getTargetLowering() const override
Definition: AArch64Subtarget.h:316
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::MCCFIInstruction::cfiDefCfaOffset
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int Offset)
.cfi_def_cfa_offset modifies a rule for computing CFA.
Definition: MCDwarf.h:502
MachineRegisterInfo.h
llvm::MachineInstr::FrameDestroy
@ FrameDestroy
Definition: MachineInstr.h:84
isTargetWindows
static bool isTargetWindows(const MachineFunction &MF)
Definition: AArch64FrameLowering.cpp:1084
llvm::MachineBasicBlock::erase
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
Definition: MachineBasicBlock.cpp:1299
llvm::SwiftAsyncFramePointerMode::Never
@ Never
Never set the bit.
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::AArch64RegisterInfo::cannotEliminateFrame
bool cannotEliminateFrame(const MachineFunction &MF) const
Definition: AArch64RegisterInfo.cpp:460
clear
static void clear(coro::Shape &Shape)
Definition: Coroutines.cpp:233
llvm::classifyEHPersonality
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Definition: EHPersonalities.cpp:21
llvm::AArch64FrameLowering
Definition: AArch64FrameLowering.h:23
llvm::AArch64FrameOffsetCannotUpdate
@ AArch64FrameOffsetCannotUpdate
Offset cannot apply.
Definition: AArch64InstrInfo.h:386
llvm::CallingConv::Win64
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
Definition: CallingConv.h:169
llvm::MachineInstr::getFlags
uint16_t getFlags() const
Return the MI flags bitvector.
Definition: MachineInstr.h:325
llvm::AlignStyle::Left
@ Left
ReverseCSRRestoreSeq
static cl::opt< bool > ReverseCSRRestoreSeq("reverse-csr-restore-seq", cl::desc("reverse the CSR restore sequence"), cl::init(false), cl::Hidden)
CommandLine.h
llvm::AArch64FrameLowering::processFunctionBeforeFrameFinalized
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
Definition: AArch64FrameLowering.cpp:3083
llvm::MachineInstrBuilder::addDef
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Definition: MachineInstrBuilder.h:116
llvm::getDefRegState
unsigned getDefRegState(bool B)
Definition: MachineInstrBuilder.h:502
llvm::MachineFunction::front
const MachineBasicBlock & front() const
Definition: MachineFunction.h:820
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:636
OrderFrameObjects
static cl::opt< bool > OrderFrameObjects("aarch64-order-frame-objects", cl::desc("sort stack allocations"), cl::init(true), cl::Hidden)
llvm::MachineBasicBlock::insertAfter
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
Definition: MachineBasicBlock.h:871
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
AArch64TargetMachine.h
AArch64InstrInfo.h
llvm::AArch64FrameLowering::resolveFrameOffsetReference
StackOffset resolveFrameOffsetReference(const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE, Register &FrameReg, bool PreferFP, bool ForSimm) const
Definition: AArch64FrameLowering.cpp:2042
llvm::TargetFrameLowering::getOffsetOfLocalArea
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
Definition: TargetFrameLowering.h:140
TargetMachine.h
llvm::MutableArrayRef
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
llvm::AArch64FunctionInfo::isStackRealigned
bool isStackRealigned() const
Definition: AArch64MachineFunctionInfo.h:206
llvm::MachineOperand::CreateImm
static MachineOperand CreateImm(int64_t Val)
Definition: MachineOperand.h:773
llvm::AArch64InstrInfo
Definition: AArch64InstrInfo.h:38
llvm::encodeSLEB128
unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a SLEB128 value to an output stream.
Definition: LEB128.h:23
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::SmallVectorImpl::append
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:648
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:537
llvm::AArch64FrameLowering::eliminateCallFramePseudoInstr
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
Definition: AArch64FrameLowering.cpp:399
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:724
invalidateWindowsRegisterPairing
static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2, bool NeedsWinCFI, bool IsFirst)
Definition: AArch64FrameLowering.cpp:2194
llvm::LivePhysRegs::addLiveIns
void addLiveIns(const MachineBasicBlock &MBB)
Adds all live-in registers of basic block MBB.
Definition: LivePhysRegs.cpp:236
llvm::ISD::CATCHRET
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:999
llvm::AArch64Subtarget::isTargetILP32
bool isTargetILP32() const
Definition: AArch64Subtarget.h:514
llvm::TargetFrameLowering::getStackAlign
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Definition: TargetFrameLowering.h:100
llvm::ARM_PROC::A
@ A
Definition: ARMBaseInfo.h:34
llvm::MachineRegisterInfo::isReserved
bool isReserved(MCRegister PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
Definition: MachineRegisterInfo.h:915
llvm::BitVector::count
size_type count() const
count - Returns the number of bits which are set.
Definition: BitVector.h:154
llvm::AArch64TargetLowering::supportSwiftError
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
Definition: AArch64ISelLowering.h:785
llvm::Log2
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition: Alignment.h:207
int
Clang compiles this i1 i64 store i64 i64 store i64 i64 store i64 i64 store i64 align Which gets codegen d xmm0 movaps rbp movaps rbp movaps rbp movaps rbp rbp rbp rbp rbp It would be better to have movq s of instead of the movaps s LLVM produces ret int
Definition: README.txt:536
llvm::AArch64FunctionInfo::setHasRedZone
void setHasRedZone(bool s)
Definition: AArch64MachineFunctionInfo.h:305
getStackOffset
static StackOffset getStackOffset(const MachineFunction &MF, int64_t ObjectOffset)
Definition: AArch64FrameLowering.cpp:2014
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:129
llvm::TypeSize::Fixed
static TypeSize Fixed(ScalarTy MinVal)
Definition: TypeSize.h:423
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:49
llvm::CodeModel::Small
@ Small
Definition: CodeGen.h:28
llvm::MachineInstr::FrameSetup
@ FrameSetup
Definition: MachineInstr.h:82
llvm::make_scope_exit
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:58
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:146
llvm::MachineModuleInfo
This class contains meta information specific to a module.
Definition: MachineModuleInfo.h:78
getSVEStackSize
static StackOffset getSVEStackSize(const MachineFunction &MF)
Returns the size of the entire SVE stackframe (calleesaves + spills).
Definition: AArch64FrameLowering.cpp:332
findScratchNonCalleeSaveRegister
static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB)
Definition: AArch64FrameLowering.cpp:588
llvm::AArch64_AM::getShifterImm
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
Definition: AArch64AddressingModes.h:98
llvm::RegScavenger::backward
void backward()
Update internal register state and move MBB iterator backwards.
Definition: RegisterScavenging.cpp:241
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::MachineFrameInfo::getStackID
uint8_t getStackID(int ObjectIdx) const
Definition: MachineFrameInfo.h:699
llvm::RegScavenger::enterBasicBlockEnd
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
Definition: RegisterScavenging.cpp:89
llvm::MachineBasicBlock::instr_back
MachineInstr & instr_back()
Definition: MachineBasicBlock.h:243
llvm::AArch64_AM::getShiftValue
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
Definition: AArch64AddressingModes.h:85
llvm::MachineFunction::setHasWinCFI
void setHasWinCFI(bool v)
Definition: MachineFunction.h:711
llvm::MachineFrameInfo::getStackSize
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
Definition: MachineFrameInfo.h:553
DebugLoc.h
llvm::MachineFrameInfo::getObjectOffset
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
Definition: MachineFrameInfo.h:494
llvm::AArch64FrameLowering::getFrameIndexReference
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - Provide a base+offset reference to an FI slot for debug info.
Definition: AArch64FrameLowering.cpp:1985
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::BitVector
Definition: BitVector.h:74
appendVGScaledOffsetExpr
static void appendVGScaledOffsetExpr(SmallVectorImpl< char > &Expr, int NumBytes, int NumVGScaledBytes, unsigned VG, llvm::raw_string_ostream &Comment)
Definition: AArch64FrameLowering.cpp:445
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
llvm::StackOffset::getScalable
ScalarTy getScalable() const
Definition: TypeSize.h:150
llvm::MCCFIInstruction::createEscape
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
Definition: MCDwarf.h:587
llvm::emitFrameOffset
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
Definition: AArch64InstrInfo.cpp:4041
llvm::AArch64FunctionInfo::hasStackFrame
bool hasStackFrame() const
Definition: AArch64MachineFunctionInfo.h:203
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::MachineFrameInfo::getObjectIndexBegin
int getObjectIndexBegin() const
Return the minimum frame object index.
Definition: MachineFrameInfo.h:390
llvm::StringRef::str
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:245
llvm::MachineInstrBuilder::addExternalSymbol
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:184
llvm::None
const NoneType None
Definition: None.h:23
llvm::MachineFrameInfo::isCalleeSavedInfoValid
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
Definition: MachineFrameInfo.h:768
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
fixupCalleeSaveRestoreStackOffset
static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, uint64_t LocalStackSize, bool NeedsWinCFI, bool *HasWinCFI)
Definition: AArch64FrameLowering.cpp:993
llvm::MCCFIInstruction
Definition: MCDwarf.h:434
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
llvm::MachineFrameInfo::isDeadObjectIndex
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
Definition: MachineFrameInfo.h:713
llvm::SmallString< 64 >
llvm::Function::getAttributes
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:326
AArch64AddressingModes.h
llvm::OutputFileType::Object
@ Object
llvm::cl::ZeroOrMore
@ ZeroOrMore
Definition: CommandLine.h:120
StackTaggingMergeSetTag
static cl::opt< bool > StackTaggingMergeSetTag("stack-tagging-merge-settag", cl::desc("merge settag instruction in function epilog"), cl::init(true), cl::Hidden)
llvm::TargetOptions::DisableFramePointerElim
bool DisableFramePointerElim(const MachineFunction &MF) const
DisableFramePointerElim - This returns true if frame pointer elimination optimization should be disab...
Definition: TargetOptionsImpl.cpp:24
llvm::MachineFunction::getMMI
MachineModuleInfo & getMMI() const
Definition: MachineFunction.h:577
llvm::SmallString::append
void append(StringRef RHS)
Append from a StringRef.
Definition: SmallString.h:67
llvm::TargetRegisterInfo::getSpillAlign
Align getSpillAlign(const TargetRegisterClass &RC) const
Return the minimum required alignment in bytes for a spill slot for a register of this class.
Definition: TargetRegisterInfo.h:288
llvm::AArch64FrameLowering::resolveFrameIndexReference
StackOffset resolveFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP, bool ForSimm) const
Definition: AArch64FrameLowering.cpp:2031
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:626
llvm::MachineInstrBuilder::addFrameIndex
const MachineInstrBuilder & addFrameIndex(int Idx) const
Definition: MachineInstrBuilder.h:152
llvm::MachineInstrBuilder::setMIFlag
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
Definition: MachineInstrBuilder.h:278
llvm::isAArch64FrameOffsetLegal
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
Definition: AArch64InstrInfo.cpp:4261
llvm::AArch64FunctionInfo::getCalleeSaveBaseToFrameRecordOffset
int getCalleeSaveBaseToFrameRecordOffset() const
Definition: AArch64MachineFunctionInfo.h:387
llvm::Function::hasFnAttribute
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:626
llvm::cl::opt< bool >
llvm::TargetRegisterInfo::getSpillSize
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
Definition: TargetRegisterInfo.h:282
llvm::WinEHFuncInfo
Definition: WinEHFuncInfo.h:90
getSVECalleeSaveSlotRange
static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI, int &Min, int &Max)
returns true if there are any SVE callee saves.
Definition: AArch64FrameLowering.cpp:2978
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
uint64_t
llvm::MachineFrameInfo::getObjectSize
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Definition: MachineFrameInfo.h:453
AArch64FrameLowering.h
llvm::Function::getCallingConv
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:239
computeCalleeSaveRegisterPairs
static void computeCalleeSaveRegisterPairs(MachineFunction &MF, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI, SmallVectorImpl< RegPairInfo > &RegPairs, bool &NeedShadowCallStackProlog, bool NeedsFrameRecord)
Definition: AArch64FrameLowering.cpp:2270
llvm::AArch64FrameLowering::spillCalleeSavedRegisters
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
Definition: AArch64FrameLowering.cpp:2470
llvm::TargetOptions::SwiftAsyncFramePointer
SwiftAsyncFramePointerMode SwiftAsyncFramePointer
Control when and how the Swift async frame pointer bit should be set.
Definition: TargetOptions.h:235
LEB128.h
llvm::LivePhysRegs::available
bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const
Returns true if register Reg and no aliasing register is in the set.
Definition: LivePhysRegs.cpp:139
llvm::MCCFIInstruction::createNegateRAState
static MCCFIInstruction createNegateRAState(MCSymbol *L)
.cfi_negate_ra_state AArch64 negate RA state.
Definition: MCDwarf.h:552
llvm::MachineRegisterInfo::getCalleeSavedRegs
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
Definition: MachineRegisterInfo.cpp:621
llvm::AArch64FunctionInfo::setCalleeSaveBaseToFrameRecordOffset
void setCalleeSaveBaseToFrameRecordOffset(int Offset)
Definition: AArch64MachineFunctionInfo.h:390
llvm::ISD::CLEANUPRET
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:1003
llvm::AArch64FunctionInfo
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
Definition: AArch64MachineFunctionInfo.h:37
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::MCPhysReg
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:20
llvm::RegScavenger
Definition: RegisterScavenging.h:34
llvm::MachineFrameInfo::getObjectAlign
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:467
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::MachineFrameInfo::setObjectOffset
void setObjectOffset(int ObjectIdx, int64_t SPOffset)
Set the stack frame offset of the specified object.
Definition: MachineFrameInfo.h:528
llvm::TargetStackID::ScalableVector
@ ScalableVector
Definition: TargetFrameLowering.h:30
llvm::StackOffset::getScalable
static StackOffset getScalable(ScalarTy Scalable)
Definition: TypeSize.h:144
windowsRequiresStackProbe
static bool windowsRequiresStackProbe(MachineFunction &MF, uint64_t StackSizeInBytes)
Definition: AArch64FrameLowering.cpp:632
llvm::MachineBasicBlock::getLastNonDebugInstr
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
Definition: MachineBasicBlock.cpp:267
llvm::AArch64FrameLowering::getStackIDForScalableVectors
TargetStackID::Value getStackIDForScalableVectors() const override
Returns the StackID that scalable vectors should be associated with.
Definition: AArch64FrameLowering.cpp:309
llvm::TargetMachine::Options
TargetOptions Options
Definition: TargetMachine.h:120
llvm::MachineInstr::setFlags
void setFlags(unsigned flags)
Definition: MachineInstr.h:339
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::AArch64FunctionInfo::getLocalStackSize
uint64_t getLocalStackSize() const
Definition: AArch64MachineFunctionInfo.h:219
InsertSEH
static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI, const TargetInstrInfo &TII, MachineInstr::MIFlag Flag)
Definition: AArch64FrameLowering.cpp:739
llvm::MachineFrameInfo::CreateFixedObject
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
Definition: MachineFrameInfo.cpp:83
llvm::AArch64FunctionInfo::setStackRealigned
void setStackRealigned(bool s)
Definition: AArch64MachineFunctionInfo.h:207
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:840
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:642
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:225
llvm::RegScavenger::addScavengingFrameIndex
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
Definition: RegisterScavenging.h:123
llvm::MachineInstrBuilder::addMemOperand
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Definition: MachineInstrBuilder.h:202
llvm::TargetFrameLowering::getStackGrowthDirection
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
Definition: TargetFrameLowering.h:89
llvm::AArch64FunctionInfo::getVarArgsGPRSize
unsigned getVarArgsGPRSize() const
Definition: AArch64MachineFunctionInfo.h:313
llvm::AArch64Subtarget::isCallingConvWin64
bool isCallingConvWin64(CallingConv::ID CC) const
Definition: AArch64Subtarget.h:590
llvm::MCCFIInstruction::cfiDefCfa
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int Offset)
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
Definition: MCDwarf.h:488
MachineModuleInfo.h
llvm::WinEHFuncInfo::UnwindHelpFrameIdx
int UnwindHelpFrameIdx
Definition: WinEHFuncInfo.h:99
llvm::AArch64FrameLowering::getNonLocalFrameIndexReference
StackOffset getNonLocalFrameIndexReference(const MachineFunction &MF, int FI) const override
getNonLocalFrameIndexReference - This method returns the offset used to reference a frame index locat...
Definition: AArch64FrameLowering.cpp:1995
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Definition: MachineInstrBuilder.h:123
llvm::AArch64FrameLowering::canUseRedZone
bool canUseRedZone(const MachineFunction &MF) const
Can this function use the red zone for local allocations.
Definition: AArch64FrameLowering.cpp:337
llvm::RegState::InternalRead
@ InternalRead
Register reads a value that is defined inside the same instruction or bundle.
Definition: MachineInstrBuilder.h:59
llvm::MachineInstr::MIFlag
MIFlag
Definition: MachineInstr.h:80
llvm::MachineFunction
Definition: MachineFunction.h:230
convertCalleeSaveRestoreToSPPrePostIncDec
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, bool NeedsWinCFI, bool *HasWinCFI, bool InProlog=true)
Definition: AArch64FrameLowering.cpp:881
TargetOptions.h
llvm::MachineFrameInfo::getCalleeSavedInfo
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
Definition: MachineFrameInfo.h:755
llvm::MachineBasicBlock::getFirstTerminator
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Definition: MachineBasicBlock.cpp:242
llvm::TargetMachine::getMCAsmInfo
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
Definition: TargetMachine.h:207
EnableRedZone
static cl::opt< bool > EnableRedZone("aarch64-redzone", cl::desc("enable use of redzone on AArch64"), cl::init(false), cl::Hidden)
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
adaptForLdStOpt
static void adaptForLdStOpt(MachineBasicBlock &MBB, MachineBasicBlock::iterator FirstSPPopI, MachineBasicBlock::iterator LastPopI)
Definition: AArch64FrameLowering.cpp:1052
llvm::AArch64Subtarget::isXRegisterReserved
bool isXRegisterReserved(size_t i) const
Definition: AArch64Subtarget.h:367
llvm::MachineFrameInfo::setStackID
void setStackID(int ObjectIdx, uint8_t ID)
Definition: MachineFrameInfo.h:704
llvm::MachineFrameInfo::hasPatchPoint
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
Definition: MachineFrameInfo.h:386
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
MCAsmInfo.h
DataLayout.h
llvm::MachineFrameInfo::CreateStackObject
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Definition: MachineFrameInfo.cpp:51
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:950
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: AArch64SLSHardening.cpp:75
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
if
if(llvm_vc STREQUAL "") set(fake_version_inc "$
Definition: CMakeLists.txt:14
uint32_t
llvm::StackOffset
StackOffset is a class to represent an offset with 2 dimensions, named fixed and scalable,...
Definition: TypeSize.h:134
TargetSubtargetInfo.h
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::SwiftAsyncFramePointerMode::DeploymentBased
@ DeploymentBased
Determine whether to set the bit statically or dynamically based on the deployment target.
llvm::AArch64FunctionInfo::getTailCallReservedStack
unsigned getTailCallReservedStack() const
Definition: AArch64MachineFunctionInfo.h:189
llvm::AArch64Subtarget::hasPAuth
bool hasPAuth() const
Definition: AArch64Subtarget.h:531
llvm::AArch64RegisterInfo
Definition: AArch64RegisterInfo.h:26
llvm::AArch64FunctionInfo::setMinMaxSVECSFrameIndex
void setMinMaxSVECSFrameIndex(int Min, int Max)
Definition: AArch64MachineFunctionInfo.h:291
LLVM_FALLTHROUGH
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:273
llvm::MCContext::createTempSymbol
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Definition: MCContext.cpp:303
llvm::CodeModel::Tiny
@ Tiny
Definition: CodeGen.h:28
llvm::EHPersonality
EHPersonality
Definition: EHPersonalities.h:22
llvm::StackOffset::getFixed
static StackOffset getFixed(ScalarTy Fixed)
Definition: TypeSize.h:143
llvm::TargetSubtargetInfo
TargetSubtargetInfo - Generic base class for all target subtargets.
Definition: TargetSubtargetInfo.h:59
Prolog
@ Prolog
Definition: AArch64LowerHomogeneousPrologEpilog.cpp:125
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:135
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::MachineFrameInfo::getMaxAlign
Align getMaxAlign() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
Definition: MachineFrameInfo.h:569
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::Function::hasOptSize
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
Definition: Function.h:671
llvm::MachineBasicBlock::addLiveIn
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
Definition: MachineBasicBlock.h:367
llvm::MachineFrameInfo::isFrameAddressTaken
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
Definition: MachineFrameInfo.h:368
llvm::AArch64_AM::getArithExtendImm
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
Definition: AArch64AddressingModes.h:170
llvm::MachineFrameInfo::hasCalls
bool hasCalls() const
Return true if the current function has any function calls.
Definition: MachineFrameInfo.h:581
llvm::AArch64FrameLowering::hasReservedCallFrame
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
Definition: AArch64FrameLowering.cpp:395
CallingConv.h
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::AArch64Subtarget::getRegisterInfo
const AArch64RegisterInfo * getRegisterInfo() const override
Definition: AArch64Subtarget.h:320
Attributes.h
isFuncletReturnInstr
static bool isFuncletReturnInstr(const MachineInstr &MI)
Definition: AArch64FrameLowering.cpp:1671
llvm::BitVector::test
bool test(unsigned Idx) const
Definition: BitVector.h:447
llvm::stable_sort
void stable_sort(R &&Range)
Definition: STLExtras.h:1682
llvm::AArch64_AM::UXTX
@ UXTX
Definition: AArch64AddressingModes.h:43
llvm::AArch64FrameLowering::getWinEHParentFrameOffset
unsigned getWinEHParentFrameOffset(const MachineFunction &MF) const override
The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve the parent's frame pointer...
Definition: AArch64FrameLowering.cpp:3542
llvm::MachineRegisterInfo::isLiveIn
bool isLiveIn(Register Reg) const
Definition: MachineRegisterInfo.cpp:436
kSetTagLoopThreshold
static const int kSetTagLoopThreshold
Definition: AArch64SelectionDAGInfo.cpp:58
llvm::AArch64FunctionInfo::hasCalleeSaveStackFreeSpace
bool hasCalleeSaveStackFreeSpace() const
Definition: AArch64MachineFunctionInfo.h:209
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:592
llvm::MachineFunction::needsFrameMoves
bool needsFrameMoves() const
True if this function needs frame moves for debug or exceptions.
Definition: MachineFunction.cpp:587
llvm::TargetRegisterInfo::getRegSizeInBits
unsigned getRegSizeInBits(const TargetRegisterClass &RC) const
Return the size in bits of a register from class RC.
Definition: TargetRegisterInfo.h:276
uint16_t
llvm::AArch64FunctionInfo::getStackSizeSVE
uint64_t getStackSizeSVE() const
Definition: AArch64MachineFunctionInfo.h:201
llvm::MachineFunction::getTarget
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Definition: MachineFunction.h:622
MachineFrameInfo.h
llvm::MachineFunction::getWinEHFuncInfo
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
Definition: MachineFunction.h:670
llvm::AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced
void processFunctionBeforeFrameIndicesReplaced(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameIndicesReplaced - This method is called immediately before MO_FrameIndex op...
Definition: AArch64FrameLowering.cpp:3515
llvm::MachineOperand::getIndex
int getIndex() const
Definition: MachineOperand.h:557
Success
#define Success
Definition: AArch64Disassembler.cpp:260
llvm::TypeSize
Definition: TypeSize.h:417
llvm::CallingConv::PreserveMost
@ PreserveMost
Definition: CallingConv.h:66
Function.h
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::AArch64FunctionInfo::setLocalStackSize
void setLocalStackSize(uint64_t Size)
Definition: AArch64MachineFunctionInfo.h:218
llvm::MachineInstrBuilder::setMemRefs
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
Definition: MachineInstrBuilder.h:208
llvm::AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets
static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized)
Definition: AArch64InstrInfo.cpp:3898
llvm::TargetMachine::getCodeModel
CodeModel::Model getCodeModel() const
Returns the code model.
Definition: TargetMachine.cpp:74
llvm::AArch64FunctionInfo::getSVECalleeSavedStackSize
unsigned getSVECalleeSavedStackSize() const
Definition: AArch64MachineFunctionInfo.h:287
llvm::SmallVectorImpl::clear
void clear()
Definition: SmallVector.h:585
llvm::AArch64FrameLowering::getSEHFrameIndexOffset
int getSEHFrameIndexOffset(const MachineFunction &MF, int FI) const
Definition: AArch64FrameLowering.cpp:2021
llvm::AArch64FrameLowering::getFrameIndexReferencePreferSP
StackOffset getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI, Register &FrameReg, bool IgnoreSPUpdates) const override
For Win64 AArch64 EH, the offset to the Unwind object is from the SP before the update.
Definition: AArch64FrameLowering.cpp:3526
llvm::SmallString::str
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:259
llvm::MachineFunction::hasEHFunclets
bool hasEHFunclets() const
Definition: MachineFunction.h:1050
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:137
llvm::AMDGPU::Hwreg::Width
Width
Definition: SIDefines.h:410
WinEHFuncInfo.h
llvm::AArch64FrameLowering::emitPrologue
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
Definition: AArch64FrameLowering.cpp:1102
llvm::TargetRegisterInfo::hasStackRealignment
bool hasStackRealignment(const MachineFunction &MF) const
True if stack realignment is required and still possible.
Definition: TargetRegisterInfo.h:936
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:50
llvm::AArch64Subtarget::isTargetMachO
bool isTargetMachO() const
Definition: AArch64Subtarget.h:512
llvm::AArch64FunctionInfo::getArgumentStackToRestore
unsigned getArgumentStackToRestore() const
Definition: AArch64MachineFunctionInfo.h:184
llvm::CodeModel::Large
@ Large
Definition: CodeGen.h:28
InsertReturnAddressAuth
static void InsertReturnAddressAuth(MachineFunction &MF, MachineBasicBlock &MBB)
Definition: AArch64FrameLowering.cpp:1640
invalidateRegisterPairing
static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2, bool UsesWinAAPCS, bool NeedsWinCFI, bool NeedsFrameRecord, bool IsFirst)
Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
Definition: AArch64FrameLowering.cpp:2224
llvm::getKillRegState
unsigned getKillRegState(bool B)
Definition: MachineInstrBuilder.h:508
llvm::AArch64TargetLowering::getRedZoneSize
unsigned getRedZoneSize(const Function &F) const
Definition: AArch64ISelLowering.h:827
llvm::MachineFunction::addFrameInst
LLVM_NODISCARD unsigned addFrameInst(const MCCFIInstruction &Inst)
Definition: MachineFunction.cpp:285
llvm::MachineFrameInfo
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Definition: MachineFrameInfo.h:107
AArch64Subtarget.h
SmallVector.h
estimateRSStackSizeLimit
static unsigned estimateRSStackSizeLimit(MachineFunction &MF)
Look at each instruction that references stack frames and return the stack size limit beyond which so...
Definition: AArch64FrameLowering.cpp:283
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1003
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:268
MachineInstrBuilder.h
llvm::AArch64FrameLowering::emitEpilogue
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
Definition: AArch64FrameLowering.cpp:1681
llvm::MachineInstrBuilder::setMIFlags
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
Definition: MachineInstrBuilder.h:273
llvm::AArch64FunctionInfo::hasSwiftAsyncContext
bool hasSwiftAsyncContext() const
Definition: AArch64MachineFunctionInfo.h:404
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
getFPOffset
static StackOffset getFPOffset(const MachineFunction &MF, int64_t ObjectOffset)
Definition: AArch64FrameLowering.cpp:2000
llvm::AArch64FrameLowering::canUseAsPrologue
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
Definition: AArch64FrameLowering.cpp:617
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
llvm::AArch64FrameLowering::getWinEHFuncletFrameSize
unsigned getWinEHFuncletFrameSize(const MachineFunction &MF) const
Funclets only need to account for space for the callee saved registers, as the locals are accounted f...
Definition: AArch64FrameLowering.cpp:3549
llvm::RegState::Kill
@ Kill
The last use of a register.
Definition: MachineInstrBuilder.h:48
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:340
llvm::MachineBasicBlock::empty
bool empty() const
Definition: MachineBasicBlock.h:240
llvm::encodeULEB128
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
Definition: LEB128.h:80
ScopeExit.h
llvm::AArch64FunctionInfo::getCalleeSavedStackSize
unsigned getCalleeSavedStackSize(const MachineFrameInfo &MFI) const
Definition: AArch64MachineFunctionInfo.h:234
MachineMemOperand.h
llvm::SmallVectorImpl< char >
MachineOperand.h
llvm::TargetFrameLowering::StackGrowsDown
@ StackGrowsDown
Definition: TargetFrameLowering.h:47
llvm::Function::hasMinSize
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:668
llvm::MCCFIInstruction::createOffset
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset)
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
Definition: MCDwarf.h:526
llvm::AArch64FrameLowering::determineCalleeSaves
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
Definition: AArch64FrameLowering.cpp:2755
llvm::AArch64FrameLowering::restoreCalleeSavedRegisters
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
Definition: AArch64FrameLowering.cpp:2628
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::TargetFrameLowering::determineCalleeSaves
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
Definition: TargetFrameLoweringImpl.cpp:78
llvm::MachineFrameInfo::hasStackMap
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
Definition: MachineFrameInfo.h:380
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::cl::desc
Definition: CommandLine.h:414
llvm::TargetRegisterInfo::getMinimalPhysRegClass
const TargetRegisterClass * getMinimalPhysRegClass(MCRegister Reg, MVT VT=MVT::Other) const
Returns the Register Class of a physical register of the given type, picking the most sub register cl...
Definition: TargetRegisterInfo.cpp:211
RegisterScavenging.h
llvm::AArch64Subtarget
Definition: AArch64Subtarget.h:38
raw_ostream.h
MachineFunction.h
llvm::printReg
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Definition: TargetRegisterInfo.cpp:110
llvm::AArch64FrameLowering::orderFrameObjects
void orderFrameObjects(const MachineFunction &MF, SmallVectorImpl< int > &ObjectsToAllocate) const override
Order the symbols in the local stack frame.
Definition: AArch64FrameLowering.cpp:3623
llvm::AArch64RegisterInfo::hasBasePointer
bool hasBasePointer(const MachineFunction &MF) const
Definition: AArch64RegisterInfo.cpp:384
llvm::MachineInstr::eraseFromParent
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Definition: MachineInstr.cpp:677
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::MCAsmInfo::usesWindowsCFI
bool usesWindowsCFI() const
Definition: MCAsmInfo.h:778
llvm::abs
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1284
llvm::HexagonInstrInfo::copyPhysReg
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
Emit instructions to copy a pair of physical registers.
Definition: HexagonInstrInfo.cpp:812
llvm::AArch64II::MO_GOT
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
Definition: AArch64BaseInfo.h:713
TargetRegisterInfo.h
EnableHomogeneousPrologEpilog
cl::opt< bool > EnableHomogeneousPrologEpilog("homogeneous-prolog-epilog", cl::init(false), cl::ZeroOrMore, cl::Hidden, cl::desc("Emit homogeneous prologue and epilogue for the size " "optimization (default = off)"))
Debug.h
needsWinCFI
static bool needsWinCFI(const MachineFunction &MF)
Definition: AArch64FrameLowering.cpp:649
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:270
llvm::SwiftAsyncFramePointerMode::Always
@ Always
Always set the bit.
llvm::CallingConv::GHC
@ GHC
Definition: CallingConv.h:51
llvm::AArch64RegisterInfo::isReservedReg
bool isReservedReg(const MachineFunction &MF, MCRegister Reg) const
Definition: AArch64RegisterInfo.cpp:342
llvm::AArch64InstrInfo::isSEHInstruction
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
Definition: AArch64InstrInfo.cpp:1007
llvm::AArch64FunctionInfo::setStackSizeSVE
void setStackSizeSVE(uint64_t S)
Definition: AArch64MachineFunctionInfo.h:196
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:908
getArgumentStackToRestore
static int64_t getArgumentStackToRestore(MachineFunction &MF, MachineBasicBlock &MBB)
Returns how much of the incoming argument stack area (in bytes) we should clean up in an epilogue.
Definition: AArch64FrameLowering.cpp:202
IsSVECalleeSave
static bool IsSVECalleeSave(MachineBasicBlock::iterator I)
Definition: AArch64FrameLowering.cpp:1089
LivePhysRegs.h
llvm::StackOffset::get
static StackOffset get(ScalarTy Fixed, ScalarTy Scalable)
Definition: TypeSize.h:145