LLVM  10.0.0svn
AArch64FrameLowering.cpp
Go to the documentation of this file.
1 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the AArch64 implementation of TargetFrameLowering class.
10 //
11 // On AArch64, stack frames are structured as follows:
12 //
13 // The stack grows downward.
14 //
15 // All of the individual frame areas on the frame below are optional, i.e. it's
16 // possible to create a function so that the particular area isn't present
17 // in the frame.
18 //
19 // At function entry, the "frame" looks as follows:
20 //
21 // | | Higher address
22 // |-----------------------------------|
23 // | |
24 // | arguments passed on the stack |
25 // | |
26 // |-----------------------------------| <- sp
27 // | | Lower address
28 //
29 //
30 // After the prologue has run, the frame has the following general structure.
31 // Note that this doesn't depict the case where a red-zone is used. Also,
32 // technically the last frame area (VLAs) doesn't get created until in the
33 // main function body, after the prologue is run. However, it's depicted here
34 // for completeness.
35 //
36 // | | Higher address
37 // |-----------------------------------|
38 // | |
39 // | arguments passed on the stack |
40 // | |
41 // |-----------------------------------|
42 // | |
43 // | (Win64 only) varargs from reg |
44 // | |
45 // |-----------------------------------|
46 // | |
47 // | callee-saved gpr registers | <--.
48 // | | | On Darwin platforms these
49 // |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
50 // | | | (frame record first)
51 // | prev_fp, prev_lr | <--'
52 // | (a.k.a. "frame record") |
53 // |-----------------------------------| <- fp(=x29)
54 // | |
55 // | callee-saved fp/simd/SVE regs |
56 // | |
57 // |-----------------------------------|
58 // | |
59 // | SVE stack objects |
60 // | |
61 // |-----------------------------------|
62 // |.empty.space.to.make.part.below....|
63 // |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
64 // |.the.standard.16-byte.alignment....| compile time; if present)
65 // |-----------------------------------|
66 // | |
67 // | local variables of fixed size |
68 // | including spill slots |
69 // |-----------------------------------| <- bp(not defined by ABI,
70 // |.variable-sized.local.variables....| LLVM chooses X19)
71 // |.(VLAs)............................| (size of this area is unknown at
72 // |...................................| compile time)
73 // |-----------------------------------| <- sp
74 // | | Lower address
75 //
76 //
77 // To access the data in a frame, at-compile time, a constant offset must be
78 // computable from one of the pointers (fp, bp, sp) to access it. The size
79 // of the areas with a dotted background cannot be computed at compile-time
80 // if they are present, making it required to have all three of fp, bp and
81 // sp to be set up to be able to access all contents in the frame areas,
82 // assuming all of the frame areas are non-empty.
83 //
84 // For most functions, some of the frame areas are empty. For those functions,
85 // it may not be necessary to set up fp or bp:
86 // * A base pointer is definitely needed when there are both VLAs and local
87 // variables with more-than-default alignment requirements.
88 // * A frame pointer is definitely needed when there are local variables with
89 // more-than-default alignment requirements.
90 //
91 // For Darwin platforms the frame-record (fp, lr) is stored at the top of the
92 // callee-saved area, since the unwind encoding does not allow for encoding
93 // this dynamically and existing tools depend on this layout. For other
94 // platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
95 // area to allow SVE stack objects (allocated directly below the callee-saves,
96 // if available) to be accessed directly from the framepointer.
97 // The SVE spill/fill instructions have VL-scaled addressing modes such
98 // as:
99 // ldr z8, [fp, #-7 mul vl]
100 // For SVE the size of the vector length (VL) is not known at compile-time, so
101 // '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
102 // layout, we don't need to add an unscaled offset to the framepointer before
103 // accessing the SVE object in the frame.
104 //
105 // In some cases when a base pointer is not strictly needed, it is generated
106 // anyway when offsets from the frame pointer to access local variables become
107 // so large that the offset can't be encoded in the immediate fields of loads
108 // or stores.
109 //
110 // FIXME: also explain the redzone concept.
111 // FIXME: also explain the concept of reserved call frames.
112 //
113 //===----------------------------------------------------------------------===//
114 
115 #include "AArch64FrameLowering.h"
116 #include "AArch64InstrInfo.h"
118 #include "AArch64RegisterInfo.h"
119 #include "AArch64StackOffset.h"
120 #include "AArch64Subtarget.h"
121 #include "AArch64TargetMachine.h"
123 #include "llvm/ADT/ScopeExit.h"
124 #include "llvm/ADT/SmallVector.h"
125 #include "llvm/ADT/Statistic.h"
141 #include "llvm/IR/Attributes.h"
142 #include "llvm/IR/CallingConv.h"
143 #include "llvm/IR/DataLayout.h"
144 #include "llvm/IR/DebugLoc.h"
145 #include "llvm/IR/Function.h"
146 #include "llvm/MC/MCAsmInfo.h"
147 #include "llvm/MC/MCDwarf.h"
149 #include "llvm/Support/Debug.h"
151 #include "llvm/Support/MathExtras.h"
155 #include <cassert>
156 #include <cstdint>
157 #include <iterator>
158 #include <vector>
159 
160 using namespace llvm;
161 
162 #define DEBUG_TYPE "frame-info"
163 
164 static cl::opt<bool> EnableRedZone("aarch64-redzone",
165  cl::desc("enable use of redzone on AArch64"),
166  cl::init(false), cl::Hidden);
167 
168 static cl::opt<bool>
169  ReverseCSRRestoreSeq("reverse-csr-restore-seq",
170  cl::desc("reverse the CSR restore sequence"),
171  cl::init(false), cl::Hidden);
172 
173 STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
174 
175 /// This is the biggest offset to the stack pointer we can encode in aarch64
176 /// instructions (without using a separate calculation and a temp register).
177 /// Note that the exception here are vector stores/loads which cannot encode any
178 /// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
179 static const unsigned DefaultSafeSPDisplacement = 255;
180 
181 /// Look at each instruction that references stack frames and return the stack
182 /// size limit beyond which some of these instructions will require a scratch
183 /// register during their expansion later.
185  // FIXME: For now, just conservatively guestimate based on unscaled indexing
186  // range. We'll end up allocating an unnecessary spill slot a lot, but
187  // realistically that's not a big deal at this stage of the game.
188  for (MachineBasicBlock &MBB : MF) {
189  for (MachineInstr &MI : MBB) {
190  if (MI.isDebugInstr() || MI.isPseudo() ||
191  MI.getOpcode() == AArch64::ADDXri ||
192  MI.getOpcode() == AArch64::ADDSXri)
193  continue;
194 
195  for (const MachineOperand &MO : MI.operands()) {
196  if (!MO.isFI())
197  continue;
198 
200  if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
202  return 0;
203  }
204  }
205  }
207 }
208 
209 /// Returns the size of the entire SVE stackframe (calleesaves + spills).
212  return {(int64_t)AFI->getStackSizeSVE(), MVT::nxv1i8};
213 }
214 
216  if (!EnableRedZone)
217  return false;
218  // Don't use the red zone if the function explicitly asks us not to.
219  // This is typically used for kernel code.
220  if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone))
221  return false;
222 
223  const MachineFrameInfo &MFI = MF.getFrameInfo();
225  unsigned NumBytes = AFI->getLocalStackSize();
226 
227  return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128 ||
228  getSVEStackSize(MF));
229 }
230 
231 /// hasFP - Return true if the specified function should have a dedicated frame
232 /// pointer register.
234  const MachineFrameInfo &MFI = MF.getFrameInfo();
235  const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
236  // Win64 EH requires a frame pointer if funclets are present, as the locals
237  // are accessed off the frame pointer in both the parent function and the
238  // funclets.
239  if (MF.hasEHFunclets())
240  return true;
241  // Retain behavior of always omitting the FP for leaf functions when possible.
242  if (MFI.hasCalls() && MF.getTarget().Options.DisableFramePointerElim(MF))
243  return true;
244  if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
245  MFI.hasStackMap() || MFI.hasPatchPoint() ||
246  RegInfo->needsStackRealignment(MF))
247  return true;
248  // With large callframes around we may need to use FP to access the scavenging
249  // emergency spillslot.
250  //
251  // Unfortunately some calls to hasFP() like machine verifier ->
252  // getReservedReg() -> hasFP in the middle of global isel are too early
253  // to know the max call frame size. Hopefully conservatively returning "true"
254  // in those cases is fine.
255  // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
256  if (!MFI.isMaxCallFrameSizeComputed() ||
258  return true;
259 
260  return false;
261 }
262 
263 /// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
264 /// not required, we reserve argument space for call sites in the function
265 /// immediately on entry to the current function. This eliminates the need for
266 /// add/sub sp brackets around call sites. Returns true if the call frame is
267 /// included as part of the stack frame.
268 bool
270  return !MF.getFrameInfo().hasVarSizedObjects();
271 }
272 
276  const AArch64InstrInfo *TII =
277  static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
278  DebugLoc DL = I->getDebugLoc();
279  unsigned Opc = I->getOpcode();
280  bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
281  uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
282 
283  if (!hasReservedCallFrame(MF)) {
284  unsigned Align = getStackAlignment();
285 
286  int64_t Amount = I->getOperand(0).getImm();
287  Amount = alignTo(Amount, Align);
288  if (!IsDestroy)
289  Amount = -Amount;
290 
291  // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
292  // doesn't have to pop anything), then the first operand will be zero too so
293  // this adjustment is a no-op.
294  if (CalleePopAmount == 0) {
295  // FIXME: in-function stack adjustment for calls is limited to 24-bits
296  // because there's no guaranteed temporary register available.
297  //
298  // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
299  // 1) For offset <= 12-bit, we use LSL #0
300  // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
301  // LSL #0, and the other uses LSL #12.
302  //
303  // Most call frames will be allocated at the start of a function so
304  // this is OK, but it is a limitation that needs dealing with.
305  assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
306  emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP, {Amount, MVT::i8},
307  TII);
308  }
309  } else if (CalleePopAmount != 0) {
310  // If the calling convention demands that the callee pops arguments from the
311  // stack, we want to add it back if we have a reserved call frame.
312  assert(CalleePopAmount < 0xffffff && "call frame too large");
313  emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
314  {-(int64_t)CalleePopAmount, MVT::i8}, TII);
315  }
316  return MBB.erase(I);
317 }
318 
320  // The function should be signed in the following situations:
321  // - sign-return-address=all
322  // - sign-return-address=non-leaf and the functions spills the LR
323 
324  const Function &F = MF.getFunction();
325  if (!F.hasFnAttribute("sign-return-address"))
326  return false;
327 
328  StringRef Scope = F.getFnAttribute("sign-return-address").getValueAsString();
329  if (Scope.equals("none"))
330  return false;
331 
332  if (Scope.equals("all"))
333  return true;
334 
335  assert(Scope.equals("non-leaf") && "Expected all, none or non-leaf");
336 
337  for (const auto &Info : MF.getFrameInfo().getCalleeSavedInfo())
338  if (Info.getReg() == AArch64::LR)
339  return true;
340 
341  return false;
342 }
343 
346  MachineFunction &MF = *MBB.getParent();
347  MachineFrameInfo &MFI = MF.getFrameInfo();
348  const TargetSubtargetInfo &STI = MF.getSubtarget();
349  const MCRegisterInfo *MRI = STI.getRegisterInfo();
350  const TargetInstrInfo *TII = STI.getInstrInfo();
351  DebugLoc DL = MBB.findDebugLoc(MBBI);
352 
353  // Add callee saved registers to move list.
354  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
355  if (CSI.empty())
356  return;
357 
358  for (const auto &Info : CSI) {
359  unsigned Reg = Info.getReg();
360  int64_t Offset =
361  MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea();
362  unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
363  unsigned CFIIndex = MF.addFrameInst(
364  MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
365  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
366  .addCFIIndex(CFIIndex)
368  }
369 }
370 
371 // Find a scratch register that we can use at the start of the prologue to
372 // re-align the stack pointer. We avoid using callee-save registers since they
373 // may appear to be free when this is called from canUseAsPrologue (during
374 // shrink wrapping), but then no longer be free when this is called from
375 // emitPrologue.
376 //
377 // FIXME: This is a bit conservative, since in the above case we could use one
378 // of the callee-save registers as a scratch temp to re-align the stack pointer,
379 // but we would then have to make sure that we were in fact saving at least one
380 // callee-save register in the prologue, which is additional complexity that
381 // doesn't seem worth the benefit.
383  MachineFunction *MF = MBB->getParent();
384 
385  // If MBB is an entry block, use X9 as the scratch register
386  if (&MF->front() == MBB)
387  return AArch64::X9;
388 
389  const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
390  const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
391  LivePhysRegs LiveRegs(TRI);
392  LiveRegs.addLiveIns(*MBB);
393 
394  // Mark callee saved registers as used so we will not choose them.
395  const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
396  for (unsigned i = 0; CSRegs[i]; ++i)
397  LiveRegs.addReg(CSRegs[i]);
398 
399  // Prefer X9 since it was historically used for the prologue scratch reg.
400  const MachineRegisterInfo &MRI = MF->getRegInfo();
401  if (LiveRegs.available(MRI, AArch64::X9))
402  return AArch64::X9;
403 
404  for (unsigned Reg : AArch64::GPR64RegClass) {
405  if (LiveRegs.available(MRI, Reg))
406  return Reg;
407  }
408  return AArch64::NoRegister;
409 }
410 
412  const MachineBasicBlock &MBB) const {
413  const MachineFunction *MF = MBB.getParent();
414  MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
415  const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
416  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
417 
418  // Don't need a scratch register if we're not going to re-align the stack.
419  if (!RegInfo->needsStackRealignment(*MF))
420  return true;
421  // Otherwise, we can use any block as long as it has a scratch register
422  // available.
423  return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
424 }
425 
427  unsigned StackSizeInBytes) {
428  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
429  if (!Subtarget.isTargetWindows())
430  return false;
431  const Function &F = MF.getFunction();
432  // TODO: When implementing stack protectors, take that into account
433  // for the probe threshold.
434  unsigned StackProbeSize = 4096;
435  if (F.hasFnAttribute("stack-probe-size"))
436  F.getFnAttribute("stack-probe-size")
438  .getAsInteger(0, StackProbeSize);
439  return (StackSizeInBytes >= StackProbeSize) &&
440  !F.hasFnAttribute("no-stack-arg-probe");
441 }
442 
443 bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
444  MachineFunction &MF, unsigned StackBumpBytes) const {
446  const MachineFrameInfo &MFI = MF.getFrameInfo();
447  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
448  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
449 
450  if (AFI->getLocalStackSize() == 0)
451  return false;
452 
453  // 512 is the maximum immediate for stp/ldp that will be used for
454  // callee-save save/restores
455  if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
456  return false;
457 
458  if (MFI.hasVarSizedObjects())
459  return false;
460 
461  if (RegInfo->needsStackRealignment(MF))
462  return false;
463 
464  // This isn't strictly necessary, but it simplifies things a bit since the
465  // current RedZone handling code assumes the SP is adjusted by the
466  // callee-save save/restore code.
467  if (canUseRedZone(MF))
468  return false;
469 
470  // When there is an SVE area on the stack, always allocate the
471  // callee-saves and spills/locals separately.
472  if (getSVEStackSize(MF))
473  return false;
474 
475  return true;
476 }
477 
478 // Given a load or a store instruction, generate an appropriate unwinding SEH
479 // code on Windows.
481  const TargetInstrInfo &TII,
483  unsigned Opc = MBBI->getOpcode();
484  MachineBasicBlock *MBB = MBBI->getParent();
485  MachineFunction &MF = *MBB->getParent();
486  DebugLoc DL = MBBI->getDebugLoc();
487  unsigned ImmIdx = MBBI->getNumOperands() - 1;
488  int Imm = MBBI->getOperand(ImmIdx).getImm();
490  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
491  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
492 
493  switch (Opc) {
494  default:
495  llvm_unreachable("No SEH Opcode for this instruction");
496  case AArch64::LDPDpost:
497  Imm = -Imm;
499  case AArch64::STPDpre: {
500  unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
501  unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
502  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
503  .addImm(Reg0)
504  .addImm(Reg1)
505  .addImm(Imm * 8)
506  .setMIFlag(Flag);
507  break;
508  }
509  case AArch64::LDPXpost:
510  Imm = -Imm;
512  case AArch64::STPXpre: {
513  Register Reg0 = MBBI->getOperand(1).getReg();
514  Register Reg1 = MBBI->getOperand(2).getReg();
515  if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
516  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
517  .addImm(Imm * 8)
518  .setMIFlag(Flag);
519  else
520  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
521  .addImm(RegInfo->getSEHRegNum(Reg0))
522  .addImm(RegInfo->getSEHRegNum(Reg1))
523  .addImm(Imm * 8)
524  .setMIFlag(Flag);
525  break;
526  }
527  case AArch64::LDRDpost:
528  Imm = -Imm;
530  case AArch64::STRDpre: {
531  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
532  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
533  .addImm(Reg)
534  .addImm(Imm)
535  .setMIFlag(Flag);
536  break;
537  }
538  case AArch64::LDRXpost:
539  Imm = -Imm;
541  case AArch64::STRXpre: {
542  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
543  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
544  .addImm(Reg)
545  .addImm(Imm)
546  .setMIFlag(Flag);
547  break;
548  }
549  case AArch64::STPDi:
550  case AArch64::LDPDi: {
551  unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
552  unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
553  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
554  .addImm(Reg0)
555  .addImm(Reg1)
556  .addImm(Imm * 8)
557  .setMIFlag(Flag);
558  break;
559  }
560  case AArch64::STPXi:
561  case AArch64::LDPXi: {
562  Register Reg0 = MBBI->getOperand(0).getReg();
563  Register Reg1 = MBBI->getOperand(1).getReg();
564  if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
565  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
566  .addImm(Imm * 8)
567  .setMIFlag(Flag);
568  else
569  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
570  .addImm(RegInfo->getSEHRegNum(Reg0))
571  .addImm(RegInfo->getSEHRegNum(Reg1))
572  .addImm(Imm * 8)
573  .setMIFlag(Flag);
574  break;
575  }
576  case AArch64::STRXui:
577  case AArch64::LDRXui: {
578  int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
579  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
580  .addImm(Reg)
581  .addImm(Imm * 8)
582  .setMIFlag(Flag);
583  break;
584  }
585  case AArch64::STRDui:
586  case AArch64::LDRDui: {
587  unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
588  MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
589  .addImm(Reg)
590  .addImm(Imm * 8)
591  .setMIFlag(Flag);
592  break;
593  }
594  }
595  auto I = MBB->insertAfter(MBBI, MIB);
596  return I;
597 }
598 
599 // Fix up the SEH opcode associated with the save/restore instruction.
601  unsigned LocalStackSize) {
602  MachineOperand *ImmOpnd = nullptr;
603  unsigned ImmIdx = MBBI->getNumOperands() - 1;
604  switch (MBBI->getOpcode()) {
605  default:
606  llvm_unreachable("Fix the offset in the SEH instruction");
607  case AArch64::SEH_SaveFPLR:
608  case AArch64::SEH_SaveRegP:
609  case AArch64::SEH_SaveReg:
610  case AArch64::SEH_SaveFRegP:
611  case AArch64::SEH_SaveFReg:
612  ImmOpnd = &MBBI->getOperand(ImmIdx);
613  break;
614  }
615  if (ImmOpnd)
616  ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
617 }
618 
619 // Convert callee-save register save/restore instruction to do stack pointer
620 // decrement/increment to allocate/deallocate the callee-save stack area by
621 // converting store/load to use pre/post increment version.
624  const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
625  bool NeedsWinCFI, bool *HasWinCFI, bool InProlog = true) {
626  // Ignore instructions that do not operate on SP, i.e. shadow call stack
627  // instructions and associated CFI instruction.
628  while (MBBI->getOpcode() == AArch64::STRXpost ||
629  MBBI->getOpcode() == AArch64::LDRXpre ||
630  MBBI->getOpcode() == AArch64::CFI_INSTRUCTION) {
631  if (MBBI->getOpcode() != AArch64::CFI_INSTRUCTION)
632  assert(MBBI->getOperand(0).getReg() != AArch64::SP);
633  ++MBBI;
634  }
635  unsigned NewOpc;
636  int Scale = 1;
637  switch (MBBI->getOpcode()) {
638  default:
639  llvm_unreachable("Unexpected callee-save save/restore opcode!");
640  case AArch64::STPXi:
641  NewOpc = AArch64::STPXpre;
642  Scale = 8;
643  break;
644  case AArch64::STPDi:
645  NewOpc = AArch64::STPDpre;
646  Scale = 8;
647  break;
648  case AArch64::STPQi:
649  NewOpc = AArch64::STPQpre;
650  Scale = 16;
651  break;
652  case AArch64::STRXui:
653  NewOpc = AArch64::STRXpre;
654  break;
655  case AArch64::STRDui:
656  NewOpc = AArch64::STRDpre;
657  break;
658  case AArch64::STRQui:
659  NewOpc = AArch64::STRQpre;
660  break;
661  case AArch64::LDPXi:
662  NewOpc = AArch64::LDPXpost;
663  Scale = 8;
664  break;
665  case AArch64::LDPDi:
666  NewOpc = AArch64::LDPDpost;
667  Scale = 8;
668  break;
669  case AArch64::LDPQi:
670  NewOpc = AArch64::LDPQpost;
671  Scale = 16;
672  break;
673  case AArch64::LDRXui:
674  NewOpc = AArch64::LDRXpost;
675  break;
676  case AArch64::LDRDui:
677  NewOpc = AArch64::LDRDpost;
678  break;
679  case AArch64::LDRQui:
680  NewOpc = AArch64::LDRQpost;
681  break;
682  }
683  // Get rid of the SEH code associated with the old instruction.
684  if (NeedsWinCFI) {
685  auto SEH = std::next(MBBI);
687  SEH->eraseFromParent();
688  }
689 
690  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
691  MIB.addReg(AArch64::SP, RegState::Define);
692 
693  // Copy all operands other than the immediate offset.
694  unsigned OpndIdx = 0;
695  for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
696  ++OpndIdx)
697  MIB.add(MBBI->getOperand(OpndIdx));
698 
699  assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
700  "Unexpected immediate offset in first/last callee-save save/restore "
701  "instruction!");
702  assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
703  "Unexpected base register in callee-save save/restore instruction!");
704  assert(CSStackSizeInc % Scale == 0);
705  MIB.addImm(CSStackSizeInc / Scale);
706 
707  MIB.setMIFlags(MBBI->getFlags());
708  MIB.setMemRefs(MBBI->memoperands());
709 
710  // Generate a new SEH code that corresponds to the new instruction.
711  if (NeedsWinCFI) {
712  *HasWinCFI = true;
713  InsertSEH(*MIB, *TII,
715  }
716 
717  return std::prev(MBB.erase(MBBI));
718 }
719 
720 // Fixup callee-save register save/restore instructions to take into account
721 // combined SP bump by adding the local stack size to the stack offsets.
723  unsigned LocalStackSize,
724  bool NeedsWinCFI,
725  bool *HasWinCFI) {
727  return;
728 
729  unsigned Opc = MI.getOpcode();
730 
731  // Ignore instructions that do not operate on SP, i.e. shadow call stack
732  // instructions and associated CFI instruction.
733  if (Opc == AArch64::STRXpost || Opc == AArch64::LDRXpre ||
734  Opc == AArch64::CFI_INSTRUCTION) {
735  if (Opc != AArch64::CFI_INSTRUCTION)
736  assert(MI.getOperand(0).getReg() != AArch64::SP);
737  return;
738  }
739 
740  unsigned Scale;
741  switch (Opc) {
742  case AArch64::STPXi:
743  case AArch64::STRXui:
744  case AArch64::STPDi:
745  case AArch64::STRDui:
746  case AArch64::LDPXi:
747  case AArch64::LDRXui:
748  case AArch64::LDPDi:
749  case AArch64::LDRDui:
750  Scale = 8;
751  break;
752  case AArch64::STPQi:
753  case AArch64::STRQui:
754  case AArch64::LDPQi:
755  case AArch64::LDRQui:
756  Scale = 16;
757  break;
758  default:
759  llvm_unreachable("Unexpected callee-save save/restore opcode!");
760  }
761 
762  unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
763  assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
764  "Unexpected base register in callee-save save/restore instruction!");
765  // Last operand is immediate offset that needs fixing.
766  MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
767  // All generated opcodes have scaled offsets.
768  assert(LocalStackSize % Scale == 0);
769  OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
770 
771  if (NeedsWinCFI) {
772  *HasWinCFI = true;
773  auto MBBI = std::next(MachineBasicBlock::iterator(MI));
774  assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
776  "Expecting a SEH instruction");
777  fixupSEHOpcode(MBBI, LocalStackSize);
778  }
779 }
780 
782  MachineBasicBlock::iterator FirstSPPopI,
783  MachineBasicBlock::iterator LastPopI) {
784  // Sometimes (when we restore in the same order as we save), we can end up
785  // with code like this:
786  //
787  // ldp x26, x25, [sp]
788  // ldp x24, x23, [sp, #16]
789  // ldp x22, x21, [sp, #32]
790  // ldp x20, x19, [sp, #48]
791  // add sp, sp, #64
792  //
793  // In this case, it is always better to put the first ldp at the end, so
794  // that the load-store optimizer can run and merge the ldp and the add into
795  // a post-index ldp.
796  // If we managed to grab the first pop instruction, move it to the end.
798  MBB.splice(FirstSPPopI, &MBB, LastPopI);
799  // We should end up with something like this now:
800  //
801  // ldp x24, x23, [sp, #16]
802  // ldp x22, x21, [sp, #32]
803  // ldp x20, x19, [sp, #48]
804  // ldp x26, x25, [sp]
805  // add sp, sp, #64
806  //
807  // and the load-store optimizer can merge the last two instructions into:
808  //
809  // ldp x26, x25, [sp], #64
810  //
811 }
812 
814  const Function &F = MF.getFunction();
815  if (!F.hasFnAttribute("sign-return-address-key"))
816  return true;
817 
818  const StringRef Key =
819  F.getFnAttribute("sign-return-address-key").getValueAsString();
820  assert(Key.equals_lower("a_key") || Key.equals_lower("b_key"));
821  return Key.equals_lower("a_key");
822 }
823 
824 static bool needsWinCFI(const MachineFunction &MF) {
825  const Function &F = MF.getFunction();
826  return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
828 }
829 
830 static bool isTargetDarwin(const MachineFunction &MF) {
832 }
833 
835  MachineBasicBlock &MBB) const {
836  MachineBasicBlock::iterator MBBI = MBB.begin();
837  const MachineFrameInfo &MFI = MF.getFrameInfo();
838  const Function &F = MF.getFunction();
839  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
840  const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
841  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
842  MachineModuleInfo &MMI = MF.getMMI();
844  bool needsFrameMoves = (MMI.hasDebugInfo() || F.needsUnwindTableEntry()) &&
846  bool HasFP = hasFP(MF);
847  bool NeedsWinCFI = needsWinCFI(MF);
848  bool HasWinCFI = false;
849  auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); });
850 
851  bool IsFunclet = MBB.isEHFuncletEntry();
852 
853  // At this point, we're going to decide whether or not the function uses a
854  // redzone. In most cases, the function doesn't have a redzone so let's
855  // assume that's false and set it to true in the case that there's a redzone.
856  AFI->setHasRedZone(false);
857 
858  // Debug location must be unknown since the first debug location is used
859  // to determine the end of the prologue.
860  DebugLoc DL;
861 
862  if (ShouldSignReturnAddress(MF)) {
863  if (ShouldSignWithAKey(MF))
864  BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
865  .setMIFlag(MachineInstr::FrameSetup);
866  else {
867  BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
868  .setMIFlag(MachineInstr::FrameSetup);
869  BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIBSP))
870  .setMIFlag(MachineInstr::FrameSetup);
871  }
872 
873  unsigned CFIIndex =
875  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
876  .addCFIIndex(CFIIndex)
878  }
879 
880  // All calls are tail calls in GHC calling conv, and functions have no
881  // prologue/epilogue.
883  return;
884 
885  // Set tagged base pointer to the bottom of the stack frame.
886  // Ideally it should match SP value after prologue.
888 
889  const StackOffset &SVEStackSize = getSVEStackSize(MF);
890 
891  // getStackSize() includes all the locals in its size calculation. We don't
892  // include these locals when computing the stack size of a funclet, as they
893  // are allocated in the parent's stack frame and accessed via the frame
894  // pointer from the funclet. We only save the callee saved registers in the
895  // funclet, which are really the callee saved registers of the parent
896  // function, including the funclet.
897  int NumBytes = IsFunclet ? (int)getWinEHFuncletFrameSize(MF)
898  : (int)MFI.getStackSize();
899  if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
900  assert(!HasFP && "unexpected function without stack frame but with FP");
901  assert(!SVEStackSize &&
902  "unexpected function without stack frame but with SVE objects");
903  // All of the stack allocation is for locals.
904  AFI->setLocalStackSize(NumBytes);
905  if (!NumBytes)
906  return;
907  // REDZONE: If the stack size is less than 128 bytes, we don't need
908  // to actually allocate.
909  if (canUseRedZone(MF)) {
910  AFI->setHasRedZone(true);
911  ++NumRedZoneFunctions;
912  } else {
913  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
914  {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup,
915  false, NeedsWinCFI, &HasWinCFI);
916  if (!NeedsWinCFI) {
917  // Label used to tie together the PROLOG_LABEL and the MachineMoves.
918  MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
919  // Encode the stack size of the leaf function.
920  unsigned CFIIndex = MF.addFrameInst(
921  MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes));
922  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
923  .addCFIIndex(CFIIndex)
924  .setMIFlags(MachineInstr::FrameSetup);
925  }
926  }
927 
928  if (NeedsWinCFI) {
929  HasWinCFI = true;
930  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
931  .setMIFlag(MachineInstr::FrameSetup);
932  }
933 
934  return;
935  }
936 
937  bool IsWin64 =
939  // Var args are accounted for in the containing function, so don't
940  // include them for funclets.
941  unsigned FixedObject = (IsWin64 && !IsFunclet) ?
942  alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
943 
944  auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
945  // All of the remaining stack allocations are for locals.
946  AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
947  bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
948  if (CombineSPBump) {
949  assert(!SVEStackSize && "Cannot combine SP bump with SVE");
950  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
951  {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup, false,
952  NeedsWinCFI, &HasWinCFI);
953  NumBytes = 0;
954  } else if (PrologueSaveSize != 0) {
956  MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI);
957  NumBytes -= PrologueSaveSize;
958  }
959  assert(NumBytes >= 0 && "Negative stack allocation size!?");
960 
961  // Move past the saves of the callee-saved registers, fixing up the offsets
962  // and pre-inc if we decided to combine the callee-save and local stack
963  // pointer bump above.
964  MachineBasicBlock::iterator End = MBB.end();
965  while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup)) {
966  if (CombineSPBump)
968  NeedsWinCFI, &HasWinCFI);
969  ++MBBI;
970  }
971 
972  // The code below is not applicable to funclets. We have emitted all the SEH
973  // opcodes that we needed to emit. The FP and BP belong to the containing
974  // function.
975  if (IsFunclet) {
976  if (NeedsWinCFI) {
977  HasWinCFI = true;
978  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
979  .setMIFlag(MachineInstr::FrameSetup);
980  }
981 
982  // SEH funclets are passed the frame pointer in X1. If the parent
983  // function uses the base register, then the base register is used
984  // directly, and is not retrieved from X1.
985  if (F.hasPersonalityFn()) {
987  if (isAsynchronousEHPersonality(Per)) {
988  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
989  .addReg(AArch64::X1).setMIFlag(MachineInstr::FrameSetup);
990  MBB.addLiveIn(AArch64::X1);
991  }
992  }
993 
994  return;
995  }
996 
997  if (HasFP) {
998  // Only set up FP if we actually need to.
999  int FPOffset = isTargetDarwin(MF) ? (AFI->getCalleeSavedStackSize() - 16) : 0;
1000 
1001  if (CombineSPBump)
1002  FPOffset += AFI->getLocalStackSize();
1003 
1004  // Issue sub fp, sp, FPOffset or
1005  // mov fp,sp when FPOffset is zero.
1006  // Note: All stores of callee-saved registers are marked as "FrameSetup".
1007  // This code marks the instruction(s) that set the FP also.
1008  emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP,
1009  {FPOffset, MVT::i8}, TII, MachineInstr::FrameSetup, false,
1010  NeedsWinCFI, &HasWinCFI);
1011  }
1012 
1013  if (windowsRequiresStackProbe(MF, NumBytes)) {
1014  uint32_t NumWords = NumBytes >> 4;
1015  if (NeedsWinCFI) {
1016  HasWinCFI = true;
1017  // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
1018  // exceed this amount. We need to move at most 2^24 - 1 into x15.
1019  // This is at most two instructions, MOVZ follwed by MOVK.
1020  // TODO: Fix to use multiple stack alloc unwind codes for stacks
1021  // exceeding 256MB in size.
1022  if (NumBytes >= (1 << 28))
1023  report_fatal_error("Stack size cannot exceed 256MB for stack "
1024  "unwinding purposes");
1025 
1026  uint32_t LowNumWords = NumWords & 0xFFFF;
1027  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
1028  .addImm(LowNumWords)
1031  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1032  .setMIFlag(MachineInstr::FrameSetup);
1033  if ((NumWords & 0xFFFF0000) != 0) {
1034  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
1035  .addReg(AArch64::X15)
1036  .addImm((NumWords & 0xFFFF0000) >> 16) // High half
1039  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1040  .setMIFlag(MachineInstr::FrameSetup);
1041  }
1042  } else {
1043  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
1044  .addImm(NumWords)
1046  }
1047 
1048  switch (MF.getTarget().getCodeModel()) {
1049  case CodeModel::Tiny:
1050  case CodeModel::Small:
1051  case CodeModel::Medium:
1052  case CodeModel::Kernel:
1053  BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
1054  .addExternalSymbol("__chkstk")
1055  .addReg(AArch64::X15, RegState::Implicit)
1060  if (NeedsWinCFI) {
1061  HasWinCFI = true;
1062  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1063  .setMIFlag(MachineInstr::FrameSetup);
1064  }
1065  break;
1066  case CodeModel::Large:
1067  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
1068  .addReg(AArch64::X16, RegState::Define)
1069  .addExternalSymbol("__chkstk")
1070  .addExternalSymbol("__chkstk")
1072  if (NeedsWinCFI) {
1073  HasWinCFI = true;
1074  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1075  .setMIFlag(MachineInstr::FrameSetup);
1076  }
1077 
1078  BuildMI(MBB, MBBI, DL, TII->get(AArch64::BLR))
1079  .addReg(AArch64::X16, RegState::Kill)
1080  .addReg(AArch64::X15, RegState::Implicit | RegState::Define)
1085  if (NeedsWinCFI) {
1086  HasWinCFI = true;
1087  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1088  .setMIFlag(MachineInstr::FrameSetup);
1089  }
1090  break;
1091  }
1092 
1093  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
1094  .addReg(AArch64::SP, RegState::Kill)
1095  .addReg(AArch64::X15, RegState::Kill)
1098  if (NeedsWinCFI) {
1099  HasWinCFI = true;
1100  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1101  .addImm(NumBytes)
1103  }
1104  NumBytes = 0;
1105  }
1106 
1107  emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, -SVEStackSize, TII,
1109 
1110  // Allocate space for the rest of the frame.
1111  if (NumBytes) {
1112  const bool NeedsRealignment = RegInfo->needsStackRealignment(MF);
1113  unsigned scratchSPReg = AArch64::SP;
1114 
1115  if (NeedsRealignment) {
1116  scratchSPReg = findScratchNonCalleeSaveRegister(&MBB);
1117  assert(scratchSPReg != AArch64::NoRegister);
1118  }
1119 
1120  // If we're a leaf function, try using the red zone.
1121  if (!canUseRedZone(MF))
1122  // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
1123  // the correct value here, as NumBytes also includes padding bytes,
1124  // which shouldn't be counted here.
1125  emitFrameOffset(MBB, MBBI, DL, scratchSPReg, AArch64::SP,
1126  {-NumBytes, MVT::i8}, TII, MachineInstr::FrameSetup,
1127  false, NeedsWinCFI, &HasWinCFI);
1128 
1129  if (NeedsRealignment) {
1130  const unsigned Alignment = MFI.getMaxAlignment();
1131  const unsigned NrBitsToZero = countTrailingZeros(Alignment);
1132  assert(NrBitsToZero > 1);
1133  assert(scratchSPReg != AArch64::SP);
1134 
1135  // SUB X9, SP, NumBytes
1136  // -- X9 is temporary register, so shouldn't contain any live data here,
1137  // -- free to use. This is already produced by emitFrameOffset above.
1138  // AND SP, X9, 0b11111...0000
1139  // The logical immediates have a non-trivial encoding. The following
1140  // formula computes the encoded immediate with all ones but
1141  // NrBitsToZero zero bits as least significant bits.
1142  uint32_t andMaskEncoded = (1 << 12) // = N
1143  | ((64 - NrBitsToZero) << 6) // immr
1144  | ((64 - NrBitsToZero - 1) << 0); // imms
1145 
1146  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
1147  .addReg(scratchSPReg, RegState::Kill)
1148  .addImm(andMaskEncoded);
1149  AFI->setStackRealigned(true);
1150  if (NeedsWinCFI) {
1151  HasWinCFI = true;
1152  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
1153  .addImm(NumBytes & andMaskEncoded)
1154  .setMIFlag(MachineInstr::FrameSetup);
1155  }
1156  }
1157  }
1158 
1159  // If we need a base pointer, set it up here. It's whatever the value of the
1160  // stack pointer is at this point. Any variable size objects will be allocated
1161  // after this, so we can still use the base pointer to reference locals.
1162  //
1163  // FIXME: Clarify FrameSetup flags here.
1164  // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
1165  // needed.
1166  if (RegInfo->hasBasePointer(MF)) {
1167  TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
1168  false);
1169  if (NeedsWinCFI) {
1170  HasWinCFI = true;
1171  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1172  .setMIFlag(MachineInstr::FrameSetup);
1173  }
1174  }
1175 
1176  // The very last FrameSetup instruction indicates the end of prologue. Emit a
1177  // SEH opcode indicating the prologue end.
1178  if (NeedsWinCFI && HasWinCFI) {
1179  BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1180  .setMIFlag(MachineInstr::FrameSetup);
1181  }
1182 
1183  if (needsFrameMoves) {
1184  const DataLayout &TD = MF.getDataLayout();
1185  const int StackGrowth = isTargetDarwin(MF)
1186  ? (2 * -TD.getPointerSize(0))
1187  : -AFI->getCalleeSavedStackSize();
1188  Register FramePtr = RegInfo->getFrameRegister(MF);
1189  // An example of the prologue:
1190  //
1191  // .globl __foo
1192  // .align 2
1193  // __foo:
1194  // Ltmp0:
1195  // .cfi_startproc
1196  // .cfi_personality 155, ___gxx_personality_v0
1197  // Leh_func_begin:
1198  // .cfi_lsda 16, Lexception33
1199  //
1200  // stp xa,bx, [sp, -#offset]!
1201  // ...
1202  // stp x28, x27, [sp, #offset-32]
1203  // stp fp, lr, [sp, #offset-16]
1204  // add fp, sp, #offset - 16
1205  // sub sp, sp, #1360
1206  //
1207  // The Stack:
1208  // +-------------------------------------------+
1209  // 10000 | ........ | ........ | ........ | ........ |
1210  // 10004 | ........ | ........ | ........ | ........ |
1211  // +-------------------------------------------+
1212  // 10008 | ........ | ........ | ........ | ........ |
1213  // 1000c | ........ | ........ | ........ | ........ |
1214  // +===========================================+
1215  // 10010 | X28 Register |
1216  // 10014 | X28 Register |
1217  // +-------------------------------------------+
1218  // 10018 | X27 Register |
1219  // 1001c | X27 Register |
1220  // +===========================================+
1221  // 10020 | Frame Pointer |
1222  // 10024 | Frame Pointer |
1223  // +-------------------------------------------+
1224  // 10028 | Link Register |
1225  // 1002c | Link Register |
1226  // +===========================================+
1227  // 10030 | ........ | ........ | ........ | ........ |
1228  // 10034 | ........ | ........ | ........ | ........ |
1229  // +-------------------------------------------+
1230  // 10038 | ........ | ........ | ........ | ........ |
1231  // 1003c | ........ | ........ | ........ | ........ |
1232  // +-------------------------------------------+
1233  //
1234  // [sp] = 10030 :: >>initial value<<
1235  // sp = 10020 :: stp fp, lr, [sp, #-16]!
1236  // fp = sp == 10020 :: mov fp, sp
1237  // [sp] == 10020 :: stp x28, x27, [sp, #-16]!
1238  // sp == 10010 :: >>final value<<
1239  //
1240  // The frame pointer (w29) points to address 10020. If we use an offset of
1241  // '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
1242  // for w27, and -32 for w28:
1243  //
1244  // Ltmp1:
1245  // .cfi_def_cfa w29, 16
1246  // Ltmp2:
1247  // .cfi_offset w30, -8
1248  // Ltmp3:
1249  // .cfi_offset w29, -16
1250  // Ltmp4:
1251  // .cfi_offset w27, -24
1252  // Ltmp5:
1253  // .cfi_offset w28, -32
1254 
1255  if (HasFP) {
1256  // Define the current CFA rule to use the provided FP.
1257  unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true);
1258  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
1259  nullptr, Reg, StackGrowth - FixedObject));
1260  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1261  .addCFIIndex(CFIIndex)
1263  } else {
1264  // Encode the stack size of the leaf function.
1265  unsigned CFIIndex = MF.addFrameInst(
1267  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1268  .addCFIIndex(CFIIndex)
1270  }
1271 
1272  // Now emit the moves for whatever callee saved regs we have (including FP,
1273  // LR if those are saved).
1274  emitCalleeSavedFrameMoves(MBB, MBBI);
1275  }
1276 }
1277 
1279  MachineBasicBlock &MBB) {
1280  if (!ShouldSignReturnAddress(MF))
1281  return;
1282  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1283  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1284 
1286  DebugLoc DL;
1287  if (MBBI != MBB.end())
1288  DL = MBBI->getDebugLoc();
1289 
1290  // The AUTIASP instruction assembles to a hint instruction before v8.3a so
1291  // this instruction can safely used for any v8a architecture.
1292  // From v8.3a onwards there are optimised authenticate LR and return
1293  // instructions, namely RETA{A,B}, that can be used instead.
1294  if (Subtarget.hasV8_3aOps() && MBBI != MBB.end() &&
1295  MBBI->getOpcode() == AArch64::RET_ReallyLR) {
1296  BuildMI(MBB, MBBI, DL,
1297  TII->get(ShouldSignWithAKey(MF) ? AArch64::RETAA : AArch64::RETAB))
1298  .copyImplicitOps(*MBBI);
1299  MBB.erase(MBBI);
1300  } else {
1301  BuildMI(
1302  MBB, MBBI, DL,
1303  TII->get(ShouldSignWithAKey(MF) ? AArch64::AUTIASP : AArch64::AUTIBSP))
1304  .setMIFlag(MachineInstr::FrameDestroy);
1305  }
1306 }
1307 
1308 static bool isFuncletReturnInstr(const MachineInstr &MI) {
1309  switch (MI.getOpcode()) {
1310  default:
1311  return false;
1312  case AArch64::CATCHRET:
1313  case AArch64::CLEANUPRET:
1314  return true;
1315  }
1316 }
1317 
1319  MachineBasicBlock &MBB) const {
1321  MachineFrameInfo &MFI = MF.getFrameInfo();
1322  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1323  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1324  DebugLoc DL;
1325  bool IsTailCallReturn = false;
1326  bool NeedsWinCFI = needsWinCFI(MF);
1327  bool HasWinCFI = false;
1328  bool IsFunclet = false;
1329  auto WinCFI = make_scope_exit([&]() {
1330  if (!MF.hasWinCFI())
1331  MF.setHasWinCFI(HasWinCFI);
1332  });
1333 
1334  if (MBB.end() != MBBI) {
1335  DL = MBBI->getDebugLoc();
1336  unsigned RetOpcode = MBBI->getOpcode();
1337  IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi ||
1338  RetOpcode == AArch64::TCRETURNri ||
1339  RetOpcode == AArch64::TCRETURNriBTI;
1340  IsFunclet = isFuncletReturnInstr(*MBBI);
1341  }
1342 
1343  int NumBytes = IsFunclet ? (int)getWinEHFuncletFrameSize(MF)
1344  : MFI.getStackSize();
1346 
1347  // All calls are tail calls in GHC calling conv, and functions have no
1348  // prologue/epilogue.
1350  return;
1351 
1352  // Initial and residual are named for consistency with the prologue. Note that
1353  // in the epilogue, the residual adjustment is executed first.
1354  uint64_t ArgumentPopSize = 0;
1355  if (IsTailCallReturn) {
1356  MachineOperand &StackAdjust = MBBI->getOperand(1);
1357 
1358  // For a tail-call in a callee-pops-arguments environment, some or all of
1359  // the stack may actually be in use for the call's arguments, this is
1360  // calculated during LowerCall and consumed here...
1361  ArgumentPopSize = StackAdjust.getImm();
1362  } else {
1363  // ... otherwise the amount to pop is *all* of the argument space,
1364  // conveniently stored in the MachineFunctionInfo by
1365  // LowerFormalArguments. This will, of course, be zero for the C calling
1366  // convention.
1367  ArgumentPopSize = AFI->getArgumentStackToRestore();
1368  }
1369 
1370  // The stack frame should be like below,
1371  //
1372  // ---------------------- ---
1373  // | | |
1374  // | BytesInStackArgArea| CalleeArgStackSize
1375  // | (NumReusableBytes) | (of tail call)
1376  // | | ---
1377  // | | |
1378  // ---------------------| --- |
1379  // | | | |
1380  // | CalleeSavedReg | | |
1381  // | (CalleeSavedStackSize)| | |
1382  // | | | |
1383  // ---------------------| | NumBytes
1384  // | | StackSize (StackAdjustUp)
1385  // | LocalStackSize | | |
1386  // | (covering callee | | |
1387  // | args) | | |
1388  // | | | |
1389  // ---------------------- --- ---
1390  //
1391  // So NumBytes = StackSize + BytesInStackArgArea - CalleeArgStackSize
1392  // = StackSize + ArgumentPopSize
1393  //
1394  // AArch64TargetLowering::LowerCall figures out ArgumentPopSize and keeps
1395  // it as the 2nd argument of AArch64ISD::TC_RETURN.
1396 
1397  auto Cleanup = make_scope_exit([&] { InsertReturnAddressAuth(MF, MBB); });
1398 
1399  bool IsWin64 =
1400  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1401  // Var args are accounted for in the containing function, so don't
1402  // include them for funclets.
1403  unsigned FixedObject =
1404  (IsWin64 && !IsFunclet) ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
1405 
1406  uint64_t AfterCSRPopSize = ArgumentPopSize;
1407  auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1408  // We cannot rely on the local stack size set in emitPrologue if the function
1409  // has funclets, as funclets have different local stack size requirements, and
1410  // the current value set in emitPrologue may be that of the containing
1411  // function.
1412  if (MF.hasEHFunclets())
1413  AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1414  bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1415  // Assume we can't combine the last pop with the sp restore.
1416 
1417  if (!CombineSPBump && PrologueSaveSize != 0) {
1418  MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
1420  Pop = std::prev(Pop);
1421  // Converting the last ldp to a post-index ldp is valid only if the last
1422  // ldp's offset is 0.
1423  const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
1424  // If the offset is 0, convert it to a post-index ldp.
1425  if (OffsetOp.getImm() == 0)
1427  MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, false);
1428  else {
1429  // If not, make sure to emit an add after the last ldp.
1430  // We're doing this by transfering the size to be restored from the
1431  // adjustment *before* the CSR pops to the adjustment *after* the CSR
1432  // pops.
1433  AfterCSRPopSize += PrologueSaveSize;
1434  }
1435  }
1436 
1437  // Move past the restores of the callee-saved registers.
1438  // If we plan on combining the sp bump of the local stack size and the callee
1439  // save stack size, we might need to adjust the CSR save and restore offsets.
1441  MachineBasicBlock::iterator Begin = MBB.begin();
1442  while (LastPopI != Begin) {
1443  --LastPopI;
1444  if (!LastPopI->getFlag(MachineInstr::FrameDestroy)) {
1445  ++LastPopI;
1446  break;
1447  } else if (CombineSPBump)
1449  NeedsWinCFI, &HasWinCFI);
1450  }
1451 
1452  if (NeedsWinCFI) {
1453  HasWinCFI = true;
1454  BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
1455  .setMIFlag(MachineInstr::FrameDestroy);
1456  }
1457 
1458  const StackOffset &SVEStackSize = getSVEStackSize(MF);
1459 
1460  // If there is a single SP update, insert it before the ret and we're done.
1461  if (CombineSPBump) {
1462  assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1463  emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
1464  {NumBytes + (int64_t)AfterCSRPopSize, MVT::i8}, TII,
1465  MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1466  if (NeedsWinCFI && HasWinCFI)
1467  BuildMI(MBB, MBB.getFirstTerminator(), DL,
1468  TII->get(AArch64::SEH_EpilogEnd))
1469  .setMIFlag(MachineInstr::FrameDestroy);
1470  return;
1471  }
1472 
1473  NumBytes -= PrologueSaveSize;
1474  assert(NumBytes >= 0 && "Negative stack allocation size!?");
1475 
1476  // Deallocate the SVE area.
1477  if (SVEStackSize)
1478  if (!AFI->isStackRealigned())
1479  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP, SVEStackSize,
1481 
1482  if (!hasFP(MF)) {
1483  bool RedZone = canUseRedZone(MF);
1484  // If this was a redzone leaf function, we don't need to restore the
1485  // stack pointer (but we may need to pop stack args for fastcc).
1486  if (RedZone && AfterCSRPopSize == 0)
1487  return;
1488 
1489  bool NoCalleeSaveRestore = PrologueSaveSize == 0;
1490  int StackRestoreBytes = RedZone ? 0 : NumBytes;
1491  if (NoCalleeSaveRestore)
1492  StackRestoreBytes += AfterCSRPopSize;
1493 
1494  // If we were able to combine the local stack pop with the argument pop,
1495  // then we're done.
1496  bool Done = NoCalleeSaveRestore || AfterCSRPopSize == 0;
1497 
1498  // If we're done after this, make sure to help the load store optimizer.
1499  if (Done)
1500  adaptForLdStOpt(MBB, MBB.getFirstTerminator(), LastPopI);
1501 
1502  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1503  {StackRestoreBytes, MVT::i8}, TII,
1504  MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1505  if (Done) {
1506  if (NeedsWinCFI) {
1507  HasWinCFI = true;
1508  BuildMI(MBB, MBB.getFirstTerminator(), DL,
1509  TII->get(AArch64::SEH_EpilogEnd))
1510  .setMIFlag(MachineInstr::FrameDestroy);
1511  }
1512  return;
1513  }
1514 
1515  NumBytes = 0;
1516  }
1517 
1518  // Restore the original stack pointer.
1519  // FIXME: Rather than doing the math here, we should instead just use
1520  // non-post-indexed loads for the restores if we aren't actually going to
1521  // be able to save any instructions.
1522  if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
1523  int64_t OffsetToFrameRecord =
1524  isTargetDarwin(MF) ? (-(int64_t)AFI->getCalleeSavedStackSize() + 16) : 0;
1525  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
1526  {OffsetToFrameRecord, MVT::i8},
1527  TII, MachineInstr::FrameDestroy, false, NeedsWinCFI);
1528  } else if (NumBytes)
1529  emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
1530  {NumBytes, MVT::i8}, TII, MachineInstr::FrameDestroy, false,
1531  NeedsWinCFI);
1532 
1533  // This must be placed after the callee-save restore code because that code
1534  // assumes the SP is at the same location as it was after the callee-save save
1535  // code in the prologue.
1536  if (AfterCSRPopSize) {
1537  // Find an insertion point for the first ldp so that it goes before the
1538  // shadow call stack epilog instruction. This ensures that the restore of
1539  // lr from x18 is placed after the restore from sp.
1540  auto FirstSPPopI = MBB.getFirstTerminator();
1541  while (FirstSPPopI != Begin) {
1542  auto Prev = std::prev(FirstSPPopI);
1543  if (Prev->getOpcode() != AArch64::LDRXpre ||
1544  Prev->getOperand(0).getReg() == AArch64::SP)
1545  break;
1546  FirstSPPopI = Prev;
1547  }
1548 
1549  adaptForLdStOpt(MBB, FirstSPPopI, LastPopI);
1550 
1551  emitFrameOffset(MBB, FirstSPPopI, DL, AArch64::SP, AArch64::SP,
1552  {(int64_t)AfterCSRPopSize, MVT::i8}, TII,
1553  MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
1554  }
1555  if (NeedsWinCFI && HasWinCFI)
1556  BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd))
1557  .setMIFlag(MachineInstr::FrameDestroy);
1558 
1559  MF.setHasWinCFI(HasWinCFI);
1560 }
1561 
1562 /// getFrameIndexReference - Provide a base+offset reference to an FI slot for
1563 /// debug info. It's the same as what we use for resolving the code-gen
1564 /// references for now. FIXME: This can go wrong when references are
1565 /// SP-relative and simple call frames aren't used.
1567  int FI,
1568  unsigned &FrameReg) const {
1570  MF, FI, FrameReg,
1571  /*PreferFP=*/
1572  MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress),
1573  /*ForSimm=*/false)
1574  .getBytes();
1575 }
1576 
1578  const MachineFunction &MF, int FI) const {
1579  return getSEHFrameIndexOffset(MF, FI);
1580 }
1581 
1582 static StackOffset getFPOffset(const MachineFunction &MF, int ObjectOffset) {
1583  const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1584  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1585  bool IsWin64 =
1586  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
1587  unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
1588  unsigned FPAdjust = isTargetDarwin(MF) ? 16 : AFI->getCalleeSavedStackSize();
1589  return {ObjectOffset + FixedObject + FPAdjust, MVT::i8};
1590 }
1591 
1592 static StackOffset getStackOffset(const MachineFunction &MF, int ObjectOffset) {
1593  const auto &MFI = MF.getFrameInfo();
1594  return {ObjectOffset + (int)MFI.getStackSize(), MVT::i8};
1595 }
1596 
1598  int FI) const {
1599  const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
1600  MF.getSubtarget().getRegisterInfo());
1601  int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI);
1602  return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
1603  ? getFPOffset(MF, ObjectOffset).getBytes()
1604  : getStackOffset(MF, ObjectOffset).getBytes();
1605 }
1606 
1608  const MachineFunction &MF, int FI, unsigned &FrameReg, bool PreferFP,
1609  bool ForSimm) const {
1610  const auto &MFI = MF.getFrameInfo();
1611  int ObjectOffset = MFI.getObjectOffset(FI);
1612  bool isFixed = MFI.isFixedObjectIndex(FI);
1613  bool isSVE = MFI.getStackID(FI) == TargetStackID::SVEVector;
1614  return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
1615  PreferFP, ForSimm);
1616 }
1617 
1619  const MachineFunction &MF, int ObjectOffset, bool isFixed, bool isSVE,
1620  unsigned &FrameReg, bool PreferFP, bool ForSimm) const {
1621  const auto &MFI = MF.getFrameInfo();
1622  const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
1623  MF.getSubtarget().getRegisterInfo());
1624  const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
1625  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1626 
1627  int FPOffset = getFPOffset(MF, ObjectOffset).getBytes();
1628  int Offset = getStackOffset(MF, ObjectOffset).getBytes();
1629  bool isCSR =
1630  !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize());
1631 
1632  const StackOffset &SVEStackSize = getSVEStackSize(MF);
1633 
1634  // Use frame pointer to reference fixed objects. Use it for locals if
1635  // there are VLAs or a dynamically realigned SP (and thus the SP isn't
1636  // reliable as a base). Make sure useFPForScavengingIndex() does the
1637  // right thing for the emergency spill slot.
1638  bool UseFP = false;
1639  if (AFI->hasStackFrame() && !isSVE) {
1640  // We shouldn't prefer using the FP when there is an SVE area
1641  // in between the FP and the non-SVE locals/spills.
1642  PreferFP &= !SVEStackSize;
1643 
1644  // Note: Keeping the following as multiple 'if' statements rather than
1645  // merging to a single expression for readability.
1646  //
1647  // Argument access should always use the FP.
1648  if (isFixed) {
1649  UseFP = hasFP(MF);
1650  } else if (isCSR && RegInfo->needsStackRealignment(MF)) {
1651  // References to the CSR area must use FP if we're re-aligning the stack
1652  // since the dynamically-sized alignment padding is between the SP/BP and
1653  // the CSR area.
1654  assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1655  UseFP = true;
1656  } else if (hasFP(MF) && !RegInfo->needsStackRealignment(MF)) {
1657  // If the FPOffset is negative and we're producing a signed immediate, we
1658  // have to keep in mind that the available offset range for negative
1659  // offsets is smaller than for positive ones. If an offset is available
1660  // via the FP and the SP, use whichever is closest.
1661  bool FPOffsetFits = !ForSimm || FPOffset >= -256;
1662  PreferFP |= Offset > -FPOffset;
1663 
1664  if (MFI.hasVarSizedObjects()) {
1665  // If we have variable sized objects, we can use either FP or BP, as the
1666  // SP offset is unknown. We can use the base pointer if we have one and
1667  // FP is not preferred. If not, we're stuck with using FP.
1668  bool CanUseBP = RegInfo->hasBasePointer(MF);
1669  if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
1670  UseFP = PreferFP;
1671  else if (!CanUseBP) { // Can't use BP. Forced to use FP.
1672  assert(!SVEStackSize && "Expected BP to be available");
1673  UseFP = true;
1674  }
1675  // else we can use BP and FP, but the offset from FP won't fit.
1676  // That will make us scavenge registers which we can probably avoid by
1677  // using BP. If it won't fit for BP either, we'll scavenge anyway.
1678  } else if (FPOffset >= 0) {
1679  // Use SP or FP, whichever gives us the best chance of the offset
1680  // being in range for direct access. If the FPOffset is positive,
1681  // that'll always be best, as the SP will be even further away.
1682  UseFP = true;
1683  } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
1684  // Funclets access the locals contained in the parent's stack frame
1685  // via the frame pointer, so we have to use the FP in the parent
1686  // function.
1687  (void) Subtarget;
1688  assert(
1689  Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
1690  "Funclets should only be present on Win64");
1691  UseFP = true;
1692  } else {
1693  // We have the choice between FP and (SP or BP).
1694  if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
1695  UseFP = true;
1696  }
1697  }
1698  }
1699 
1700  assert(((isFixed || isCSR) || !RegInfo->needsStackRealignment(MF) || !UseFP) &&
1701  "In the presence of dynamic stack pointer realignment, "
1702  "non-argument/CSR objects cannot be accessed through the frame pointer");
1703 
1704  if (isSVE) {
1705  int64_t OffsetToSVEArea =
1706  MFI.getStackSize() - AFI->getCalleeSavedStackSize();
1707  StackOffset FPOffset = {ObjectOffset, MVT::nxv1i8};
1708  StackOffset SPOffset = SVEStackSize +
1709  StackOffset(ObjectOffset, MVT::nxv1i8) +
1710  StackOffset(OffsetToSVEArea, MVT::i8);
1711  // Always use the FP for SVE spills if available and beneficial.
1712  if (hasFP(MF) &&
1713  (SPOffset.getBytes() ||
1714  FPOffset.getScalableBytes() < SPOffset.getScalableBytes() ||
1715  RegInfo->needsStackRealignment(MF))) {
1716  FrameReg = RegInfo->getFrameRegister(MF);
1717  return FPOffset;
1718  }
1719 
1720  FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
1721  : (unsigned)AArch64::SP;
1722  return SPOffset;
1723  }
1724 
1725  StackOffset ScalableOffset = {};
1726  if (UseFP && !(isFixed || isCSR))
1727  ScalableOffset = -SVEStackSize;
1728  if (!UseFP && (isFixed || isCSR))
1729  ScalableOffset = SVEStackSize;
1730 
1731  if (UseFP) {
1732  FrameReg = RegInfo->getFrameRegister(MF);
1733  return StackOffset(FPOffset, MVT::i8) + ScalableOffset;
1734  }
1735 
1736  // Use the base pointer if we have one.
1737  if (RegInfo->hasBasePointer(MF))
1738  FrameReg = RegInfo->getBaseRegister();
1739  else {
1740  assert(!MFI.hasVarSizedObjects() &&
1741  "Can't use SP when we have var sized objects.");
1742  FrameReg = AArch64::SP;
1743  // If we're using the red zone for this function, the SP won't actually
1744  // be adjusted, so the offsets will be negative. They're also all
1745  // within range of the signed 9-bit immediate instructions.
1746  if (canUseRedZone(MF))
1747  Offset -= AFI->getLocalStackSize();
1748  }
1749 
1750  return StackOffset(Offset, MVT::i8) + ScalableOffset;
1751 }
1752 
1753 static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
1754  // Do not set a kill flag on values that are also marked as live-in. This
1755  // happens with the @llvm-returnaddress intrinsic and with arguments passed in
1756  // callee saved registers.
1757  // Omitting the kill flags is conservatively correct even if the live-in
1758  // is not used after all.
1759  bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
1760  return getKillRegState(!IsLiveIn);
1761 }
1762 
1764  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1766  return Subtarget.isTargetMachO() &&
1767  !(Subtarget.getTargetLowering()->supportSwiftError() &&
1768  Attrs.hasAttrSomewhere(Attribute::SwiftError));
1769 }
1770 
1771 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
1772  bool NeedsWinCFI) {
1773  // If we are generating register pairs for a Windows function that requires
1774  // EH support, then pair consecutive registers only. There are no unwind
1775  // opcodes for saves/restores of non-consectuve register pairs.
1776  // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x.
1777  // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
1778 
1779  // TODO: LR can be paired with any register. We don't support this yet in
1780  // the MCLayer. We need to add support for the save_lrpair unwind code.
1781  if (!NeedsWinCFI)
1782  return false;
1783  if (Reg2 == Reg1 + 1)
1784  return false;
1785  return true;
1786 }
1787 
1788 /// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
1789 /// WindowsCFI requires that only consecutive registers can be paired.
1790 /// LR and FP need to be allocated together when the frame needs to save
1791 /// the frame-record. This means any other register pairing with LR is invalid.
1792 static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2,
1793  bool NeedsWinCFI, bool NeedsFrameRecord) {
1794  if (NeedsWinCFI)
1795  return invalidateWindowsRegisterPairing(Reg1, Reg2, true);
1796 
1797  // If we need to store the frame record, don't pair any register
1798  // with LR other than FP.
1799  if (NeedsFrameRecord)
1800  return Reg2 == AArch64::LR;
1801 
1802  return false;
1803 }
1804 
1805 namespace {
1806 
1807 struct RegPairInfo {
1808  unsigned Reg1 = AArch64::NoRegister;
1809  unsigned Reg2 = AArch64::NoRegister;
1810  int FrameIdx;
1811  int Offset;
1812  enum RegType { GPR, FPR64, FPR128 } Type;
1813 
1814  RegPairInfo() = default;
1815 
1816  bool isPaired() const { return Reg2 != AArch64::NoRegister; }
1817 };
1818 
1819 } // end anonymous namespace
1820 
1822  MachineFunction &MF, const std::vector<CalleeSavedInfo> &CSI,
1824  bool &NeedShadowCallStackProlog, bool NeedsFrameRecord) {
1825 
1826  if (CSI.empty())
1827  return;
1828 
1829  bool NeedsWinCFI = needsWinCFI(MF);
1831  MachineFrameInfo &MFI = MF.getFrameInfo();
1833  unsigned Count = CSI.size();
1834  (void)CC;
1835  // MachO's compact unwind format relies on all registers being stored in
1836  // pairs.
1838  CC == CallingConv::PreserveMost ||
1839  (Count & 1) == 0) &&
1840  "Odd number of callee-saved regs to spill!");
1841  int Offset = AFI->getCalleeSavedStackSize();
1842  // On Linux, we will have either one or zero non-paired register. On Windows
1843  // with CFI, we can have multiple unpaired registers in order to utilize the
1844  // available unwind codes. This flag assures that the alignment fixup is done
1845  // only once, as intened.
1846  bool FixupDone = false;
1847  for (unsigned i = 0; i < Count; ++i) {
1848  RegPairInfo RPI;
1849  RPI.Reg1 = CSI[i].getReg();
1850 
1851  if (AArch64::GPR64RegClass.contains(RPI.Reg1))
1852  RPI.Type = RegPairInfo::GPR;
1853  else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
1854  RPI.Type = RegPairInfo::FPR64;
1855  else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
1856  RPI.Type = RegPairInfo::FPR128;
1857  else
1858  llvm_unreachable("Unsupported register class.");
1859 
1860  // Add the next reg to the pair if it is in the same register class.
1861  if (i + 1 < Count) {
1862  unsigned NextReg = CSI[i + 1].getReg();
1863  switch (RPI.Type) {
1864  case RegPairInfo::GPR:
1865  if (AArch64::GPR64RegClass.contains(NextReg) &&
1866  !invalidateRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI,
1867  NeedsFrameRecord))
1868  RPI.Reg2 = NextReg;
1869  break;
1870  case RegPairInfo::FPR64:
1871  if (AArch64::FPR64RegClass.contains(NextReg) &&
1872  !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI))
1873  RPI.Reg2 = NextReg;
1874  break;
1875  case RegPairInfo::FPR128:
1876  if (AArch64::FPR128RegClass.contains(NextReg))
1877  RPI.Reg2 = NextReg;
1878  break;
1879  }
1880  }
1881 
1882  // If either of the registers to be saved is the lr register, it means that
1883  // we also need to save lr in the shadow call stack.
1884  if ((RPI.Reg1 == AArch64::LR || RPI.Reg2 == AArch64::LR) &&
1885  MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
1887  report_fatal_error("Must reserve x18 to use shadow call stack");
1888  NeedShadowCallStackProlog = true;
1889  }
1890 
1891  // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
1892  // list to come in sorted by frame index so that we can issue the store
1893  // pair instructions directly. Assert if we see anything otherwise.
1894  //
1895  // The order of the registers in the list is controlled by
1896  // getCalleeSavedRegs(), so they will always be in-order, as well.
1897  assert((!RPI.isPaired() ||
1898  (CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx())) &&
1899  "Out of order callee saved regs!");
1900 
1901  assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
1902  RPI.Reg1 == AArch64::LR) &&
1903  "FrameRecord must be allocated together with LR");
1904 
1905  // MachO's compact unwind format relies on all registers being stored in
1906  // adjacent register pairs.
1908  CC == CallingConv::PreserveMost ||
1909  (RPI.isPaired() &&
1910  ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
1911  RPI.Reg1 + 1 == RPI.Reg2))) &&
1912  "Callee-save registers not saved as adjacent register pair!");
1913 
1914  RPI.FrameIdx = CSI[i].getFrameIdx();
1915 
1916  int Scale = RPI.Type == RegPairInfo::FPR128 ? 16 : 8;
1917  Offset -= RPI.isPaired() ? 2 * Scale : Scale;
1918 
1919  // Round up size of non-pair to pair size if we need to pad the
1920  // callee-save area to ensure 16-byte alignment.
1921  if (AFI->hasCalleeSaveStackFreeSpace() && !FixupDone &&
1922  RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired()) {
1923  FixupDone = true;
1924  Offset -= 8;
1925  assert(Offset % 16 == 0);
1926  assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16);
1927  MFI.setObjectAlignment(RPI.FrameIdx, 16);
1928  }
1929 
1930  assert(Offset % Scale == 0);
1931  RPI.Offset = Offset / Scale;
1932  assert((RPI.Offset >= -64 && RPI.Offset <= 63) &&
1933  "Offset out of bounds for LDP/STP immediate");
1934 
1935  RegPairs.push_back(RPI);
1936  if (RPI.isPaired())
1937  ++i;
1938  }
1939 }
1940 
1943  const std::vector<CalleeSavedInfo> &CSI,
1944  const TargetRegisterInfo *TRI) const {
1945  MachineFunction &MF = *MBB.getParent();
1946  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1947  bool NeedsWinCFI = needsWinCFI(MF);
1948  DebugLoc DL;
1949  SmallVector<RegPairInfo, 8> RegPairs;
1950 
1951  bool NeedShadowCallStackProlog = false;
1952  computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
1953  NeedShadowCallStackProlog, hasFP(MF));
1954  const MachineRegisterInfo &MRI = MF.getRegInfo();
1955 
1956  if (NeedShadowCallStackProlog) {
1957  // Shadow call stack prolog: str x30, [x18], #8
1958  BuildMI(MBB, MI, DL, TII.get(AArch64::STRXpost))
1959  .addReg(AArch64::X18, RegState::Define)
1960  .addReg(AArch64::LR)
1961  .addReg(AArch64::X18)
1962  .addImm(8)
1964 
1965  if (NeedsWinCFI)
1966  BuildMI(MBB, MI, DL, TII.get(AArch64::SEH_Nop))
1967  .setMIFlag(MachineInstr::FrameSetup);
1968 
1969  if (!MF.getFunction().hasFnAttribute(Attribute::NoUnwind)) {
1970  // Emit a CFI instruction that causes 8 to be subtracted from the value of
1971  // x18 when unwinding past this frame.
1972  static const char CFIInst[] = {
1973  dwarf::DW_CFA_val_expression,
1974  18, // register
1975  2, // length
1976  static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
1977  static_cast<char>(-8) & 0x7f, // addend (sleb128)
1978  };
1979  unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
1980  nullptr, StringRef(CFIInst, sizeof(CFIInst))));
1981  BuildMI(MBB, MI, DL, TII.get(AArch64::CFI_INSTRUCTION))
1982  .addCFIIndex(CFIIndex)
1984  }
1985 
1986  // This instruction also makes x18 live-in to the entry block.
1987  MBB.addLiveIn(AArch64::X18);
1988  }
1989 
1990  for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
1991  ++RPII) {
1992  RegPairInfo RPI = *RPII;
1993  unsigned Reg1 = RPI.Reg1;
1994  unsigned Reg2 = RPI.Reg2;
1995  unsigned StrOpc;
1996 
1997  // Issue sequence of spills for cs regs. The first spill may be converted
1998  // to a pre-decrement store later by emitPrologue if the callee-save stack
1999  // area allocation can't be combined with the local stack area allocation.
2000  // For example:
2001  // stp x22, x21, [sp, #0] // addImm(+0)
2002  // stp x20, x19, [sp, #16] // addImm(+2)
2003  // stp fp, lr, [sp, #32] // addImm(+4)
2004  // Rationale: This sequence saves uop updates compared to a sequence of
2005  // pre-increment spills like stp xi,xj,[sp,#-16]!
2006  // Note: Similar rationale and sequence for restores in epilog.
2007  unsigned Size, Align;
2008  switch (RPI.Type) {
2009  case RegPairInfo::GPR:
2010  StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
2011  Size = 8;
2012  Align = 8;
2013  break;
2014  case RegPairInfo::FPR64:
2015  StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
2016  Size = 8;
2017  Align = 8;
2018  break;
2019  case RegPairInfo::FPR128:
2020  StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
2021  Size = 16;
2022  Align = 16;
2023  break;
2024  }
2025  LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
2026  if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2027  dbgs() << ") -> fi#(" << RPI.FrameIdx;
2028  if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2029  dbgs() << ")\n");
2030 
2031  assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
2032  "Windows unwdinding requires a consecutive (FP,LR) pair");
2033  // Windows unwind codes require consecutive registers if registers are
2034  // paired. Make the switch here, so that the code below will save (x,x+1)
2035  // and not (x+1,x).
2036  unsigned FrameIdxReg1 = RPI.FrameIdx;
2037  unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2038  if (NeedsWinCFI && RPI.isPaired()) {
2039  std::swap(Reg1, Reg2);
2040  std::swap(FrameIdxReg1, FrameIdxReg2);
2041  }
2042  MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
2043  if (!MRI.isReserved(Reg1))
2044  MBB.addLiveIn(Reg1);
2045  if (RPI.isPaired()) {
2046  if (!MRI.isReserved(Reg2))
2047  MBB.addLiveIn(Reg2);
2048  MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
2050  MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2051  MachineMemOperand::MOStore, Size, Align));
2052  }
2053  MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
2054  .addReg(AArch64::SP)
2055  .addImm(RPI.Offset) // [sp, #offset*scale],
2056  // where factor*scale is implicit
2059  MachinePointerInfo::getFixedStack(MF,FrameIdxReg1),
2060  MachineMemOperand::MOStore, Size, Align));
2061  if (NeedsWinCFI)
2063 
2064  }
2065  return true;
2066 }
2067 
2070  std::vector<CalleeSavedInfo> &CSI,
2071  const TargetRegisterInfo *TRI) const {
2072  MachineFunction &MF = *MBB.getParent();
2073  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2074  DebugLoc DL;
2075  SmallVector<RegPairInfo, 8> RegPairs;
2076  bool NeedsWinCFI = needsWinCFI(MF);
2077 
2078  if (MI != MBB.end())
2079  DL = MI->getDebugLoc();
2080 
2081  bool NeedShadowCallStackProlog = false;
2082  computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs,
2083  NeedShadowCallStackProlog, hasFP(MF));
2084 
2085  auto EmitMI = [&](const RegPairInfo &RPI) {
2086  unsigned Reg1 = RPI.Reg1;
2087  unsigned Reg2 = RPI.Reg2;
2088 
2089  // Issue sequence of restores for cs regs. The last restore may be converted
2090  // to a post-increment load later by emitEpilogue if the callee-save stack
2091  // area allocation can't be combined with the local stack area allocation.
2092  // For example:
2093  // ldp fp, lr, [sp, #32] // addImm(+4)
2094  // ldp x20, x19, [sp, #16] // addImm(+2)
2095  // ldp x22, x21, [sp, #0] // addImm(+0)
2096  // Note: see comment in spillCalleeSavedRegisters()
2097  unsigned LdrOpc;
2098  unsigned Size, Align;
2099  switch (RPI.Type) {
2100  case RegPairInfo::GPR:
2101  LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
2102  Size = 8;
2103  Align = 8;
2104  break;
2105  case RegPairInfo::FPR64:
2106  LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
2107  Size = 8;
2108  Align = 8;
2109  break;
2110  case RegPairInfo::FPR128:
2111  LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
2112  Size = 16;
2113  Align = 16;
2114  break;
2115  }
2116  LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
2117  if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
2118  dbgs() << ") -> fi#(" << RPI.FrameIdx;
2119  if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
2120  dbgs() << ")\n");
2121 
2122  // Windows unwind codes require consecutive registers if registers are
2123  // paired. Make the switch here, so that the code below will save (x,x+1)
2124  // and not (x+1,x).
2125  unsigned FrameIdxReg1 = RPI.FrameIdx;
2126  unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
2127  if (NeedsWinCFI && RPI.isPaired()) {
2128  std::swap(Reg1, Reg2);
2129  std::swap(FrameIdxReg1, FrameIdxReg2);
2130  }
2131  MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc));
2132  if (RPI.isPaired()) {
2133  MIB.addReg(Reg2, getDefRegState(true));
2135  MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
2136  MachineMemOperand::MOLoad, Size, Align));
2137  }
2138  MIB.addReg(Reg1, getDefRegState(true))
2139  .addReg(AArch64::SP)
2140  .addImm(RPI.Offset) // [sp, #offset*scale]
2141  // where factor*scale is implicit
2144  MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
2145  MachineMemOperand::MOLoad, Size, Align));
2146  if (NeedsWinCFI)
2148  };
2150  for (const RegPairInfo &RPI : reverse(RegPairs))
2151  EmitMI(RPI);
2152  else
2153  for (const RegPairInfo &RPI : RegPairs)
2154  EmitMI(RPI);
2155 
2156  if (NeedShadowCallStackProlog) {
2157  // Shadow call stack epilog: ldr x30, [x18, #-8]!
2158  BuildMI(MBB, MI, DL, TII.get(AArch64::LDRXpre))
2159  .addReg(AArch64::X18, RegState::Define)
2160  .addReg(AArch64::LR, RegState::Define)
2161  .addReg(AArch64::X18)
2162  .addImm(-8)
2164  }
2165 
2166  return true;
2167 }
2168 
2170  BitVector &SavedRegs,
2171  RegScavenger *RS) const {
2172  // All calls are tail calls in GHC calling conv, and functions have no
2173  // prologue/epilogue.
2175  return;
2176 
2177  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2178  const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
2179  MF.getSubtarget().getRegisterInfo());
2181  unsigned UnspilledCSGPR = AArch64::NoRegister;
2182  unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
2183 
2184  MachineFrameInfo &MFI = MF.getFrameInfo();
2185  const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
2186 
2187  unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
2188  ? RegInfo->getBaseRegister()
2189  : (unsigned)AArch64::NoRegister;
2190 
2191  unsigned ExtraCSSpill = 0;
2192  // Figure out which callee-saved registers to save/restore.
2193  for (unsigned i = 0; CSRegs[i]; ++i) {
2194  const unsigned Reg = CSRegs[i];
2195 
2196  // Add the base pointer register to SavedRegs if it is callee-save.
2197  if (Reg == BasePointerReg)
2198  SavedRegs.set(Reg);
2199 
2200  bool RegUsed = SavedRegs.test(Reg);
2201  unsigned PairedReg = CSRegs[i ^ 1];
2202  if (!RegUsed) {
2203  if (AArch64::GPR64RegClass.contains(Reg) &&
2204  !RegInfo->isReservedReg(MF, Reg)) {
2205  UnspilledCSGPR = Reg;
2206  UnspilledCSGPRPaired = PairedReg;
2207  }
2208  continue;
2209  }
2210 
2211  // MachO's compact unwind format relies on all registers being stored in
2212  // pairs.
2213  // FIXME: the usual format is actually better if unwinding isn't needed.
2214  if (produceCompactUnwindFrame(MF) && PairedReg != AArch64::NoRegister &&
2215  !SavedRegs.test(PairedReg)) {
2216  SavedRegs.set(PairedReg);
2217  if (AArch64::GPR64RegClass.contains(PairedReg) &&
2218  !RegInfo->isReservedReg(MF, PairedReg))
2219  ExtraCSSpill = PairedReg;
2220  }
2221  }
2222 
2223  // Calculates the callee saved stack size.
2224  unsigned CSStackSize = 0;
2226  const MachineRegisterInfo &MRI = MF.getRegInfo();
2227  for (unsigned Reg : SavedRegs.set_bits())
2228  CSStackSize += TRI->getRegSizeInBits(Reg, MRI) / 8;
2229 
2230  // Save number of saved regs, so we can easily update CSStackSize later.
2231  unsigned NumSavedRegs = SavedRegs.count();
2232 
2233  // The frame record needs to be created by saving the appropriate registers
2234  unsigned EstimatedStackSize = MFI.estimateStackSize(MF);
2235  if (hasFP(MF) ||
2236  windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
2237  SavedRegs.set(AArch64::FP);
2238  SavedRegs.set(AArch64::LR);
2239  }
2240 
2241  LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
2242  for (unsigned Reg
2243  : SavedRegs.set_bits()) dbgs()
2244  << ' ' << printReg(Reg, RegInfo);
2245  dbgs() << "\n";);
2246 
2247  // If any callee-saved registers are used, the frame cannot be eliminated.
2248  unsigned MaxAlign = getStackAlignment();
2249  int64_t SVEStackSize =
2250  alignTo(determineSVEStackSize(MFI, MaxAlign), MaxAlign);
2251  assert(MaxAlign <= 16 && "Cannot align scalable vectors more than 16 bytes");
2252  bool CanEliminateFrame = (SavedRegs.count() == 0) && !SVEStackSize;
2253 
2254  // The CSR spill slots have not been allocated yet, so estimateStackSize
2255  // won't include them.
2256  unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
2257 
2258  // Conservatively always assume BigStack when there are SVE spills.
2259  bool BigStack = SVEStackSize ||
2260  (EstimatedStackSize + CSStackSize) > EstimatedStackSizeLimit;
2261  if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
2262  AFI->setHasStackFrame(true);
2263 
2264  // Estimate if we might need to scavenge a register at some point in order
2265  // to materialize a stack offset. If so, either spill one additional
2266  // callee-saved register or reserve a special spill slot to facilitate
2267  // register scavenging. If we already spilled an extra callee-saved register
2268  // above to keep the number of spills even, we don't need to do anything else
2269  // here.
2270  if (BigStack) {
2271  if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
2272  LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
2273  << " to get a scratch register.\n");
2274  SavedRegs.set(UnspilledCSGPR);
2275  // MachO's compact unwind format relies on all registers being stored in
2276  // pairs, so if we need to spill one extra for BigStack, then we need to
2277  // store the pair.
2278  if (produceCompactUnwindFrame(MF))
2279  SavedRegs.set(UnspilledCSGPRPaired);
2280  ExtraCSSpill = UnspilledCSGPR;
2281  }
2282 
2283  // If we didn't find an extra callee-saved register to spill, create
2284  // an emergency spill slot.
2285  if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
2286  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
2287  const TargetRegisterClass &RC = AArch64::GPR64RegClass;
2288  unsigned Size = TRI->getSpillSize(RC);
2289  unsigned Align = TRI->getSpillAlignment(RC);
2290  int FI = MFI.CreateStackObject(Size, Align, false);
2291  RS->addScavengingFrameIndex(FI);
2292  LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
2293  << " as the emergency spill slot.\n");
2294  }
2295  }
2296 
2297  // Adding the size of additional 64bit GPR saves.
2298  CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
2299  unsigned AlignedCSStackSize = alignTo(CSStackSize, 16);
2300  LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
2301  << EstimatedStackSize + AlignedCSStackSize
2302  << " bytes.\n");
2303 
2304  // Round up to register pair alignment to avoid additional SP adjustment
2305  // instructions.
2306  AFI->setCalleeSavedStackSize(AlignedCSStackSize);
2307  AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
2308 }
2309 
2311  const MachineFunction &MF) const {
2312  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
2313  return AFI->hasCalleeSaveStackFreeSpace();
2314 }
2315 
2316 int64_t AArch64FrameLowering::determineSVEStackSize(MachineFrameInfo &MFI,
2317  unsigned &MaxAlign) const {
2318  // Process all fixed stack objects.
2319  int64_t Offset = 0;
2320  for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
2321  if (MFI.getStackID(I) == TargetStackID::SVEVector) {
2322  int64_t FixedOffset = -MFI.getObjectOffset(I);
2323  if (FixedOffset > Offset)
2324  Offset = FixedOffset;
2325  }
2326 
2327  // Note: We don't take allocatable stack objects into
2328  // account yet, because allocation for those is not yet
2329  // implemented.
2330  return Offset;
2331 }
2332 
2334  MachineFunction &MF, RegScavenger *RS) const {
2335  MachineFrameInfo &MFI = MF.getFrameInfo();
2336 
2338  "Upwards growing stack unsupported");
2339 
2340  unsigned MaxAlign = getStackAlignment();
2341  int64_t SVEStackSize = determineSVEStackSize(MFI, MaxAlign);
2342 
2344  AFI->setStackSizeSVE(alignTo(SVEStackSize, MaxAlign));
2345  assert(MaxAlign <= 16 && "Cannot align scalable vectors more than 16 bytes");
2346 
2347  // If this function isn't doing Win64-style C++ EH, we don't need to do
2348  // anything.
2349  if (!MF.hasEHFunclets())
2350  return;
2351  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
2352  WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
2353 
2354  MachineBasicBlock &MBB = MF.front();
2355  auto MBBI = MBB.begin();
2356  while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
2357  ++MBBI;
2358 
2359  // Create an UnwindHelp object.
2360  int UnwindHelpFI =
2361  MFI.CreateStackObject(/*size*/8, /*alignment*/16, false);
2362  EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
2363  // We need to store -2 into the UnwindHelp object at the start of the
2364  // function.
2365  DebugLoc DL;
2366  RS->enterBasicBlockEnd(MBB);
2367  RS->backward(std::prev(MBBI));
2368  unsigned DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
2369  assert(DstReg && "There must be a free register after frame setup");
2370  BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
2371  BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
2372  .addReg(DstReg, getKillRegState(true))
2373  .addFrameIndex(UnwindHelpFI)
2374  .addImm(0);
2375 }
2376 
2377 /// For Win64 AArch64 EH, the offset to the Unwind object is from the SP before
2378 /// the update. This is easily retrieved as it is exactly the offset that is set
2379 /// in processFunctionBeforeFrameFinalized.
2381  const MachineFunction &MF, int FI, unsigned &FrameReg,
2382  bool IgnoreSPUpdates) const {
2383  const MachineFrameInfo &MFI = MF.getFrameInfo();
2384  LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
2385  << MFI.getObjectOffset(FI) << "\n");
2386  FrameReg = AArch64::SP;
2387  return MFI.getObjectOffset(FI);
2388 }
2389 
2390 /// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
2391 /// the parent's frame pointer
2393  const MachineFunction &MF) const {
2394  return 0;
2395 }
2396 
2397 /// Funclets only need to account for space for the callee saved registers,
2398 /// as the locals are accounted for in the parent's stack frame.
2400  const MachineFunction &MF) const {
2401  // This is the size of the pushed CSRs.
2402  unsigned CSSize =
2403  MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
2404  // This is the amount of stack a funclet needs to allocate.
2405  return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
2406  getStackAlignment());
2407 }
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, std::vector< CalleeSavedInfo > &CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
static cl::opt< bool > ReverseCSRRestoreSeq("reverse-csr-restore-seq", cl::desc("reverse the CSR restore sequence"), cl::init(false), cl::Hidden)
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand *> MMOs) const
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:112
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
bool usesWindowsCFI() const
Definition: MCAsmInfo.h:607
BitVector & set()
Definition: BitVector.h:397
bool hasStackMap() const
This method may be called any time after instruction selection is complete to determine if there is a...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
StackOffset is a wrapper around scalable and non-scalable offsets and is used in several functions su...
bool isReserved(Register PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
bool hasDebugInfo() const
Returns true if valid debug info is present.
static bool ShouldSignWithAKey(MachineFunction &MF)
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
LLVM_NODISCARD bool equals_lower(StringRef RHS) const
equals_lower - Check for string equality, ignoring case.
Definition: StringRef.h:181
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
class llvm::RegisterBankInfo GPR
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
static void Cleanup(DenseMap< Instruction *, Instruction *> &NewPredicates, SetVector< Instruction *> &MaybeDead, Loop *L)
bool cannotEliminateFrame(const MachineFunction &MF) const
LLVM_NODISCARD unsigned addFrameInst(const MCCFIInstruction &Inst)
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int Offset)
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
Definition: MCDwarf.h:507
unsigned Reg
bool test(unsigned Idx) const
Definition: BitVector.h:501
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:58
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:323
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
F(f)
MachineModuleInfo & getMMI() const
bool canUseRedZone(const MachineFunction &MF) const
Can this function use the red zone for local allocations.
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2, bool NeedsWinCFI)
static MCCFIInstruction createDefCfaOffset(MCSymbol *L, int Offset)
.cfi_def_cfa_offset modifies a rule for computing CFA.
Definition: MCDwarf.h:494
bool isCallingConvWin64(CallingConv::ID CC) const
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, unsigned LocalStackSize, bool NeedsWinCFI, bool *HasWinCFI)
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
return AArch64::GPR64RegClass contains(Reg)
static bool isFuncletReturnInstr(const MachineInstr &MI)
static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB)
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
StackOffset resolveFrameOffsetReference(const MachineFunction &MF, int ObjectOffset, bool isFixed, bool isSVE, unsigned &FrameReg, bool PreferFP, bool ForSimm) const
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg)
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:156
unsigned getSpillAlignment(const TargetRegisterClass &RC) const
Return the minimum required alignment in bytes for a spill slot for a register of this class...
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:722
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
int getSEHFrameIndexOffset(const MachineFunction &MF, int FI) const
int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset, bool *OutUseUnscaledOp=nullptr, unsigned *OutUnscaledOp=nullptr, int64_t *EmittableOffset=nullptr)
Check if the Offset is a valid frame offset for MI.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
static bool isSEHInstruction(const MachineInstr &MI)
Return true if the instructions is a SEH instruciton used for unwinding on Windows.
const HexagonInstrInfo * TII
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required, we reserve argument space for call sites in the function immediately on entry to the current function.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
int getFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg) const override
getFrameIndexReference - Provide a base+offset reference to an FI slot for debug info.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
This file contains the simple types necessary to represent the attributes associated with functions a...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
void backward()
Update internal register state and move MBB iterator backwards.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:411
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
static StackOffset getFPOffset(const MachineFunction &MF, int ObjectOffset)
int getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI, unsigned &FrameReg, bool IgnoreSPUpdates) const override
For Win64 AArch64 EH, the offset to the Unwind object is from the SP before the update.
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
unsigned getWinEHParentFrameOffset(const MachineFunction &MF) const override
The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve the parent&#39;s frame pointer...
int64_t getScalableBytes() const
Returns the scalable part of the offset in bytes.
static unsigned estimateRSStackSizeLimit(MachineFunction &MF)
Look at each instruction that references stack frames and return the stack size limit beyond which so...
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:261
Key
PAL metadata keys.
int getDwarfRegNum(MCRegister RegNum, bool isEH) const
Map a target register to an equivalent dwarf register number.
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
virtual void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, bool KillSrc) const
Emit instructions to copy a pair of physical registers.
const AArch64RegisterInfo * getRegisterInfo() const override
bool DisableFramePointerElim(const MachineFunction &MF) const
DisableFramePointerElim - This returns true if frame pointer elimination optimization should be disab...
const MCContext & getContext() const
int getObjectIndexBegin() const
Return the minimum frame object index.
iterator getLastNonDebugInstr()
Returns an iterator to the last non-debug instruction in the basic block, or end().
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:223
bool hasPersonalityFn() const
Check whether this function has a personality function.
Definition: Function.h:732
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
unsigned getWinEHFuncletFrameSize(const MachineFunction &MF) const
Funclets only need to account for space for the callee saved registers, as the locals are accounted f...
virtual const TargetInstrInfo * getInstrInfo() const
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
bool isXRegisterReserved(size_t i) const
int64_t getBytes() const
Returns the non-scalable part of the offset in bytes.
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
unsigned getKillRegState(bool B)
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:19
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:131
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
TargetInstrInfo - Interface to description of machine instruction set.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
int getNonLocalFrameIndexReference(const MachineFunction &MF, int FI) const override
getNonLocalFrameIndexReference - This method returns the offset used to reference a frame index locat...
unsigned getDefRegState(bool B)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
const AArch64TargetLowering * getTargetLowering() const override
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1165
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
static MCCFIInstruction createNegateRAState(MCSymbol *L)
.cfi_negate_ra_state AArch64 negate RA state.
Definition: MCDwarf.h:533
StackOffset resolveFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg, bool PreferFP, bool ForSimm) const
This file declares the machine register scavenger class.
MCSymbol * createTempSymbol(bool CanBeUnnamed=true)
Create and return a new assembler temporary symbol with a unique but unspecified name.
Definition: MCContext.cpp:225
unsigned const MachineRegisterInfo * MRI
static MCCFIInstruction createDefCfa(MCSymbol *L, unsigned Register, int Offset)
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it...
Definition: MCDwarf.h:480
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static bool ShouldSignReturnAddress(MachineFunction &MF)
static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2, bool NeedsWinCFI, bool NeedsFrameRecord)
Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
bool hasEHFunclets() const
unsigned getPointerSize(unsigned AS=0) const
Layout pointer size FIXME: The defaults need to be removed once all of the backends/clients are updat...
Definition: DataLayout.cpp:654
void addLiveIns(const MachineBasicBlock &MBB)
Adds all live-in registers of basic block MBB.
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any DBG_VALUE and DBG_LABEL instructions...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
unsigned getMaxAlignment() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
void setImm(int64_t immVal)
static cl::opt< bool > EnableRedZone("aarch64-redzone", cl::desc("enable use of redzone on AArch64"), cl::init(false), cl::Hidden)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const MachineBasicBlock & front() const
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:40
void emitCalleeSavedFrameMoves(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const
The memory access writes data.
std::enable_if< std::numeric_limits< T >::is_signed, bool >::type getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:492
static StackOffset getStackOffset(const MachineFunction &MF, int ObjectOffset)
static StackOffset getSVEStackSize(const MachineFunction &MF)
Returns the size of the entire SVE stackframe (calleesaves + spills).
size_type count() const
count - Returns the number of bits which are set.
Definition: BitVector.h:172
static void InsertReturnAddressAuth(MachineFunction &MF, MachineBasicBlock &MBB)
unsigned getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call...
static void computeCalleeSaveRegisterPairs(MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI, const TargetRegisterInfo *TRI, SmallVectorImpl< RegPairInfo > &RegPairs, bool &NeedShadowCallStackProlog, bool NeedsFrameRecord)
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:212
bool available(const MachineRegisterInfo &MRI, MCPhysReg Reg) const
Returns true if register Reg and no aliasing register is in the set.
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
unsigned estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
static const unsigned DefaultSafeSPDisplacement
This is the biggest offset to the stack pointer we can encode in aarch64 instructions (without using ...
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling...
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
int64_t getImm() const
const Function & getFunction() const
Return the LLVM function that this machine code represents.
static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI, const TargetInstrInfo &TII, MachineInstr::MIFlag Flag)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
bool needsUnwindTableEntry() const
True if this function needs an unwind table.
Definition: Function.h:594
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
bool isPhysRegUsed(unsigned PhysReg) const
Return true if the specified register is modified or read in this function.
bool enableStackSlotScavenging(const MachineFunction &MF) const override
Returns true if the stack slot holes in the fixed and callee-save stack area should be used when allo...
CodeModel::Model getCodeModel() const
Returns the code model.
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
static bool needsWinCFI(const MachineFunction &MF)
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static bool windowsRequiresStackProbe(MachineFunction &MF, unsigned StackSizeInBytes)
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register...
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:718
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:256
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
TargetSubtargetInfo - Generic base class for all target subtargets.
static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, unsigned LocalStackSize)
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
LLVM_NODISCARD bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:174
bool isReservedReg(const MachineFunction &MF, unsigned Reg) const
bool isLiveIn(unsigned Reg) const
Representation of each machine instruction.
Definition: MachineInstr.h:64
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
static void adaptForLdStOpt(MachineBasicBlock &MBB, MachineBasicBlock::iterator FirstSPPopI, MachineBasicBlock::iterator LastPopI)
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:163
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB &#39;Other&#39; at the position From, and insert it into this MBB right before &#39;...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:48
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:220
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
Register FindUnusedReg(const TargetRegisterClass *RC) const
Find an unused register of the specified register class.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:44
TargetOptions Options
#define I(x, y, z)
Definition: MD5.cpp:58
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector< CalleeSavedInfo > &CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
uint32_t Size
Definition: Profile.cpp:46
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
static bool isTargetDarwin(const MachineFunction &MF)
static bool produceCompactUnwindFrame(MachineFunction &MF)
uint8_t getStackID(int ObjectIdx) const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it...
bool hasBasePointer(const MachineFunction &MF) const
iterator_range< const_set_bits_iterator > set_bits() const
Definition: BitVector.h:129
bool needsStackRealignment(const MachineFunction &MF) const
True if storage within the function requires the stack pointer to be aligned more than the normal cal...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1465
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc, bool NeedsWinCFI, bool *HasWinCFI, bool InProlog=true)
const AArch64InstrInfo * getInstrInfo() const override
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:273
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
void addReg(MCPhysReg Reg)
Adds a physical register and all its sub-registers to the set.
Definition: LivePhysRegs.h:79
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:333
bool isMaxCallFrameSizeComputed() const
IRTranslator LLVM IR MI
bool hasPatchPoint() const
This method may be called any time after instruction selection is complete to determine if there is a...
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
void setTaggedBasePointerOffset(unsigned Offset)
unsigned getRegSizeInBits(const TargetRegisterClass &RC) const
Return the size in bits of a register from class RC.
Register getReg() const
getReg - Returns the register number.
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals)
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
Definition: MCDwarf.h:568
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects...
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
void setCalleeSavedStackSize(unsigned Size)
This class contains meta information specific to a module.
bool hasCalls() const
Return true if the current function has any function calls.