LLVM 19.0.0git
AArch64FrameLowering.cpp
Go to the documentation of this file.
1//===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of TargetFrameLowering class.
10//
11// On AArch64, stack frames are structured as follows:
12//
13// The stack grows downward.
14//
15// All of the individual frame areas on the frame below are optional, i.e. it's
16// possible to create a function so that the particular area isn't present
17// in the frame.
18//
19// At function entry, the "frame" looks as follows:
20//
21// | | Higher address
22// |-----------------------------------|
23// | |
24// | arguments passed on the stack |
25// | |
26// |-----------------------------------| <- sp
27// | | Lower address
28//
29//
30// After the prologue has run, the frame has the following general structure.
31// Note that this doesn't depict the case where a red-zone is used. Also,
32// technically the last frame area (VLAs) doesn't get created until in the
33// main function body, after the prologue is run. However, it's depicted here
34// for completeness.
35//
36// | | Higher address
37// |-----------------------------------|
38// | |
39// | arguments passed on the stack |
40// | |
41// |-----------------------------------|
42// | |
43// | (Win64 only) varargs from reg |
44// | |
45// |-----------------------------------|
46// | |
47// | callee-saved gpr registers | <--.
48// | | | On Darwin platforms these
49// |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
50// | prev_lr | | (frame record first)
51// | prev_fp | <--'
52// | async context if needed |
53// | (a.k.a. "frame record") |
54// |-----------------------------------| <- fp(=x29)
55// | |
56// | callee-saved fp/simd/SVE regs |
57// | |
58// |-----------------------------------|
59// | |
60// | SVE stack objects |
61// | |
62// |-----------------------------------|
63// |.empty.space.to.make.part.below....|
64// |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
65// |.the.standard.16-byte.alignment....| compile time; if present)
66// |-----------------------------------|
67// | |
68// | local variables of fixed size |
69// | including spill slots |
70// |-----------------------------------| <- bp(not defined by ABI,
71// |.variable-sized.local.variables....| LLVM chooses X19)
72// |.(VLAs)............................| (size of this area is unknown at
73// |...................................| compile time)
74// |-----------------------------------| <- sp
75// | | Lower address
76//
77//
78// To access the data in a frame, at-compile time, a constant offset must be
79// computable from one of the pointers (fp, bp, sp) to access it. The size
80// of the areas with a dotted background cannot be computed at compile-time
81// if they are present, making it required to have all three of fp, bp and
82// sp to be set up to be able to access all contents in the frame areas,
83// assuming all of the frame areas are non-empty.
84//
85// For most functions, some of the frame areas are empty. For those functions,
86// it may not be necessary to set up fp or bp:
87// * A base pointer is definitely needed when there are both VLAs and local
88// variables with more-than-default alignment requirements.
89// * A frame pointer is definitely needed when there are local variables with
90// more-than-default alignment requirements.
91//
92// For Darwin platforms the frame-record (fp, lr) is stored at the top of the
93// callee-saved area, since the unwind encoding does not allow for encoding
94// this dynamically and existing tools depend on this layout. For other
95// platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
96// area to allow SVE stack objects (allocated directly below the callee-saves,
97// if available) to be accessed directly from the framepointer.
98// The SVE spill/fill instructions have VL-scaled addressing modes such
99// as:
100// ldr z8, [fp, #-7 mul vl]
101// For SVE the size of the vector length (VL) is not known at compile-time, so
102// '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
103// layout, we don't need to add an unscaled offset to the framepointer before
104// accessing the SVE object in the frame.
105//
106// In some cases when a base pointer is not strictly needed, it is generated
107// anyway when offsets from the frame pointer to access local variables become
108// so large that the offset can't be encoded in the immediate fields of loads
109// or stores.
110//
111// Outgoing function arguments must be at the bottom of the stack frame when
112// calling another function. If we do not have variable-sized stack objects, we
113// can allocate a "reserved call frame" area at the bottom of the local
114// variable area, large enough for all outgoing calls. If we do have VLAs, then
115// the stack pointer must be decremented and incremented around each call to
116// make space for the arguments below the VLAs.
117//
118// FIXME: also explain the redzone concept.
119//
120// An example of the prologue:
121//
122// .globl __foo
123// .align 2
124// __foo:
125// Ltmp0:
126// .cfi_startproc
127// .cfi_personality 155, ___gxx_personality_v0
128// Leh_func_begin:
129// .cfi_lsda 16, Lexception33
130//
131// stp xa,bx, [sp, -#offset]!
132// ...
133// stp x28, x27, [sp, #offset-32]
134// stp fp, lr, [sp, #offset-16]
135// add fp, sp, #offset - 16
136// sub sp, sp, #1360
137//
138// The Stack:
139// +-------------------------------------------+
140// 10000 | ........ | ........ | ........ | ........ |
141// 10004 | ........ | ........ | ........ | ........ |
142// +-------------------------------------------+
143// 10008 | ........ | ........ | ........ | ........ |
144// 1000c | ........ | ........ | ........ | ........ |
145// +===========================================+
146// 10010 | X28 Register |
147// 10014 | X28 Register |
148// +-------------------------------------------+
149// 10018 | X27 Register |
150// 1001c | X27 Register |
151// +===========================================+
152// 10020 | Frame Pointer |
153// 10024 | Frame Pointer |
154// +-------------------------------------------+
155// 10028 | Link Register |
156// 1002c | Link Register |
157// +===========================================+
158// 10030 | ........ | ........ | ........ | ........ |
159// 10034 | ........ | ........ | ........ | ........ |
160// +-------------------------------------------+
161// 10038 | ........ | ........ | ........ | ........ |
162// 1003c | ........ | ........ | ........ | ........ |
163// +-------------------------------------------+
164//
165// [sp] = 10030 :: >>initial value<<
166// sp = 10020 :: stp fp, lr, [sp, #-16]!
167// fp = sp == 10020 :: mov fp, sp
168// [sp] == 10020 :: stp x28, x27, [sp, #-16]!
169// sp == 10010 :: >>final value<<
170//
171// The frame pointer (w29) points to address 10020. If we use an offset of
172// '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
173// for w27, and -32 for w28:
174//
175// Ltmp1:
176// .cfi_def_cfa w29, 16
177// Ltmp2:
178// .cfi_offset w30, -8
179// Ltmp3:
180// .cfi_offset w29, -16
181// Ltmp4:
182// .cfi_offset w27, -24
183// Ltmp5:
184// .cfi_offset w28, -32
185//
186//===----------------------------------------------------------------------===//
187
188#include "AArch64FrameLowering.h"
189#include "AArch64InstrInfo.h"
191#include "AArch64RegisterInfo.h"
192#include "AArch64Subtarget.h"
193#include "AArch64TargetMachine.h"
196#include "llvm/ADT/ScopeExit.h"
197#include "llvm/ADT/SmallVector.h"
198#include "llvm/ADT/Statistic.h"
214#include "llvm/IR/Attributes.h"
215#include "llvm/IR/CallingConv.h"
216#include "llvm/IR/DataLayout.h"
217#include "llvm/IR/DebugLoc.h"
218#include "llvm/IR/Function.h"
219#include "llvm/MC/MCAsmInfo.h"
220#include "llvm/MC/MCDwarf.h"
222#include "llvm/Support/Debug.h"
228#include <cassert>
229#include <cstdint>
230#include <iterator>
231#include <optional>
232#include <vector>
233
234using namespace llvm;
235
236#define DEBUG_TYPE "frame-info"
237
238static cl::opt<bool> EnableRedZone("aarch64-redzone",
239 cl::desc("enable use of redzone on AArch64"),
240 cl::init(false), cl::Hidden);
241
243 "stack-tagging-merge-settag",
244 cl::desc("merge settag instruction in function epilog"), cl::init(true),
245 cl::Hidden);
246
247static cl::opt<bool> OrderFrameObjects("aarch64-order-frame-objects",
248 cl::desc("sort stack allocations"),
249 cl::init(true), cl::Hidden);
250
252 "homogeneous-prolog-epilog", cl::Hidden,
253 cl::desc("Emit homogeneous prologue and epilogue for the size "
254 "optimization (default = off)"));
255
256STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
257
258/// Returns how much of the incoming argument stack area (in bytes) we should
259/// clean up in an epilogue. For the C calling convention this will be 0, for
260/// guaranteed tail call conventions it can be positive (a normal return or a
261/// tail call to a function that uses less stack space for arguments) or
262/// negative (for a tail call to a function that needs more stack space than us
263/// for arguments).
268 bool IsTailCallReturn = (MBB.end() != MBBI)
270 : false;
271
272 int64_t ArgumentPopSize = 0;
273 if (IsTailCallReturn) {
274 MachineOperand &StackAdjust = MBBI->getOperand(1);
275
276 // For a tail-call in a callee-pops-arguments environment, some or all of
277 // the stack may actually be in use for the call's arguments, this is
278 // calculated during LowerCall and consumed here...
279 ArgumentPopSize = StackAdjust.getImm();
280 } else {
281 // ... otherwise the amount to pop is *all* of the argument space,
282 // conveniently stored in the MachineFunctionInfo by
283 // LowerFormalArguments. This will, of course, be zero for the C calling
284 // convention.
285 ArgumentPopSize = AFI->getArgumentStackToRestore();
286 }
287
288 return ArgumentPopSize;
289}
290
292static bool needsWinCFI(const MachineFunction &MF);
295
296/// Returns true if a homogeneous prolog or epilog code can be emitted
297/// for the size optimization. If possible, a frame helper call is injected.
298/// When Exit block is given, this check is for epilog.
299bool AArch64FrameLowering::homogeneousPrologEpilog(
300 MachineFunction &MF, MachineBasicBlock *Exit) const {
301 if (!MF.getFunction().hasMinSize())
302 return false;
304 return false;
305 if (EnableRedZone)
306 return false;
307
308 // TODO: Window is supported yet.
309 if (needsWinCFI(MF))
310 return false;
311 // TODO: SVE is not supported yet.
312 if (getSVEStackSize(MF))
313 return false;
314
315 // Bail on stack adjustment needed on return for simplicity.
316 const MachineFrameInfo &MFI = MF.getFrameInfo();
318 if (MFI.hasVarSizedObjects() || RegInfo->hasStackRealignment(MF))
319 return false;
320 if (Exit && getArgumentStackToRestore(MF, *Exit))
321 return false;
322
323 auto *AFI = MF.getInfo<AArch64FunctionInfo>();
324 if (AFI->hasSwiftAsyncContext() || AFI->hasStreamingModeChanges())
325 return false;
326
327 // If there are an odd number of GPRs before LR and FP in the CSRs list,
328 // they will not be paired into one RegPairInfo, which is incompatible with
329 // the assumption made by the homogeneous prolog epilog pass.
330 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
331 unsigned NumGPRs = 0;
332 for (unsigned I = 0; CSRegs[I]; ++I) {
333 Register Reg = CSRegs[I];
334 if (Reg == AArch64::LR) {
335 assert(CSRegs[I + 1] == AArch64::FP);
336 if (NumGPRs % 2 != 0)
337 return false;
338 break;
339 }
340 if (AArch64::GPR64RegClass.contains(Reg))
341 ++NumGPRs;
342 }
343
344 return true;
345}
346
347/// Returns true if CSRs should be paired.
348bool AArch64FrameLowering::producePairRegisters(MachineFunction &MF) const {
349 return produceCompactUnwindFrame(MF) || homogeneousPrologEpilog(MF);
350}
351
352/// This is the biggest offset to the stack pointer we can encode in aarch64
353/// instructions (without using a separate calculation and a temp register).
354/// Note that the exception here are vector stores/loads which cannot encode any
355/// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
356static const unsigned DefaultSafeSPDisplacement = 255;
357
358/// Look at each instruction that references stack frames and return the stack
359/// size limit beyond which some of these instructions will require a scratch
360/// register during their expansion later.
362 // FIXME: For now, just conservatively guestimate based on unscaled indexing
363 // range. We'll end up allocating an unnecessary spill slot a lot, but
364 // realistically that's not a big deal at this stage of the game.
365 for (MachineBasicBlock &MBB : MF) {
366 for (MachineInstr &MI : MBB) {
367 if (MI.isDebugInstr() || MI.isPseudo() ||
368 MI.getOpcode() == AArch64::ADDXri ||
369 MI.getOpcode() == AArch64::ADDSXri)
370 continue;
371
372 for (const MachineOperand &MO : MI.operands()) {
373 if (!MO.isFI())
374 continue;
375
377 if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
379 return 0;
380 }
381 }
382 }
384}
385
389}
390
391/// Returns the size of the fixed object area (allocated next to sp on entry)
392/// On Win64 this may include a var args area and an UnwindHelp object for EH.
393static unsigned getFixedObjectSize(const MachineFunction &MF,
394 const AArch64FunctionInfo *AFI, bool IsWin64,
395 bool IsFunclet) {
396 if (!IsWin64 || IsFunclet) {
397 return AFI->getTailCallReservedStack();
398 } else {
399 if (AFI->getTailCallReservedStack() != 0 &&
401 Attribute::SwiftAsync))
402 report_fatal_error("cannot generate ABI-changing tail call for Win64");
403 // Var args are stored here in the primary function.
404 const unsigned VarArgsArea = AFI->getVarArgsGPRSize();
405 // To support EH funclets we allocate an UnwindHelp object
406 const unsigned UnwindHelpObject = (MF.hasEHFunclets() ? 8 : 0);
407 return AFI->getTailCallReservedStack() +
408 alignTo(VarArgsArea + UnwindHelpObject, 16);
409 }
410}
411
412/// Returns the size of the entire SVE stackframe (calleesaves + spills).
415 return StackOffset::getScalable((int64_t)AFI->getStackSizeSVE());
416}
417
419 if (!EnableRedZone)
420 return false;
421
422 // Don't use the red zone if the function explicitly asks us not to.
423 // This is typically used for kernel code.
424 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
425 const unsigned RedZoneSize =
427 if (!RedZoneSize)
428 return false;
429
430 const MachineFrameInfo &MFI = MF.getFrameInfo();
432 uint64_t NumBytes = AFI->getLocalStackSize();
433
434 // If neither NEON or SVE are available, a COPY from one Q-reg to
435 // another requires a spill -> reload sequence. We can do that
436 // using a pre-decrementing store/post-decrementing load, but
437 // if we do so, we can't use the Red Zone.
438 bool LowerQRegCopyThroughMem = Subtarget.hasFPARMv8() &&
439 !Subtarget.isNeonAvailable() &&
440 !Subtarget.hasSVE();
441
442 return !(MFI.hasCalls() || hasFP(MF) || NumBytes > RedZoneSize ||
443 getSVEStackSize(MF) || LowerQRegCopyThroughMem);
444}
445
446/// hasFP - Return true if the specified function should have a dedicated frame
447/// pointer register.
449 const MachineFrameInfo &MFI = MF.getFrameInfo();
450 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
451
452 // Win64 EH requires a frame pointer if funclets are present, as the locals
453 // are accessed off the frame pointer in both the parent function and the
454 // funclets.
455 if (MF.hasEHFunclets())
456 return true;
457 // Retain behavior of always omitting the FP for leaf functions when possible.
459 return true;
460 if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
461 MFI.hasStackMap() || MFI.hasPatchPoint() ||
462 RegInfo->hasStackRealignment(MF))
463 return true;
464 // With large callframes around we may need to use FP to access the scavenging
465 // emergency spillslot.
466 //
467 // Unfortunately some calls to hasFP() like machine verifier ->
468 // getReservedReg() -> hasFP in the middle of global isel are too early
469 // to know the max call frame size. Hopefully conservatively returning "true"
470 // in those cases is fine.
471 // DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
472 if (!MFI.isMaxCallFrameSizeComputed() ||
474 return true;
475
476 return false;
477}
478
479/// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
480/// not required, we reserve argument space for call sites in the function
481/// immediately on entry to the current function. This eliminates the need for
482/// add/sub sp brackets around call sites. Returns true if the call frame is
483/// included as part of the stack frame.
484bool
486 // The stack probing code for the dynamically allocated outgoing arguments
487 // area assumes that the stack is probed at the top - either by the prologue
488 // code, which issues a probe if `hasVarSizedObjects` return true, or by the
489 // most recent variable-sized object allocation. Changing the condition here
490 // may need to be followed up by changes to the probe issuing logic.
491 return !MF.getFrameInfo().hasVarSizedObjects();
492}
493
497 const AArch64InstrInfo *TII =
498 static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
499 const AArch64TargetLowering *TLI =
500 MF.getSubtarget<AArch64Subtarget>().getTargetLowering();
501 [[maybe_unused]] MachineFrameInfo &MFI = MF.getFrameInfo();
502 DebugLoc DL = I->getDebugLoc();
503 unsigned Opc = I->getOpcode();
504 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
505 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
506
507 if (!hasReservedCallFrame(MF)) {
508 int64_t Amount = I->getOperand(0).getImm();
509 Amount = alignTo(Amount, getStackAlign());
510 if (!IsDestroy)
511 Amount = -Amount;
512
513 // N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
514 // doesn't have to pop anything), then the first operand will be zero too so
515 // this adjustment is a no-op.
516 if (CalleePopAmount == 0) {
517 // FIXME: in-function stack adjustment for calls is limited to 24-bits
518 // because there's no guaranteed temporary register available.
519 //
520 // ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
521 // 1) For offset <= 12-bit, we use LSL #0
522 // 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
523 // LSL #0, and the other uses LSL #12.
524 //
525 // Most call frames will be allocated at the start of a function so
526 // this is OK, but it is a limitation that needs dealing with.
527 assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
528
529 if (TLI->hasInlineStackProbe(MF) &&
531 // When stack probing is enabled, the decrement of SP may need to be
532 // probed. We only need to do this if the call site needs 1024 bytes of
533 // space or more, because a region smaller than that is allowed to be
534 // unprobed at an ABI boundary. We rely on the fact that SP has been
535 // probed exactly at this point, either by the prologue or most recent
536 // dynamic allocation.
538 "non-reserved call frame without var sized objects?");
539 Register ScratchReg =
540 MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
541 inlineStackProbeFixed(I, ScratchReg, -Amount, StackOffset::get(0, 0));
542 } else {
543 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
544 StackOffset::getFixed(Amount), TII);
545 }
546 }
547 } else if (CalleePopAmount != 0) {
548 // If the calling convention demands that the callee pops arguments from the
549 // stack, we want to add it back if we have a reserved call frame.
550 assert(CalleePopAmount < 0xffffff && "call frame too large");
551 emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
552 StackOffset::getFixed(-(int64_t)CalleePopAmount), TII);
553 }
554 return MBB.erase(I);
555}
556
557void AArch64FrameLowering::emitCalleeSavedGPRLocations(
560 MachineFrameInfo &MFI = MF.getFrameInfo();
562 SMEAttrs Attrs(MF.getFunction());
563 bool LocallyStreaming =
564 Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface();
565
566 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
567 if (CSI.empty())
568 return;
569
570 const TargetSubtargetInfo &STI = MF.getSubtarget();
571 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
572 const TargetInstrInfo &TII = *STI.getInstrInfo();
574
575 for (const auto &Info : CSI) {
576 unsigned FrameIdx = Info.getFrameIdx();
577 if (MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector)
578 continue;
579
580 assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
581 int64_t DwarfReg = TRI.getDwarfRegNum(Info.getReg(), true);
582 int64_t Offset = MFI.getObjectOffset(FrameIdx) - getOffsetOfLocalArea();
583
584 // The location of VG will be emitted before each streaming-mode change in
585 // the function. Only locally-streaming functions require emitting the
586 // non-streaming VG location here.
587 if ((LocallyStreaming && FrameIdx == AFI->getStreamingVGIdx()) ||
588 (!LocallyStreaming &&
589 DwarfReg == TRI.getDwarfRegNum(AArch64::VG, true)))
590 continue;
591
592 unsigned CFIIndex = MF.addFrameInst(
593 MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
594 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
595 .addCFIIndex(CFIIndex)
597 }
598}
599
600void AArch64FrameLowering::emitCalleeSavedSVELocations(
603 MachineFrameInfo &MFI = MF.getFrameInfo();
604
605 // Add callee saved registers to move list.
606 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
607 if (CSI.empty())
608 return;
609
610 const TargetSubtargetInfo &STI = MF.getSubtarget();
611 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
612 const TargetInstrInfo &TII = *STI.getInstrInfo();
615
616 for (const auto &Info : CSI) {
617 if (!(MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
618 continue;
619
620 // Not all unwinders may know about SVE registers, so assume the lowest
621 // common demoninator.
622 assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
623 unsigned Reg = Info.getReg();
624 if (!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
625 continue;
626
628 StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) -
630
631 unsigned CFIIndex = MF.addFrameInst(createCFAOffset(TRI, Reg, Offset));
632 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
633 .addCFIIndex(CFIIndex)
635 }
636}
637
641 unsigned DwarfReg) {
642 unsigned CFIIndex =
643 MF.addFrameInst(MCCFIInstruction::createSameValue(nullptr, DwarfReg));
644 BuildMI(MBB, InsertPt, DebugLoc(), Desc).addCFIIndex(CFIIndex);
645}
646
648 MachineBasicBlock &MBB) const {
649
651 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
652 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
653 const auto &TRI =
654 static_cast<const AArch64RegisterInfo &>(*Subtarget.getRegisterInfo());
655 const auto &MFI = *MF.getInfo<AArch64FunctionInfo>();
656
657 const MCInstrDesc &CFIDesc = TII.get(TargetOpcode::CFI_INSTRUCTION);
658 DebugLoc DL;
659
660 // Reset the CFA to `SP + 0`.
662 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
663 nullptr, TRI.getDwarfRegNum(AArch64::SP, true), 0));
664 BuildMI(MBB, InsertPt, DL, CFIDesc).addCFIIndex(CFIIndex);
665
666 // Flip the RA sign state.
667 if (MFI.shouldSignReturnAddress(MF)) {
669 BuildMI(MBB, InsertPt, DL, CFIDesc).addCFIIndex(CFIIndex);
670 }
671
672 // Shadow call stack uses X18, reset it.
673 if (MFI.needsShadowCallStackPrologueEpilogue(MF))
674 insertCFISameValue(CFIDesc, MF, MBB, InsertPt,
675 TRI.getDwarfRegNum(AArch64::X18, true));
676
677 // Emit .cfi_same_value for callee-saved registers.
678 const std::vector<CalleeSavedInfo> &CSI =
680 for (const auto &Info : CSI) {
681 unsigned Reg = Info.getReg();
682 if (!TRI.regNeedsCFI(Reg, Reg))
683 continue;
684 insertCFISameValue(CFIDesc, MF, MBB, InsertPt,
685 TRI.getDwarfRegNum(Reg, true));
686 }
687}
688
691 bool SVE) {
693 MachineFrameInfo &MFI = MF.getFrameInfo();
694
695 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
696 if (CSI.empty())
697 return;
698
699 const TargetSubtargetInfo &STI = MF.getSubtarget();
700 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
701 const TargetInstrInfo &TII = *STI.getInstrInfo();
703
704 for (const auto &Info : CSI) {
705 if (SVE !=
706 (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
707 continue;
708
709 unsigned Reg = Info.getReg();
710 if (SVE &&
711 !static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
712 continue;
713
714 if (!Info.isRestored())
715 continue;
716
717 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore(
718 nullptr, TRI.getDwarfRegNum(Info.getReg(), true)));
719 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
720 .addCFIIndex(CFIIndex)
722 }
723}
724
725void AArch64FrameLowering::emitCalleeSavedGPRRestores(
728}
729
730void AArch64FrameLowering::emitCalleeSavedSVERestores(
733}
734
735// Return the maximum possible number of bytes for `Size` due to the
736// architectural limit on the size of a SVE register.
737static int64_t upperBound(StackOffset Size) {
738 static const int64_t MAX_BYTES_PER_SCALABLE_BYTE = 16;
739 return Size.getScalable() * MAX_BYTES_PER_SCALABLE_BYTE + Size.getFixed();
740}
741
742void AArch64FrameLowering::allocateStackSpace(
744 int64_t RealignmentPadding, StackOffset AllocSize, bool NeedsWinCFI,
745 bool *HasWinCFI, bool EmitCFI, StackOffset InitialOffset,
746 bool FollowupAllocs) const {
747
748 if (!AllocSize)
749 return;
750
751 DebugLoc DL;
753 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
754 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
756 const MachineFrameInfo &MFI = MF.getFrameInfo();
757
758 const int64_t MaxAlign = MFI.getMaxAlign().value();
759 const uint64_t AndMask = ~(MaxAlign - 1);
760
761 if (!Subtarget.getTargetLowering()->hasInlineStackProbe(MF)) {
762 Register TargetReg = RealignmentPadding
764 : AArch64::SP;
765 // SUB Xd/SP, SP, AllocSize
766 emitFrameOffset(MBB, MBBI, DL, TargetReg, AArch64::SP, -AllocSize, &TII,
767 MachineInstr::FrameSetup, false, NeedsWinCFI, HasWinCFI,
768 EmitCFI, InitialOffset);
769
770 if (RealignmentPadding) {
771 // AND SP, X9, 0b11111...0000
772 BuildMI(MBB, MBBI, DL, TII.get(AArch64::ANDXri), AArch64::SP)
773 .addReg(TargetReg, RegState::Kill)
776 AFI.setStackRealigned(true);
777
778 // No need for SEH instructions here; if we're realigning the stack,
779 // we've set a frame pointer and already finished the SEH prologue.
780 assert(!NeedsWinCFI);
781 }
782 return;
783 }
784
785 //
786 // Stack probing allocation.
787 //
788
789 // Fixed length allocation. If we don't need to re-align the stack and don't
790 // have SVE objects, we can use a more efficient sequence for stack probing.
791 if (AllocSize.getScalable() == 0 && RealignmentPadding == 0) {
793 assert(ScratchReg != AArch64::NoRegister);
794 BuildMI(MBB, MBBI, DL, TII.get(AArch64::PROBED_STACKALLOC))
795 .addDef(ScratchReg)
796 .addImm(AllocSize.getFixed())
797 .addImm(InitialOffset.getFixed())
798 .addImm(InitialOffset.getScalable());
799 // The fixed allocation may leave unprobed bytes at the top of the
800 // stack. If we have subsequent alocation (e.g. if we have variable-sized
801 // objects), we need to issue an extra probe, so these allocations start in
802 // a known state.
803 if (FollowupAllocs) {
804 // STR XZR, [SP]
805 BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXui))
806 .addReg(AArch64::XZR)
807 .addReg(AArch64::SP)
808 .addImm(0)
810 }
811
812 return;
813 }
814
815 // Variable length allocation.
816
817 // If the (unknown) allocation size cannot exceed the probe size, decrement
818 // the stack pointer right away.
819 int64_t ProbeSize = AFI.getStackProbeSize();
820 if (upperBound(AllocSize) + RealignmentPadding <= ProbeSize) {
821 Register ScratchReg = RealignmentPadding
823 : AArch64::SP;
824 assert(ScratchReg != AArch64::NoRegister);
825 // SUB Xd, SP, AllocSize
826 emitFrameOffset(MBB, MBBI, DL, ScratchReg, AArch64::SP, -AllocSize, &TII,
827 MachineInstr::FrameSetup, false, NeedsWinCFI, HasWinCFI,
828 EmitCFI, InitialOffset);
829 if (RealignmentPadding) {
830 // AND SP, Xn, 0b11111...0000
831 BuildMI(MBB, MBBI, DL, TII.get(AArch64::ANDXri), AArch64::SP)
832 .addReg(ScratchReg, RegState::Kill)
835 AFI.setStackRealigned(true);
836 }
837 if (FollowupAllocs || upperBound(AllocSize) + RealignmentPadding >
839 // STR XZR, [SP]
840 BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXui))
841 .addReg(AArch64::XZR)
842 .addReg(AArch64::SP)
843 .addImm(0)
845 }
846 return;
847 }
848
849 // Emit a variable-length allocation probing loop.
850 // TODO: As an optimisation, the loop can be "unrolled" into a few parts,
851 // each of them guaranteed to adjust the stack by less than the probe size.
853 assert(TargetReg != AArch64::NoRegister);
854 // SUB Xd, SP, AllocSize
855 emitFrameOffset(MBB, MBBI, DL, TargetReg, AArch64::SP, -AllocSize, &TII,
856 MachineInstr::FrameSetup, false, NeedsWinCFI, HasWinCFI,
857 EmitCFI, InitialOffset);
858 if (RealignmentPadding) {
859 // AND Xn, Xn, 0b11111...0000
860 BuildMI(MBB, MBBI, DL, TII.get(AArch64::ANDXri), TargetReg)
861 .addReg(TargetReg, RegState::Kill)
864 }
865
866 BuildMI(MBB, MBBI, DL, TII.get(AArch64::PROBED_STACKALLOC_VAR))
867 .addReg(TargetReg);
868 if (EmitCFI) {
869 // Set the CFA register back to SP.
870 unsigned Reg =
871 Subtarget.getRegisterInfo()->getDwarfRegNum(AArch64::SP, true);
872 unsigned CFIIndex =
874 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
875 .addCFIIndex(CFIIndex)
877 }
878 if (RealignmentPadding)
879 AFI.setStackRealigned(true);
880}
881
882static MCRegister getRegisterOrZero(MCRegister Reg, bool HasSVE) {
883 switch (Reg.id()) {
884 default:
885 // The called routine is expected to preserve r19-r28
886 // r29 and r30 are used as frame pointer and link register resp.
887 return 0;
888
889 // GPRs
890#define CASE(n) \
891 case AArch64::W##n: \
892 case AArch64::X##n: \
893 return AArch64::X##n
894 CASE(0);
895 CASE(1);
896 CASE(2);
897 CASE(3);
898 CASE(4);
899 CASE(5);
900 CASE(6);
901 CASE(7);
902 CASE(8);
903 CASE(9);
904 CASE(10);
905 CASE(11);
906 CASE(12);
907 CASE(13);
908 CASE(14);
909 CASE(15);
910 CASE(16);
911 CASE(17);
912 CASE(18);
913#undef CASE
914
915 // FPRs
916#define CASE(n) \
917 case AArch64::B##n: \
918 case AArch64::H##n: \
919 case AArch64::S##n: \
920 case AArch64::D##n: \
921 case AArch64::Q##n: \
922 return HasSVE ? AArch64::Z##n : AArch64::Q##n
923 CASE(0);
924 CASE(1);
925 CASE(2);
926 CASE(3);
927 CASE(4);
928 CASE(5);
929 CASE(6);
930 CASE(7);
931 CASE(8);
932 CASE(9);
933 CASE(10);
934 CASE(11);
935 CASE(12);
936 CASE(13);
937 CASE(14);
938 CASE(15);
939 CASE(16);
940 CASE(17);
941 CASE(18);
942 CASE(19);
943 CASE(20);
944 CASE(21);
945 CASE(22);
946 CASE(23);
947 CASE(24);
948 CASE(25);
949 CASE(26);
950 CASE(27);
951 CASE(28);
952 CASE(29);
953 CASE(30);
954 CASE(31);
955#undef CASE
956 }
957}
958
959void AArch64FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
960 MachineBasicBlock &MBB) const {
961 // Insertion point.
963
964 // Fake a debug loc.
965 DebugLoc DL;
966 if (MBBI != MBB.end())
967 DL = MBBI->getDebugLoc();
968
969 const MachineFunction &MF = *MBB.getParent();
972
973 BitVector GPRsToZero(TRI.getNumRegs());
974 BitVector FPRsToZero(TRI.getNumRegs());
975 bool HasSVE = STI.hasSVE();
976 for (MCRegister Reg : RegsToZero.set_bits()) {
977 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
978 // For GPRs, we only care to clear out the 64-bit register.
979 if (MCRegister XReg = getRegisterOrZero(Reg, HasSVE))
980 GPRsToZero.set(XReg);
981 } else if (AArch64::FPR128RegClass.contains(Reg) ||
982 AArch64::FPR64RegClass.contains(Reg) ||
983 AArch64::FPR32RegClass.contains(Reg) ||
984 AArch64::FPR16RegClass.contains(Reg) ||
985 AArch64::FPR8RegClass.contains(Reg)) {
986 // For FPRs,
987 if (MCRegister XReg = getRegisterOrZero(Reg, HasSVE))
988 FPRsToZero.set(XReg);
989 }
990 }
991
992 const AArch64InstrInfo &TII = *STI.getInstrInfo();
993
994 // Zero out GPRs.
995 for (MCRegister Reg : GPRsToZero.set_bits())
996 TII.buildClearRegister(Reg, MBB, MBBI, DL);
997
998 // Zero out FP/vector registers.
999 for (MCRegister Reg : FPRsToZero.set_bits())
1000 TII.buildClearRegister(Reg, MBB, MBBI, DL);
1001
1002 if (HasSVE) {
1003 for (MCRegister PReg :
1004 {AArch64::P0, AArch64::P1, AArch64::P2, AArch64::P3, AArch64::P4,
1005 AArch64::P5, AArch64::P6, AArch64::P7, AArch64::P8, AArch64::P9,
1006 AArch64::P10, AArch64::P11, AArch64::P12, AArch64::P13, AArch64::P14,
1007 AArch64::P15}) {
1008 if (RegsToZero[PReg])
1009 BuildMI(MBB, MBBI, DL, TII.get(AArch64::PFALSE), PReg);
1010 }
1011 }
1012}
1013
1015 const MachineBasicBlock &MBB) {
1016 const MachineFunction *MF = MBB.getParent();
1017 LiveRegs.addLiveIns(MBB);
1018 // Mark callee saved registers as used so we will not choose them.
1019 const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
1020 for (unsigned i = 0; CSRegs[i]; ++i)
1021 LiveRegs.addReg(CSRegs[i]);
1022}
1023
1024// Find a scratch register that we can use at the start of the prologue to
1025// re-align the stack pointer. We avoid using callee-save registers since they
1026// may appear to be free when this is called from canUseAsPrologue (during
1027// shrink wrapping), but then no longer be free when this is called from
1028// emitPrologue.
1029//
1030// FIXME: This is a bit conservative, since in the above case we could use one
1031// of the callee-save registers as a scratch temp to re-align the stack pointer,
1032// but we would then have to make sure that we were in fact saving at least one
1033// callee-save register in the prologue, which is additional complexity that
1034// doesn't seem worth the benefit.
1036 MachineFunction *MF = MBB->getParent();
1037
1038 // If MBB is an entry block, use X9 as the scratch register
1039 if (&MF->front() == MBB)
1040 return AArch64::X9;
1041
1042 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
1043 const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
1044 LivePhysRegs LiveRegs(TRI);
1045 getLiveRegsForEntryMBB(LiveRegs, *MBB);
1046
1047 // Prefer X9 since it was historically used for the prologue scratch reg.
1048 const MachineRegisterInfo &MRI = MF->getRegInfo();
1049 if (LiveRegs.available(MRI, AArch64::X9))
1050 return AArch64::X9;
1051
1052 for (unsigned Reg : AArch64::GPR64RegClass) {
1053 if (LiveRegs.available(MRI, Reg))
1054 return Reg;
1055 }
1056 return AArch64::NoRegister;
1057}
1058
1060 const MachineBasicBlock &MBB) const {
1061 const MachineFunction *MF = MBB.getParent();
1062 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
1063 const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
1064 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1065 const AArch64TargetLowering *TLI = Subtarget.getTargetLowering();
1067
1068 if (AFI->hasSwiftAsyncContext()) {
1069 const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
1070 const MachineRegisterInfo &MRI = MF->getRegInfo();
1071 LivePhysRegs LiveRegs(TRI);
1072 getLiveRegsForEntryMBB(LiveRegs, MBB);
1073 // The StoreSwiftAsyncContext clobbers X16 and X17. Make sure they are
1074 // available.
1075 if (!LiveRegs.available(MRI, AArch64::X16) ||
1076 !LiveRegs.available(MRI, AArch64::X17))
1077 return false;
1078 }
1079
1080 // Certain stack probing sequences might clobber flags, then we can't use
1081 // the block as a prologue if the flags register is a live-in.
1083 MBB.isLiveIn(AArch64::NZCV))
1084 return false;
1085
1086 // Don't need a scratch register if we're not going to re-align the stack or
1087 // emit stack probes.
1088 if (!RegInfo->hasStackRealignment(*MF) && !TLI->hasInlineStackProbe(*MF))
1089 return true;
1090 // Otherwise, we can use any block as long as it has a scratch register
1091 // available.
1092 return findScratchNonCalleeSaveRegister(TmpMBB) != AArch64::NoRegister;
1093}
1094
1096 uint64_t StackSizeInBytes) {
1097 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1099 // TODO: When implementing stack protectors, take that into account
1100 // for the probe threshold.
1101 return Subtarget.isTargetWindows() && MFI.hasStackProbing() &&
1102 StackSizeInBytes >= uint64_t(MFI.getStackProbeSize());
1103}
1104
1105static bool needsWinCFI(const MachineFunction &MF) {
1106 const Function &F = MF.getFunction();
1107 return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
1108 F.needsUnwindTableEntry();
1109}
1110
1111bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
1112 MachineFunction &MF, uint64_t StackBumpBytes) const {
1114 const MachineFrameInfo &MFI = MF.getFrameInfo();
1115 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1116 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1117 if (homogeneousPrologEpilog(MF))
1118 return false;
1119
1120 if (AFI->getLocalStackSize() == 0)
1121 return false;
1122
1123 // For WinCFI, if optimizing for size, prefer to not combine the stack bump
1124 // (to force a stp with predecrement) to match the packed unwind format,
1125 // provided that there actually are any callee saved registers to merge the
1126 // decrement with.
1127 // This is potentially marginally slower, but allows using the packed
1128 // unwind format for functions that both have a local area and callee saved
1129 // registers. Using the packed unwind format notably reduces the size of
1130 // the unwind info.
1131 if (needsWinCFI(MF) && AFI->getCalleeSavedStackSize() > 0 &&
1132 MF.getFunction().hasOptSize())
1133 return false;
1134
1135 // 512 is the maximum immediate for stp/ldp that will be used for
1136 // callee-save save/restores
1137 if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
1138 return false;
1139
1140 if (MFI.hasVarSizedObjects())
1141 return false;
1142
1143 if (RegInfo->hasStackRealignment(MF))
1144 return false;
1145
1146 // This isn't strictly necessary, but it simplifies things a bit since the
1147 // current RedZone handling code assumes the SP is adjusted by the
1148 // callee-save save/restore code.
1149 if (canUseRedZone(MF))
1150 return false;
1151
1152 // When there is an SVE area on the stack, always allocate the
1153 // callee-saves and spills/locals separately.
1154 if (getSVEStackSize(MF))
1155 return false;
1156
1157 return true;
1158}
1159
1160bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
1161 MachineBasicBlock &MBB, unsigned StackBumpBytes) const {
1162 if (!shouldCombineCSRLocalStackBump(*MBB.getParent(), StackBumpBytes))
1163 return false;
1164
1165 if (MBB.empty())
1166 return true;
1167
1168 // Disable combined SP bump if the last instruction is an MTE tag store. It
1169 // is almost always better to merge SP adjustment into those instructions.
1172 while (LastI != Begin) {
1173 --LastI;
1174 if (LastI->isTransient())
1175 continue;
1176 if (!LastI->getFlag(MachineInstr::FrameDestroy))
1177 break;
1178 }
1179 switch (LastI->getOpcode()) {
1180 case AArch64::STGloop:
1181 case AArch64::STZGloop:
1182 case AArch64::STGi:
1183 case AArch64::STZGi:
1184 case AArch64::ST2Gi:
1185 case AArch64::STZ2Gi:
1186 return false;
1187 default:
1188 return true;
1189 }
1190 llvm_unreachable("unreachable");
1191}
1192
1193// Given a load or a store instruction, generate an appropriate unwinding SEH
1194// code on Windows.
1196 const TargetInstrInfo &TII,
1197 MachineInstr::MIFlag Flag) {
1198 unsigned Opc = MBBI->getOpcode();
1200 MachineFunction &MF = *MBB->getParent();
1201 DebugLoc DL = MBBI->getDebugLoc();
1202 unsigned ImmIdx = MBBI->getNumOperands() - 1;
1203 int Imm = MBBI->getOperand(ImmIdx).getImm();
1205 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1206 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1207
1208 switch (Opc) {
1209 default:
1210 llvm_unreachable("No SEH Opcode for this instruction");
1211 case AArch64::LDPDpost:
1212 Imm = -Imm;
1213 [[fallthrough]];
1214 case AArch64::STPDpre: {
1215 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
1216 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
1217 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
1218 .addImm(Reg0)
1219 .addImm(Reg1)
1220 .addImm(Imm * 8)
1221 .setMIFlag(Flag);
1222 break;
1223 }
1224 case AArch64::LDPXpost:
1225 Imm = -Imm;
1226 [[fallthrough]];
1227 case AArch64::STPXpre: {
1228 Register Reg0 = MBBI->getOperand(1).getReg();
1229 Register Reg1 = MBBI->getOperand(2).getReg();
1230 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1231 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
1232 .addImm(Imm * 8)
1233 .setMIFlag(Flag);
1234 else
1235 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
1236 .addImm(RegInfo->getSEHRegNum(Reg0))
1237 .addImm(RegInfo->getSEHRegNum(Reg1))
1238 .addImm(Imm * 8)
1239 .setMIFlag(Flag);
1240 break;
1241 }
1242 case AArch64::LDRDpost:
1243 Imm = -Imm;
1244 [[fallthrough]];
1245 case AArch64::STRDpre: {
1246 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
1247 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
1248 .addImm(Reg)
1249 .addImm(Imm)
1250 .setMIFlag(Flag);
1251 break;
1252 }
1253 case AArch64::LDRXpost:
1254 Imm = -Imm;
1255 [[fallthrough]];
1256 case AArch64::STRXpre: {
1257 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
1258 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
1259 .addImm(Reg)
1260 .addImm(Imm)
1261 .setMIFlag(Flag);
1262 break;
1263 }
1264 case AArch64::STPDi:
1265 case AArch64::LDPDi: {
1266 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
1267 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
1268 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
1269 .addImm(Reg0)
1270 .addImm(Reg1)
1271 .addImm(Imm * 8)
1272 .setMIFlag(Flag);
1273 break;
1274 }
1275 case AArch64::STPXi:
1276 case AArch64::LDPXi: {
1277 Register Reg0 = MBBI->getOperand(0).getReg();
1278 Register Reg1 = MBBI->getOperand(1).getReg();
1279 if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
1280 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
1281 .addImm(Imm * 8)
1282 .setMIFlag(Flag);
1283 else
1284 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
1285 .addImm(RegInfo->getSEHRegNum(Reg0))
1286 .addImm(RegInfo->getSEHRegNum(Reg1))
1287 .addImm(Imm * 8)
1288 .setMIFlag(Flag);
1289 break;
1290 }
1291 case AArch64::STRXui:
1292 case AArch64::LDRXui: {
1293 int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
1294 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
1295 .addImm(Reg)
1296 .addImm(Imm * 8)
1297 .setMIFlag(Flag);
1298 break;
1299 }
1300 case AArch64::STRDui:
1301 case AArch64::LDRDui: {
1302 unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
1303 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
1304 .addImm(Reg)
1305 .addImm(Imm * 8)
1306 .setMIFlag(Flag);
1307 break;
1308 }
1309 case AArch64::STPQi:
1310 case AArch64::LDPQi: {
1311 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
1312 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
1313 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveAnyRegQP))
1314 .addImm(Reg0)
1315 .addImm(Reg1)
1316 .addImm(Imm * 16)
1317 .setMIFlag(Flag);
1318 break;
1319 }
1320 case AArch64::LDPQpost:
1321 Imm = -Imm;
1322 [[fallthrough]];
1323 case AArch64::STPQpre: {
1324 unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
1325 unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
1326 MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveAnyRegQPX))
1327 .addImm(Reg0)
1328 .addImm(Reg1)
1329 .addImm(Imm * 16)
1330 .setMIFlag(Flag);
1331 break;
1332 }
1333 }
1334 auto I = MBB->insertAfter(MBBI, MIB);
1335 return I;
1336}
1337
1338// Fix up the SEH opcode associated with the save/restore instruction.
1340 unsigned LocalStackSize) {
1341 MachineOperand *ImmOpnd = nullptr;
1342 unsigned ImmIdx = MBBI->getNumOperands() - 1;
1343 switch (MBBI->getOpcode()) {
1344 default:
1345 llvm_unreachable("Fix the offset in the SEH instruction");
1346 case AArch64::SEH_SaveFPLR:
1347 case AArch64::SEH_SaveRegP:
1348 case AArch64::SEH_SaveReg:
1349 case AArch64::SEH_SaveFRegP:
1350 case AArch64::SEH_SaveFReg:
1351 case AArch64::SEH_SaveAnyRegQP:
1352 case AArch64::SEH_SaveAnyRegQPX:
1353 ImmOpnd = &MBBI->getOperand(ImmIdx);
1354 break;
1355 }
1356 if (ImmOpnd)
1357 ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
1358}
1359
1362 return AFI->hasStreamingModeChanges() &&
1363 !MF.getSubtarget<AArch64Subtarget>().hasSVE();
1364}
1365
1367 unsigned Opc = MBBI->getOpcode();
1368 if (Opc == AArch64::CNTD_XPiI || Opc == AArch64::RDSVLI_XI ||
1369 Opc == AArch64::UBFMXri)
1370 return true;
1371
1372 if (requiresGetVGCall(*MBBI->getMF())) {
1373 if (Opc == AArch64::ORRXrr)
1374 return true;
1375
1376 if (Opc == AArch64::BL) {
1377 auto Op1 = MBBI->getOperand(0);
1378 return Op1.isSymbol() &&
1379 (StringRef(Op1.getSymbolName()) == "__arm_get_current_vg");
1380 }
1381 }
1382
1383 return false;
1384}
1385
1386// Convert callee-save register save/restore instruction to do stack pointer
1387// decrement/increment to allocate/deallocate the callee-save stack area by
1388// converting store/load to use pre/post increment version.
1391 const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
1392 bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI,
1394 int CFAOffset = 0) {
1395 unsigned NewOpc;
1396
1397 // If the function contains streaming mode changes, we expect instructions
1398 // to calculate the value of VG before spilling. For locally-streaming
1399 // functions, we need to do this for both the streaming and non-streaming
1400 // vector length. Move past these instructions if necessary.
1401 MachineFunction &MF = *MBB.getParent();
1403 if (AFI->hasStreamingModeChanges())
1404 while (isVGInstruction(MBBI))
1405 ++MBBI;
1406
1407 switch (MBBI->getOpcode()) {
1408 default:
1409 llvm_unreachable("Unexpected callee-save save/restore opcode!");
1410 case AArch64::STPXi:
1411 NewOpc = AArch64::STPXpre;
1412 break;
1413 case AArch64::STPDi:
1414 NewOpc = AArch64::STPDpre;
1415 break;
1416 case AArch64::STPQi:
1417 NewOpc = AArch64::STPQpre;
1418 break;
1419 case AArch64::STRXui:
1420 NewOpc = AArch64::STRXpre;
1421 break;
1422 case AArch64::STRDui:
1423 NewOpc = AArch64::STRDpre;
1424 break;
1425 case AArch64::STRQui:
1426 NewOpc = AArch64::STRQpre;
1427 break;
1428 case AArch64::LDPXi:
1429 NewOpc = AArch64::LDPXpost;
1430 break;
1431 case AArch64::LDPDi:
1432 NewOpc = AArch64::LDPDpost;
1433 break;
1434 case AArch64::LDPQi:
1435 NewOpc = AArch64::LDPQpost;
1436 break;
1437 case AArch64::LDRXui:
1438 NewOpc = AArch64::LDRXpost;
1439 break;
1440 case AArch64::LDRDui:
1441 NewOpc = AArch64::LDRDpost;
1442 break;
1443 case AArch64::LDRQui:
1444 NewOpc = AArch64::LDRQpost;
1445 break;
1446 }
1447 // Get rid of the SEH code associated with the old instruction.
1448 if (NeedsWinCFI) {
1449 auto SEH = std::next(MBBI);
1451 SEH->eraseFromParent();
1452 }
1453
1454 TypeSize Scale = TypeSize::getFixed(1), Width = TypeSize::getFixed(0);
1455 int64_t MinOffset, MaxOffset;
1456 bool Success = static_cast<const AArch64InstrInfo *>(TII)->getMemOpInfo(
1457 NewOpc, Scale, Width, MinOffset, MaxOffset);
1458 (void)Success;
1459 assert(Success && "unknown load/store opcode");
1460
1461 // If the first store isn't right where we want SP then we can't fold the
1462 // update in so create a normal arithmetic instruction instead.
1463 if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0 ||
1464 CSStackSizeInc < MinOffset || CSStackSizeInc > MaxOffset) {
1465 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1466 StackOffset::getFixed(CSStackSizeInc), TII, FrameFlag,
1467 false, false, nullptr, EmitCFI,
1468 StackOffset::getFixed(CFAOffset));
1469
1470 return std::prev(MBBI);
1471 }
1472
1473 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
1474 MIB.addReg(AArch64::SP, RegState::Define);
1475
1476 // Copy all operands other than the immediate offset.
1477 unsigned OpndIdx = 0;
1478 for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
1479 ++OpndIdx)
1480 MIB.add(MBBI->getOperand(OpndIdx));
1481
1482 assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
1483 "Unexpected immediate offset in first/last callee-save save/restore "
1484 "instruction!");
1485 assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
1486 "Unexpected base register in callee-save save/restore instruction!");
1487 assert(CSStackSizeInc % Scale == 0);
1488 MIB.addImm(CSStackSizeInc / (int)Scale);
1489
1490 MIB.setMIFlags(MBBI->getFlags());
1491 MIB.setMemRefs(MBBI->memoperands());
1492
1493 // Generate a new SEH code that corresponds to the new instruction.
1494 if (NeedsWinCFI) {
1495 *HasWinCFI = true;
1496 InsertSEH(*MIB, *TII, FrameFlag);
1497 }
1498
1499 if (EmitCFI) {
1500 unsigned CFIIndex = MF.addFrameInst(
1501 MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset - CSStackSizeInc));
1502 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1503 .addCFIIndex(CFIIndex)
1504 .setMIFlags(FrameFlag);
1505 }
1506
1507 return std::prev(MBB.erase(MBBI));
1508}
1509
1510// Fixup callee-save register save/restore instructions to take into account
1511// combined SP bump by adding the local stack size to the stack offsets.
1513 uint64_t LocalStackSize,
1514 bool NeedsWinCFI,
1515 bool *HasWinCFI) {
1517 return;
1518
1519 unsigned Opc = MI.getOpcode();
1520 unsigned Scale;
1521 switch (Opc) {
1522 case AArch64::STPXi:
1523 case AArch64::STRXui:
1524 case AArch64::STPDi:
1525 case AArch64::STRDui:
1526 case AArch64::LDPXi:
1527 case AArch64::LDRXui:
1528 case AArch64::LDPDi:
1529 case AArch64::LDRDui:
1530 Scale = 8;
1531 break;
1532 case AArch64::STPQi:
1533 case AArch64::STRQui:
1534 case AArch64::LDPQi:
1535 case AArch64::LDRQui:
1536 Scale = 16;
1537 break;
1538 default:
1539 llvm_unreachable("Unexpected callee-save save/restore opcode!");
1540 }
1541
1542 unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
1543 assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
1544 "Unexpected base register in callee-save save/restore instruction!");
1545 // Last operand is immediate offset that needs fixing.
1546 MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
1547 // All generated opcodes have scaled offsets.
1548 assert(LocalStackSize % Scale == 0);
1549 OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
1550
1551 if (NeedsWinCFI) {
1552 *HasWinCFI = true;
1553 auto MBBI = std::next(MachineBasicBlock::iterator(MI));
1554 assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
1556 "Expecting a SEH instruction");
1557 fixupSEHOpcode(MBBI, LocalStackSize);
1558 }
1559}
1560
1561static bool isTargetWindows(const MachineFunction &MF) {
1563}
1564
1565// Convenience function to determine whether I is an SVE callee save.
1567 switch (I->getOpcode()) {
1568 default:
1569 return false;
1570 case AArch64::PTRUE_C_B:
1571 case AArch64::LD1B_2Z_IMM:
1572 case AArch64::ST1B_2Z_IMM:
1573 case AArch64::STR_ZXI:
1574 case AArch64::STR_PXI:
1575 case AArch64::LDR_ZXI:
1576 case AArch64::LDR_PXI:
1577 return I->getFlag(MachineInstr::FrameSetup) ||
1578 I->getFlag(MachineInstr::FrameDestroy);
1579 }
1580}
1581
1583 MachineFunction &MF,
1586 const DebugLoc &DL, bool NeedsWinCFI,
1587 bool NeedsUnwindInfo) {
1588 // Shadow call stack prolog: str x30, [x18], #8
1589 BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXpost))
1590 .addReg(AArch64::X18, RegState::Define)
1591 .addReg(AArch64::LR)
1592 .addReg(AArch64::X18)
1593 .addImm(8)
1595
1596 // This instruction also makes x18 live-in to the entry block.
1597 MBB.addLiveIn(AArch64::X18);
1598
1599 if (NeedsWinCFI)
1600 BuildMI(MBB, MBBI, DL, TII.get(AArch64::SEH_Nop))
1602
1603 if (NeedsUnwindInfo) {
1604 // Emit a CFI instruction that causes 8 to be subtracted from the value of
1605 // x18 when unwinding past this frame.
1606 static const char CFIInst[] = {
1607 dwarf::DW_CFA_val_expression,
1608 18, // register
1609 2, // length
1610 static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
1611 static_cast<char>(-8) & 0x7f, // addend (sleb128)
1612 };
1613 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
1614 nullptr, StringRef(CFIInst, sizeof(CFIInst))));
1615 BuildMI(MBB, MBBI, DL, TII.get(AArch64::CFI_INSTRUCTION))
1616 .addCFIIndex(CFIIndex)
1618 }
1619}
1620
1622 MachineFunction &MF,
1625 const DebugLoc &DL) {
1626 // Shadow call stack epilog: ldr x30, [x18, #-8]!
1627 BuildMI(MBB, MBBI, DL, TII.get(AArch64::LDRXpre))
1628 .addReg(AArch64::X18, RegState::Define)
1629 .addReg(AArch64::LR, RegState::Define)
1630 .addReg(AArch64::X18)
1631 .addImm(-8)
1633
1635 unsigned CFIIndex =
1637 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
1638 .addCFIIndex(CFIIndex)
1640 }
1641}
1642
1643// Define the current CFA rule to use the provided FP.
1646 const DebugLoc &DL, unsigned FixedObject) {
1649 const TargetInstrInfo *TII = STI.getInstrInfo();
1651
1652 const int OffsetToFirstCalleeSaveFromFP =
1655 Register FramePtr = TRI->getFrameRegister(MF);
1656 unsigned Reg = TRI->getDwarfRegNum(FramePtr, true);
1657 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
1658 nullptr, Reg, FixedObject - OffsetToFirstCalleeSaveFromFP));
1659 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1660 .addCFIIndex(CFIIndex)
1662}
1663
1664#ifndef NDEBUG
1665/// Collect live registers from the end of \p MI's parent up to (including) \p
1666/// MI in \p LiveRegs.
1668 LivePhysRegs &LiveRegs) {
1669
1670 MachineBasicBlock &MBB = *MI.getParent();
1671 LiveRegs.addLiveOuts(MBB);
1672 for (const MachineInstr &MI :
1673 reverse(make_range(MI.getIterator(), MBB.instr_end())))
1674 LiveRegs.stepBackward(MI);
1675}
1676#endif
1677
1679 MachineBasicBlock &MBB) const {
1681 const MachineFrameInfo &MFI = MF.getFrameInfo();
1682 const Function &F = MF.getFunction();
1683 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1684 const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1685 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1686
1687 MachineModuleInfo &MMI = MF.getMMI();
1689 bool EmitCFI = AFI->needsDwarfUnwindInfo(MF);
1690 bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
1691 bool HasFP = hasFP(MF);
1692 bool NeedsWinCFI = needsWinCFI(MF);
1693 bool HasWinCFI = false;
1694 auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); });
1695
1697#ifndef NDEBUG
1699 // Collect live register from the end of MBB up to the start of the existing
1700 // frame setup instructions.
1701 MachineBasicBlock::iterator NonFrameStart = MBB.begin();
1702 while (NonFrameStart != End &&
1703 NonFrameStart->getFlag(MachineInstr::FrameSetup))
1704 ++NonFrameStart;
1705
1706 LivePhysRegs LiveRegs(*TRI);
1707 if (NonFrameStart != MBB.end()) {
1708 getLivePhysRegsUpTo(*NonFrameStart, *TRI, LiveRegs);
1709 // Ignore registers used for stack management for now.
1710 LiveRegs.removeReg(AArch64::SP);
1711 LiveRegs.removeReg(AArch64::X19);
1712 LiveRegs.removeReg(AArch64::FP);
1713 LiveRegs.removeReg(AArch64::LR);
1714
1715 // X0 will be clobbered by a call to __arm_get_current_vg in the prologue.
1716 // This is necessary to spill VG if required where SVE is unavailable, but
1717 // X0 is preserved around this call.
1718 if (requiresGetVGCall(MF))
1719 LiveRegs.removeReg(AArch64::X0);
1720 }
1721
1722 auto VerifyClobberOnExit = make_scope_exit([&]() {
1723 if (NonFrameStart == MBB.end())
1724 return;
1725 // Check if any of the newly instructions clobber any of the live registers.
1726 for (MachineInstr &MI :
1727 make_range(MBB.instr_begin(), NonFrameStart->getIterator())) {
1728 for (auto &Op : MI.operands())
1729 if (Op.isReg() && Op.isDef())
1730 assert(!LiveRegs.contains(Op.getReg()) &&
1731 "live register clobbered by inserted prologue instructions");
1732 }
1733 });
1734#endif
1735
1736 bool IsFunclet = MBB.isEHFuncletEntry();
1737
1738 // At this point, we're going to decide whether or not the function uses a
1739 // redzone. In most cases, the function doesn't have a redzone so let's
1740 // assume that's false and set it to true in the case that there's a redzone.
1741 AFI->setHasRedZone(false);
1742
1743 // Debug location must be unknown since the first debug location is used
1744 // to determine the end of the prologue.
1745 DebugLoc DL;
1746
1747 const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
1748 if (MFnI.needsShadowCallStackPrologueEpilogue(MF))
1749 emitShadowCallStackPrologue(*TII, MF, MBB, MBBI, DL, NeedsWinCFI,
1750 MFnI.needsDwarfUnwindInfo(MF));
1751
1752 if (MFnI.shouldSignReturnAddress(MF)) {
1753 BuildMI(MBB, MBBI, DL, TII->get(AArch64::PAUTH_PROLOGUE))
1755 if (NeedsWinCFI)
1756 HasWinCFI = true; // AArch64PointerAuth pass will insert SEH_PACSignLR
1757 }
1758
1759 if (EmitCFI && MFnI.isMTETagged()) {
1760 BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITMTETAGGED))
1762 }
1763
1764 // We signal the presence of a Swift extended frame to external tools by
1765 // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple
1766 // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI
1767 // bits so that is still true.
1768 if (HasFP && AFI->hasSwiftAsyncContext()) {
1771 if (Subtarget.swiftAsyncContextIsDynamicallySet()) {
1772 // The special symbol below is absolute and has a *value* that can be
1773 // combined with the frame pointer to signal an extended frame.
1774 BuildMI(MBB, MBBI, DL, TII->get(AArch64::LOADgot), AArch64::X16)
1775 .addExternalSymbol("swift_async_extendedFramePointerFlags",
1777 if (NeedsWinCFI) {
1778 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1780 HasWinCFI = true;
1781 }
1782 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::FP)
1783 .addUse(AArch64::FP)
1784 .addUse(AArch64::X16)
1785 .addImm(Subtarget.isTargetILP32() ? 32 : 0);
1786 if (NeedsWinCFI) {
1787 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1789 HasWinCFI = true;
1790 }
1791 break;
1792 }
1793 [[fallthrough]];
1794
1796 // ORR x29, x29, #0x1000_0000_0000_0000
1797 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP)
1798 .addUse(AArch64::FP)
1799 .addImm(0x1100)
1801 if (NeedsWinCFI) {
1802 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1804 HasWinCFI = true;
1805 }
1806 break;
1807
1809 break;
1810 }
1811 }
1812
1813 // All calls are tail calls in GHC calling conv, and functions have no
1814 // prologue/epilogue.
1816 return;
1817
1818 // Set tagged base pointer to the requested stack slot.
1819 // Ideally it should match SP value after prologue.
1820 std::optional<int> TBPI = AFI->getTaggedBasePointerIndex();
1821 if (TBPI)
1823 else
1825
1826 const StackOffset &SVEStackSize = getSVEStackSize(MF);
1827
1828 // getStackSize() includes all the locals in its size calculation. We don't
1829 // include these locals when computing the stack size of a funclet, as they
1830 // are allocated in the parent's stack frame and accessed via the frame
1831 // pointer from the funclet. We only save the callee saved registers in the
1832 // funclet, which are really the callee saved registers of the parent
1833 // function, including the funclet.
1834 int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
1835 : MFI.getStackSize();
1836 if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
1837 assert(!HasFP && "unexpected function without stack frame but with FP");
1838 assert(!SVEStackSize &&
1839 "unexpected function without stack frame but with SVE objects");
1840 // All of the stack allocation is for locals.
1841 AFI->setLocalStackSize(NumBytes);
1842 if (!NumBytes)
1843 return;
1844 // REDZONE: If the stack size is less than 128 bytes, we don't need
1845 // to actually allocate.
1846 if (canUseRedZone(MF)) {
1847 AFI->setHasRedZone(true);
1848 ++NumRedZoneFunctions;
1849 } else {
1850 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1851 StackOffset::getFixed(-NumBytes), TII,
1852 MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
1853 if (EmitCFI) {
1854 // Label used to tie together the PROLOG_LABEL and the MachineMoves.
1855 MCSymbol *FrameLabel = MMI.getContext().createTempSymbol();
1856 // Encode the stack size of the leaf function.
1857 unsigned CFIIndex = MF.addFrameInst(
1858 MCCFIInstruction::cfiDefCfaOffset(FrameLabel, NumBytes));
1859 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1860 .addCFIIndex(CFIIndex)
1862 }
1863 }
1864
1865 if (NeedsWinCFI) {
1866 HasWinCFI = true;
1867 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1869 }
1870
1871 return;
1872 }
1873
1874 bool IsWin64 =
1876 unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
1877
1878 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
1879 // All of the remaining stack allocations are for locals.
1880 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
1881 bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
1882 bool HomPrologEpilog = homogeneousPrologEpilog(MF);
1883 if (CombineSPBump) {
1884 assert(!SVEStackSize && "Cannot combine SP bump with SVE");
1885 emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
1886 StackOffset::getFixed(-NumBytes), TII,
1887 MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI,
1888 EmitAsyncCFI);
1889 NumBytes = 0;
1890 } else if (HomPrologEpilog) {
1891 // Stack has been already adjusted.
1892 NumBytes -= PrologueSaveSize;
1893 } else if (PrologueSaveSize != 0) {
1895 MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI,
1896 EmitAsyncCFI);
1897 NumBytes -= PrologueSaveSize;
1898 }
1899 assert(NumBytes >= 0 && "Negative stack allocation size!?");
1900
1901 // Move past the saves of the callee-saved registers, fixing up the offsets
1902 // and pre-inc if we decided to combine the callee-save and local stack
1903 // pointer bump above.
1904 while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) &&
1906 // Move past instructions generated to calculate VG
1907 if (AFI->hasStreamingModeChanges())
1908 while (isVGInstruction(MBBI))
1909 ++MBBI;
1910
1911 if (CombineSPBump)
1913 NeedsWinCFI, &HasWinCFI);
1914 ++MBBI;
1915 }
1916
1917 // For funclets the FP belongs to the containing function.
1918 if (!IsFunclet && HasFP) {
1919 // Only set up FP if we actually need to.
1920 int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset();
1921
1922 if (CombineSPBump)
1923 FPOffset += AFI->getLocalStackSize();
1924
1925 if (AFI->hasSwiftAsyncContext()) {
1926 // Before we update the live FP we have to ensure there's a valid (or
1927 // null) asynchronous context in its slot just before FP in the frame
1928 // record, so store it now.
1929 const auto &Attrs = MF.getFunction().getAttributes();
1930 bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync);
1931 if (HaveInitialContext)
1932 MBB.addLiveIn(AArch64::X22);
1933 Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR;
1934 BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext))
1935 .addUse(Reg)
1936 .addUse(AArch64::SP)
1937 .addImm(FPOffset - 8)
1939 if (NeedsWinCFI) {
1940 // WinCFI and arm64e, where StoreSwiftAsyncContext is expanded
1941 // to multiple instructions, should be mutually-exclusive.
1942 assert(Subtarget.getTargetTriple().getArchName() != "arm64e");
1943 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
1945 HasWinCFI = true;
1946 }
1947 }
1948
1949 if (HomPrologEpilog) {
1950 auto Prolog = MBBI;
1951 --Prolog;
1952 assert(Prolog->getOpcode() == AArch64::HOM_Prolog);
1953 Prolog->addOperand(MachineOperand::CreateImm(FPOffset));
1954 } else {
1955 // Issue sub fp, sp, FPOffset or
1956 // mov fp,sp when FPOffset is zero.
1957 // Note: All stores of callee-saved registers are marked as "FrameSetup".
1958 // This code marks the instruction(s) that set the FP also.
1959 emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP,
1960 StackOffset::getFixed(FPOffset), TII,
1961 MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
1962 if (NeedsWinCFI && HasWinCFI) {
1963 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
1965 // After setting up the FP, the rest of the prolog doesn't need to be
1966 // included in the SEH unwind info.
1967 NeedsWinCFI = false;
1968 }
1969 }
1970 if (EmitAsyncCFI)
1971 emitDefineCFAWithFP(MF, MBB, MBBI, DL, FixedObject);
1972 }
1973
1974 // Now emit the moves for whatever callee saved regs we have (including FP,
1975 // LR if those are saved). Frame instructions for SVE register are emitted
1976 // later, after the instruction which actually save SVE regs.
1977 if (EmitAsyncCFI)
1978 emitCalleeSavedGPRLocations(MBB, MBBI);
1979
1980 // Alignment is required for the parent frame, not the funclet
1981 const bool NeedsRealignment =
1982 NumBytes && !IsFunclet && RegInfo->hasStackRealignment(MF);
1983 const int64_t RealignmentPadding =
1984 (NeedsRealignment && MFI.getMaxAlign() > Align(16))
1985 ? MFI.getMaxAlign().value() - 16
1986 : 0;
1987
1988 if (windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding)) {
1989 uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4;
1990 if (NeedsWinCFI) {
1991 HasWinCFI = true;
1992 // alloc_l can hold at most 256MB, so assume that NumBytes doesn't
1993 // exceed this amount. We need to move at most 2^24 - 1 into x15.
1994 // This is at most two instructions, MOVZ follwed by MOVK.
1995 // TODO: Fix to use multiple stack alloc unwind codes for stacks
1996 // exceeding 256MB in size.
1997 if (NumBytes >= (1 << 28))
1998 report_fatal_error("Stack size cannot exceed 256MB for stack "
1999 "unwinding purposes");
2000
2001 uint32_t LowNumWords = NumWords & 0xFFFF;
2002 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
2003 .addImm(LowNumWords)
2006 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
2008 if ((NumWords & 0xFFFF0000) != 0) {
2009 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
2010 .addReg(AArch64::X15)
2011 .addImm((NumWords & 0xFFFF0000) >> 16) // High half
2014 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
2016 }
2017 } else {
2018 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
2019 .addImm(NumWords)
2021 }
2022
2023 const char* ChkStk = Subtarget.getChkStkName();
2024 switch (MF.getTarget().getCodeModel()) {
2025 case CodeModel::Tiny:
2026 case CodeModel::Small:
2027 case CodeModel::Medium:
2028 case CodeModel::Kernel:
2029 BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
2030 .addExternalSymbol(ChkStk)
2031 .addReg(AArch64::X15, RegState::Implicit)
2036 if (NeedsWinCFI) {
2037 HasWinCFI = true;
2038 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
2040 }
2041 break;
2042 case CodeModel::Large:
2043 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
2044 .addReg(AArch64::X16, RegState::Define)
2045 .addExternalSymbol(ChkStk)
2046 .addExternalSymbol(ChkStk)
2048 if (NeedsWinCFI) {
2049 HasWinCFI = true;
2050 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
2052 }
2053
2054 BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF)))
2055 .addReg(AArch64::X16, RegState::Kill)
2061 if (NeedsWinCFI) {
2062 HasWinCFI = true;
2063 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
2065 }
2066 break;
2067 }
2068
2069 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
2070 .addReg(AArch64::SP, RegState::Kill)
2071 .addReg(AArch64::X15, RegState::Kill)
2074 if (NeedsWinCFI) {
2075 HasWinCFI = true;
2076 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
2077 .addImm(NumBytes)
2079 }
2080 NumBytes = 0;
2081
2082 if (RealignmentPadding > 0) {
2083 if (RealignmentPadding >= 4096) {
2084 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm))
2085 .addReg(AArch64::X16, RegState::Define)
2086 .addImm(RealignmentPadding)
2088 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXrx64), AArch64::X15)
2089 .addReg(AArch64::SP)
2090 .addReg(AArch64::X16, RegState::Kill)
2093 } else {
2094 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), AArch64::X15)
2095 .addReg(AArch64::SP)
2096 .addImm(RealignmentPadding)
2097 .addImm(0)
2099 }
2100
2101 uint64_t AndMask = ~(MFI.getMaxAlign().value() - 1);
2102 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
2103 .addReg(AArch64::X15, RegState::Kill)
2105 AFI->setStackRealigned(true);
2106
2107 // No need for SEH instructions here; if we're realigning the stack,
2108 // we've set a frame pointer and already finished the SEH prologue.
2109 assert(!NeedsWinCFI);
2110 }
2111 }
2112
2113 StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize;
2114 MachineBasicBlock::iterator CalleeSavesBegin = MBBI, CalleeSavesEnd = MBBI;
2115
2116 // Process the SVE callee-saves to determine what space needs to be
2117 // allocated.
2118 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
2119 LLVM_DEBUG(dbgs() << "SVECalleeSavedStackSize = " << CalleeSavedSize
2120 << "\n");
2121 // Find callee save instructions in frame.
2122 CalleeSavesBegin = MBBI;
2123 assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
2125 ++MBBI;
2126 CalleeSavesEnd = MBBI;
2127
2128 SVECalleeSavesSize = StackOffset::getScalable(CalleeSavedSize);
2129 SVELocalsSize = SVEStackSize - SVECalleeSavesSize;
2130 }
2131
2132 // Allocate space for the callee saves (if any).
2133 StackOffset CFAOffset =
2134 StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes);
2135 StackOffset LocalsSize = SVELocalsSize + StackOffset::getFixed(NumBytes);
2136 allocateStackSpace(MBB, CalleeSavesBegin, 0, SVECalleeSavesSize, false,
2137 nullptr, EmitAsyncCFI && !HasFP, CFAOffset,
2138 MFI.hasVarSizedObjects() || LocalsSize);
2139 CFAOffset += SVECalleeSavesSize;
2140
2141 if (EmitAsyncCFI)
2142 emitCalleeSavedSVELocations(MBB, CalleeSavesEnd);
2143
2144 // Allocate space for the rest of the frame including SVE locals. Align the
2145 // stack as necessary.
2146 assert(!(canUseRedZone(MF) && NeedsRealignment) &&
2147 "Cannot use redzone with stack realignment");
2148 if (!canUseRedZone(MF)) {
2149 // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
2150 // the correct value here, as NumBytes also includes padding bytes,
2151 // which shouldn't be counted here.
2152 allocateStackSpace(MBB, CalleeSavesEnd, RealignmentPadding,
2153 SVELocalsSize + StackOffset::getFixed(NumBytes),
2154 NeedsWinCFI, &HasWinCFI, EmitAsyncCFI && !HasFP,
2155 CFAOffset, MFI.hasVarSizedObjects());
2156 }
2157
2158 // If we need a base pointer, set it up here. It's whatever the value of the
2159 // stack pointer is at this point. Any variable size objects will be allocated
2160 // after this, so we can still use the base pointer to reference locals.
2161 //
2162 // FIXME: Clarify FrameSetup flags here.
2163 // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
2164 // needed.
2165 // For funclets the BP belongs to the containing function.
2166 if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
2167 TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
2168 false);
2169 if (NeedsWinCFI) {
2170 HasWinCFI = true;
2171 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
2173 }
2174 }
2175
2176 // The very last FrameSetup instruction indicates the end of prologue. Emit a
2177 // SEH opcode indicating the prologue end.
2178 if (NeedsWinCFI && HasWinCFI) {
2179 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
2181 }
2182
2183 // SEH funclets are passed the frame pointer in X1. If the parent
2184 // function uses the base register, then the base register is used
2185 // directly, and is not retrieved from X1.
2186 if (IsFunclet && F.hasPersonalityFn()) {
2187 EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
2188 if (isAsynchronousEHPersonality(Per)) {
2189 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
2190 .addReg(AArch64::X1)
2192 MBB.addLiveIn(AArch64::X1);
2193 }
2194 }
2195
2196 if (EmitCFI && !EmitAsyncCFI) {
2197 if (HasFP) {
2198 emitDefineCFAWithFP(MF, MBB, MBBI, DL, FixedObject);
2199 } else {
2200 StackOffset TotalSize =
2201 SVEStackSize + StackOffset::getFixed((int64_t)MFI.getStackSize());
2202 unsigned CFIIndex = MF.addFrameInst(createDefCFA(
2203 *RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP, TotalSize,
2204 /*LastAdjustmentWasScalable=*/false));
2205 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
2206 .addCFIIndex(CFIIndex)
2208 }
2209 emitCalleeSavedGPRLocations(MBB, MBBI);
2210 emitCalleeSavedSVELocations(MBB, MBBI);
2211 }
2212}
2213
2215 switch (MI.getOpcode()) {
2216 default:
2217 return false;
2218 case AArch64::CATCHRET:
2219 case AArch64::CLEANUPRET:
2220 return true;
2221 }
2222}
2223
2225 MachineBasicBlock &MBB) const {
2227 MachineFrameInfo &MFI = MF.getFrameInfo();
2229 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2230 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
2231 DebugLoc DL;
2232 bool NeedsWinCFI = needsWinCFI(MF);
2233 bool EmitCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
2234 bool HasWinCFI = false;
2235 bool IsFunclet = false;
2236
2237 if (MBB.end() != MBBI) {
2238 DL = MBBI->getDebugLoc();
2239 IsFunclet = isFuncletReturnInstr(*MBBI);
2240 }
2241
2242 MachineBasicBlock::iterator EpilogStartI = MBB.end();
2243
2244 auto FinishingTouches = make_scope_exit([&]() {
2245 if (AFI->shouldSignReturnAddress(MF)) {
2246 BuildMI(MBB, MBB.getFirstTerminator(), DL,
2247 TII->get(AArch64::PAUTH_EPILOGUE))
2248 .setMIFlag(MachineInstr::FrameDestroy);
2249 if (NeedsWinCFI)
2250 HasWinCFI = true; // AArch64PointerAuth pass will insert SEH_PACSignLR
2251 }
2254 if (EmitCFI)
2255 emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
2256 if (HasWinCFI) {
2258 TII->get(AArch64::SEH_EpilogEnd))
2260 if (!MF.hasWinCFI())
2261 MF.setHasWinCFI(true);
2262 }
2263 if (NeedsWinCFI) {
2264 assert(EpilogStartI != MBB.end());
2265 if (!HasWinCFI)
2266 MBB.erase(EpilogStartI);
2267 }
2268 });
2269
2270 int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
2271 : MFI.getStackSize();
2272
2273 // All calls are tail calls in GHC calling conv, and functions have no
2274 // prologue/epilogue.
2276 return;
2277
2278 // How much of the stack used by incoming arguments this function is expected
2279 // to restore in this particular epilogue.
2280 int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB);
2281 bool IsWin64 =
2282 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
2283 unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
2284
2285 int64_t AfterCSRPopSize = ArgumentStackToRestore;
2286 auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
2287 // We cannot rely on the local stack size set in emitPrologue if the function
2288 // has funclets, as funclets have different local stack size requirements, and
2289 // the current value set in emitPrologue may be that of the containing
2290 // function.
2291 if (MF.hasEHFunclets())
2292 AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
2293 if (homogeneousPrologEpilog(MF, &MBB)) {
2294 assert(!NeedsWinCFI);
2295 auto LastPopI = MBB.getFirstTerminator();
2296 if (LastPopI != MBB.begin()) {
2297 auto HomogeneousEpilog = std::prev(LastPopI);
2298 if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog)
2299 LastPopI = HomogeneousEpilog;
2300 }
2301
2302 // Adjust local stack
2303 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
2305 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
2306
2307 // SP has been already adjusted while restoring callee save regs.
2308 // We've bailed-out the case with adjusting SP for arguments.
2309 assert(AfterCSRPopSize == 0);
2310 return;
2311 }
2312 bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
2313 // Assume we can't combine the last pop with the sp restore.
2314
2315 bool CombineAfterCSRBump = false;
2316 if (!CombineSPBump && PrologueSaveSize != 0) {
2318 while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
2320 Pop = std::prev(Pop);
2321 // Converting the last ldp to a post-index ldp is valid only if the last
2322 // ldp's offset is 0.
2323 const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
2324 // If the offset is 0 and the AfterCSR pop is not actually trying to
2325 // allocate more stack for arguments (in space that an untimely interrupt
2326 // may clobber), convert it to a post-index ldp.
2327 if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) {
2329 MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, EmitCFI,
2330 MachineInstr::FrameDestroy, PrologueSaveSize);
2331 } else {
2332 // If not, make sure to emit an add after the last ldp.
2333 // We're doing this by transfering the size to be restored from the
2334 // adjustment *before* the CSR pops to the adjustment *after* the CSR
2335 // pops.
2336 AfterCSRPopSize += PrologueSaveSize;
2337 CombineAfterCSRBump = true;
2338 }
2339 }
2340
2341 // Move past the restores of the callee-saved registers.
2342 // If we plan on combining the sp bump of the local stack size and the callee
2343 // save stack size, we might need to adjust the CSR save and restore offsets.
2346 while (LastPopI != Begin) {
2347 --LastPopI;
2348 if (!LastPopI->getFlag(MachineInstr::FrameDestroy) ||
2349 IsSVECalleeSave(LastPopI)) {
2350 ++LastPopI;
2351 break;
2352 } else if (CombineSPBump)
2354 NeedsWinCFI, &HasWinCFI);
2355 }
2356
2357 if (NeedsWinCFI) {
2358 // Note that there are cases where we insert SEH opcodes in the
2359 // epilogue when we had no SEH opcodes in the prologue. For
2360 // example, when there is no stack frame but there are stack
2361 // arguments. Insert the SEH_EpilogStart and remove it later if it
2362 // we didn't emit any SEH opcodes to avoid generating WinCFI for
2363 // functions that don't need it.
2364 BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
2366 EpilogStartI = LastPopI;
2367 --EpilogStartI;
2368 }
2369
2370 if (hasFP(MF) && AFI->hasSwiftAsyncContext()) {
2373 // Avoid the reload as it is GOT relative, and instead fall back to the
2374 // hardcoded value below. This allows a mismatch between the OS and
2375 // application without immediately terminating on the difference.
2376 [[fallthrough]];
2378 // We need to reset FP to its untagged state on return. Bit 60 is
2379 // currently used to show the presence of an extended frame.
2380
2381 // BIC x29, x29, #0x1000_0000_0000_0000
2382 BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::ANDXri),
2383 AArch64::FP)
2384 .addUse(AArch64::FP)
2385 .addImm(0x10fe)
2387 if (NeedsWinCFI) {
2388 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
2390 HasWinCFI = true;
2391 }
2392 break;
2393
2395 break;
2396 }
2397 }
2398
2399 const StackOffset &SVEStackSize = getSVEStackSize(MF);
2400
2401 // If there is a single SP update, insert it before the ret and we're done.
2402 if (CombineSPBump) {
2403 assert(!SVEStackSize && "Cannot combine SP bump with SVE");
2404
2405 // When we are about to restore the CSRs, the CFA register is SP again.
2406 if (EmitCFI && hasFP(MF)) {
2407 const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
2408 unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
2409 unsigned CFIIndex =
2410 MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, NumBytes));
2411 BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
2412 .addCFIIndex(CFIIndex)
2414 }
2415
2416 emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
2417 StackOffset::getFixed(NumBytes + (int64_t)AfterCSRPopSize),
2418 TII, MachineInstr::FrameDestroy, false, NeedsWinCFI,
2419 &HasWinCFI, EmitCFI, StackOffset::getFixed(NumBytes));
2420 return;
2421 }
2422
2423 NumBytes -= PrologueSaveSize;
2424 assert(NumBytes >= 0 && "Negative stack allocation size!?");
2425
2426 // Process the SVE callee-saves to determine what space needs to be
2427 // deallocated.
2428 StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
2429 MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
2430 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
2431 RestoreBegin = std::prev(RestoreEnd);
2432 while (RestoreBegin != MBB.begin() &&
2433 IsSVECalleeSave(std::prev(RestoreBegin)))
2434 --RestoreBegin;
2435
2436 assert(IsSVECalleeSave(RestoreBegin) &&
2437 IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
2438
2439 StackOffset CalleeSavedSizeAsOffset =
2440 StackOffset::getScalable(CalleeSavedSize);
2441 DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
2442 DeallocateAfter = CalleeSavedSizeAsOffset;
2443 }
2444
2445 // Deallocate the SVE area.
2446 if (SVEStackSize) {
2447 // If we have stack realignment or variable sized objects on the stack,
2448 // restore the stack pointer from the frame pointer prior to SVE CSR
2449 // restoration.
2450 if (AFI->isStackRealigned() || MFI.hasVarSizedObjects()) {
2451 if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
2452 // Set SP to start of SVE callee-save area from which they can
2453 // be reloaded. The code below will deallocate the stack space
2454 // space by moving FP -> SP.
2455 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
2456 StackOffset::getScalable(-CalleeSavedSize), TII,
2458 }
2459 } else {
2460 if (AFI->getSVECalleeSavedStackSize()) {
2461 // Deallocate the non-SVE locals first before we can deallocate (and
2462 // restore callee saves) from the SVE area.
2464 MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
2466 false, false, nullptr, EmitCFI && !hasFP(MF),
2467 SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize));
2468 NumBytes = 0;
2469 }
2470
2471 emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
2472 DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
2473 false, nullptr, EmitCFI && !hasFP(MF),
2474 SVEStackSize +
2475 StackOffset::getFixed(NumBytes + PrologueSaveSize));
2476
2477 emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
2478 DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
2479 false, nullptr, EmitCFI && !hasFP(MF),
2480 DeallocateAfter +
2481 StackOffset::getFixed(NumBytes + PrologueSaveSize));
2482 }
2483 if (EmitCFI)
2484 emitCalleeSavedSVERestores(MBB, RestoreEnd);
2485 }
2486
2487 if (!hasFP(MF)) {
2488 bool RedZone = canUseRedZone(MF);
2489 // If this was a redzone leaf function, we don't need to restore the
2490 // stack pointer (but we may need to pop stack args for fastcc).
2491 if (RedZone && AfterCSRPopSize == 0)
2492 return;
2493
2494 // Pop the local variables off the stack. If there are no callee-saved
2495 // registers, it means we are actually positioned at the terminator and can
2496 // combine stack increment for the locals and the stack increment for
2497 // callee-popped arguments into (possibly) a single instruction and be done.
2498 bool NoCalleeSaveRestore = PrologueSaveSize == 0;
2499 int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
2500 if (NoCalleeSaveRestore)
2501 StackRestoreBytes += AfterCSRPopSize;
2502
2504 MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
2505 StackOffset::getFixed(StackRestoreBytes), TII,
2506 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI,
2507 StackOffset::getFixed((RedZone ? 0 : NumBytes) + PrologueSaveSize));
2508
2509 // If we were able to combine the local stack pop with the argument pop,
2510 // then we're done.
2511 if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
2512 return;
2513 }
2514
2515 NumBytes = 0;
2516 }
2517
2518 // Restore the original stack pointer.
2519 // FIXME: Rather than doing the math here, we should instead just use
2520 // non-post-indexed loads for the restores if we aren't actually going to
2521 // be able to save any instructions.
2522 if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
2524 MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
2526 TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
2527 } else if (NumBytes)
2528 emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
2529 StackOffset::getFixed(NumBytes), TII,
2530 MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
2531
2532 // When we are about to restore the CSRs, the CFA register is SP again.
2533 if (EmitCFI && hasFP(MF)) {
2534 const AArch64RegisterInfo &RegInfo = *Subtarget.getRegisterInfo();
2535 unsigned Reg = RegInfo.getDwarfRegNum(AArch64::SP, true);
2536 unsigned CFIIndex = MF.addFrameInst(
2537 MCCFIInstruction::cfiDefCfa(nullptr, Reg, PrologueSaveSize));
2538 BuildMI(MBB, LastPopI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
2539 .addCFIIndex(CFIIndex)
2541 }
2542
2543 // This must be placed after the callee-save restore code because that code
2544 // assumes the SP is at the same location as it was after the callee-save save
2545 // code in the prologue.
2546 if (AfterCSRPopSize) {
2547 assert(AfterCSRPopSize > 0 && "attempting to reallocate arg stack that an "
2548 "interrupt may have clobbered");
2549
2551 MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
2553 false, NeedsWinCFI, &HasWinCFI, EmitCFI,
2554 StackOffset::getFixed(CombineAfterCSRBump ? PrologueSaveSize : 0));
2555 }
2556}
2557
2560 MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF);
2561}
2562
2563/// getFrameIndexReference - Provide a base+offset reference to an FI slot for
2564/// debug info. It's the same as what we use for resolving the code-gen
2565/// references for now. FIXME: This can go wrong when references are
2566/// SP-relative and simple call frames aren't used.
2569 Register &FrameReg) const {
2571 MF, FI, FrameReg,
2572 /*PreferFP=*/
2573 MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress) ||
2574 MF.getFunction().hasFnAttribute(Attribute::SanitizeMemTag),
2575 /*ForSimm=*/false);
2576}
2577
2580 int FI) const {
2582}
2583
2585 int64_t ObjectOffset) {
2586 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2587 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2588 bool IsWin64 =
2589 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
2590 unsigned FixedObject =
2591 getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
2592 int64_t CalleeSaveSize = AFI->getCalleeSavedStackSize(MF.getFrameInfo());
2593 int64_t FPAdjust =
2594 CalleeSaveSize - AFI->getCalleeSaveBaseToFrameRecordOffset();
2595 return StackOffset::getFixed(ObjectOffset + FixedObject + FPAdjust);
2596}
2597
2599 int64_t ObjectOffset) {
2600 const auto &MFI = MF.getFrameInfo();
2601 return StackOffset::getFixed(ObjectOffset + (int64_t)MFI.getStackSize());
2602}
2603
2604 // TODO: This function currently does not work for scalable vectors.
2606 int FI) const {
2607 const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
2609 int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI);
2610 return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
2611 ? getFPOffset(MF, ObjectOffset).getFixed()
2612 : getStackOffset(MF, ObjectOffset).getFixed();
2613}
2614
2616 const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP,
2617 bool ForSimm) const {
2618 const auto &MFI = MF.getFrameInfo();
2619 int64_t ObjectOffset = MFI.getObjectOffset(FI);
2620 bool isFixed = MFI.isFixedObjectIndex(FI);
2621 bool isSVE = MFI.getStackID(FI) == TargetStackID::ScalableVector;
2622 return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
2623 PreferFP, ForSimm);
2624}
2625
2627 const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
2628 Register &FrameReg, bool PreferFP, bool ForSimm) const {
2629 const auto &MFI = MF.getFrameInfo();
2630 const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
2632 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
2633 const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2634
2635 int64_t FPOffset = getFPOffset(MF, ObjectOffset).getFixed();
2636 int64_t Offset = getStackOffset(MF, ObjectOffset).getFixed();
2637 bool isCSR =
2638 !isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
2639
2640 const StackOffset &SVEStackSize = getSVEStackSize(MF);
2641
2642 // Use frame pointer to reference fixed objects. Use it for locals if
2643 // there are VLAs or a dynamically realigned SP (and thus the SP isn't
2644 // reliable as a base). Make sure useFPForScavengingIndex() does the
2645 // right thing for the emergency spill slot.
2646 bool UseFP = false;
2647 if (AFI->hasStackFrame() && !isSVE) {
2648 // We shouldn't prefer using the FP to access fixed-sized stack objects when
2649 // there are scalable (SVE) objects in between the FP and the fixed-sized
2650 // objects.
2651 PreferFP &= !SVEStackSize;
2652
2653 // Note: Keeping the following as multiple 'if' statements rather than
2654 // merging to a single expression for readability.
2655 //
2656 // Argument access should always use the FP.
2657 if (isFixed) {
2658 UseFP = hasFP(MF);
2659 } else if (isCSR && RegInfo->hasStackRealignment(MF)) {
2660 // References to the CSR area must use FP if we're re-aligning the stack
2661 // since the dynamically-sized alignment padding is between the SP/BP and
2662 // the CSR area.
2663 assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
2664 UseFP = true;
2665 } else if (hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
2666 // If the FPOffset is negative and we're producing a signed immediate, we
2667 // have to keep in mind that the available offset range for negative
2668 // offsets is smaller than for positive ones. If an offset is available
2669 // via the FP and the SP, use whichever is closest.
2670 bool FPOffsetFits = !ForSimm || FPOffset >= -256;
2671 PreferFP |= Offset > -FPOffset && !SVEStackSize;
2672
2673 if (MFI.hasVarSizedObjects()) {
2674 // If we have variable sized objects, we can use either FP or BP, as the
2675 // SP offset is unknown. We can use the base pointer if we have one and
2676 // FP is not preferred. If not, we're stuck with using FP.
2677 bool CanUseBP = RegInfo->hasBasePointer(MF);
2678 if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
2679 UseFP = PreferFP;
2680 else if (!CanUseBP) // Can't use BP. Forced to use FP.
2681 UseFP = true;
2682 // else we can use BP and FP, but the offset from FP won't fit.
2683 // That will make us scavenge registers which we can probably avoid by
2684 // using BP. If it won't fit for BP either, we'll scavenge anyway.
2685 } else if (FPOffset >= 0) {
2686 // Use SP or FP, whichever gives us the best chance of the offset
2687 // being in range for direct access. If the FPOffset is positive,
2688 // that'll always be best, as the SP will be even further away.
2689 UseFP = true;
2690 } else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
2691 // Funclets access the locals contained in the parent's stack frame
2692 // via the frame pointer, so we have to use the FP in the parent
2693 // function.
2694 (void) Subtarget;
2695 assert(
2696 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
2697 "Funclets should only be present on Win64");
2698 UseFP = true;
2699 } else {
2700 // We have the choice between FP and (SP or BP).
2701 if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
2702 UseFP = true;
2703 }
2704 }
2705 }
2706
2707 assert(
2708 ((isFixed || isCSR) || !RegInfo->hasStackRealignment(MF) || !UseFP) &&
2709 "In the presence of dynamic stack pointer realignment, "
2710 "non-argument/CSR objects cannot be accessed through the frame pointer");
2711
2712 if (isSVE) {
2713 StackOffset FPOffset =
2715 StackOffset SPOffset =
2716 SVEStackSize +
2717 StackOffset::get(MFI.getStackSize() - AFI->getCalleeSavedStackSize(),
2718 ObjectOffset);
2719 // Always use the FP for SVE spills if available and beneficial.
2720 if (hasFP(MF) && (SPOffset.getFixed() ||
2721 FPOffset.getScalable() < SPOffset.getScalable() ||
2722 RegInfo->hasStackRealignment(MF))) {
2723 FrameReg = RegInfo->getFrameRegister(MF);
2724 return FPOffset;
2725 }
2726
2727 FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
2728 : (unsigned)AArch64::SP;
2729 return SPOffset;
2730 }
2731
2732 StackOffset ScalableOffset = {};
2733 if (UseFP && !(isFixed || isCSR))
2734 ScalableOffset = -SVEStackSize;
2735 if (!UseFP && (isFixed || isCSR))
2736 ScalableOffset = SVEStackSize;
2737
2738 if (UseFP) {
2739 FrameReg = RegInfo->getFrameRegister(MF);
2740 return StackOffset::getFixed(FPOffset) + ScalableOffset;
2741 }
2742
2743 // Use the base pointer if we have one.
2744 if (RegInfo->hasBasePointer(MF))
2745 FrameReg = RegInfo->getBaseRegister();
2746 else {
2747 assert(!MFI.hasVarSizedObjects() &&
2748 "Can't use SP when we have var sized objects.");
2749 FrameReg = AArch64::SP;
2750 // If we're using the red zone for this function, the SP won't actually
2751 // be adjusted, so the offsets will be negative. They're also all
2752 // within range of the signed 9-bit immediate instructions.
2753 if (canUseRedZone(MF))
2754 Offset -= AFI->getLocalStackSize();
2755 }
2756
2757 return StackOffset::getFixed(Offset) + ScalableOffset;
2758}
2759
2760static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
2761 // Do not set a kill flag on values that are also marked as live-in. This
2762 // happens with the @llvm-returnaddress intrinsic and with arguments passed in
2763 // callee saved registers.
2764 // Omitting the kill flags is conservatively correct even if the live-in
2765 // is not used after all.
2766 bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
2767 return getKillRegState(!IsLiveIn);
2768}
2769
2771 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2773 return Subtarget.isTargetMachO() &&
2774 !(Subtarget.getTargetLowering()->supportSwiftError() &&
2775 Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
2777}
2778
2779static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
2780 bool NeedsWinCFI, bool IsFirst,
2781 const TargetRegisterInfo *TRI) {
2782 // If we are generating register pairs for a Windows function that requires
2783 // EH support, then pair consecutive registers only. There are no unwind
2784 // opcodes for saves/restores of non-consectuve register pairs.
2785 // The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x,
2786 // save_lrpair.
2787 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
2788
2789 if (Reg2 == AArch64::FP)
2790 return true;
2791 if (!NeedsWinCFI)
2792 return false;
2793 if (TRI->getEncodingValue(Reg2) == TRI->getEncodingValue(Reg1) + 1)
2794 return false;
2795 // If pairing a GPR with LR, the pair can be described by the save_lrpair
2796 // opcode. If this is the first register pair, it would end up with a
2797 // predecrement, but there's no save_lrpair_x opcode, so we can only do this
2798 // if LR is paired with something else than the first register.
2799 // The save_lrpair opcode requires the first register to be an odd one.
2800 if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
2801 (Reg1 - AArch64::X19) % 2 == 0 && Reg2 == AArch64::LR && !IsFirst)
2802 return false;
2803 return true;
2804}
2805
2806/// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
2807/// WindowsCFI requires that only consecutive registers can be paired.
2808/// LR and FP need to be allocated together when the frame needs to save
2809/// the frame-record. This means any other register pairing with LR is invalid.
2810static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2,
2811 bool UsesWinAAPCS, bool NeedsWinCFI,
2812 bool NeedsFrameRecord, bool IsFirst,
2813 const TargetRegisterInfo *TRI) {
2814 if (UsesWinAAPCS)
2815 return invalidateWindowsRegisterPairing(Reg1, Reg2, NeedsWinCFI, IsFirst,
2816 TRI);
2817
2818 // If we need to store the frame record, don't pair any register
2819 // with LR other than FP.
2820 if (NeedsFrameRecord)
2821 return Reg2 == AArch64::LR;
2822
2823 return false;
2824}
2825
2826namespace {
2827
2828struct RegPairInfo {
2829 unsigned Reg1 = AArch64::NoRegister;
2830 unsigned Reg2 = AArch64::NoRegister;
2831 int FrameIdx;
2832 int Offset;
2833 enum RegType { GPR, FPR64, FPR128, PPR, ZPR, VG } Type;
2834
2835 RegPairInfo() = default;
2836
2837 bool isPaired() const { return Reg2 != AArch64::NoRegister; }
2838
2839 unsigned getScale() const {
2840 switch (Type) {
2841 case PPR:
2842 return 2;
2843 case GPR:
2844 case FPR64:
2845 case VG:
2846 return 8;
2847 case ZPR:
2848 case FPR128:
2849 return 16;
2850 }
2851 llvm_unreachable("Unsupported type");
2852 }
2853
2854 bool isScalable() const { return Type == PPR || Type == ZPR; }
2855};
2856
2857} // end anonymous namespace
2858
2859unsigned findFreePredicateReg(BitVector &SavedRegs) {
2860 for (unsigned PReg = AArch64::P8; PReg <= AArch64::P15; ++PReg) {
2861 if (SavedRegs.test(PReg)) {
2862 unsigned PNReg = PReg - AArch64::P0 + AArch64::PN0;
2863 return PNReg;
2864 }
2865 }
2866 return AArch64::NoRegister;
2867}
2868
2872 bool NeedsFrameRecord) {
2873
2874 if (CSI.empty())
2875 return;
2876
2877 bool IsWindows = isTargetWindows(MF);
2878 bool NeedsWinCFI = needsWinCFI(MF);
2880 MachineFrameInfo &MFI = MF.getFrameInfo();
2882 unsigned Count = CSI.size();
2883 (void)CC;
2884 // MachO's compact unwind format relies on all registers being stored in
2885 // pairs.
2888 CC == CallingConv::Win64 || (Count & 1) == 0) &&
2889 "Odd number of callee-saved regs to spill!");
2890 int ByteOffset = AFI->getCalleeSavedStackSize();
2891 int StackFillDir = -1;
2892 int RegInc = 1;
2893 unsigned FirstReg = 0;
2894 if (NeedsWinCFI) {
2895 // For WinCFI, fill the stack from the bottom up.
2896 ByteOffset = 0;
2897 StackFillDir = 1;
2898 // As the CSI array is reversed to match PrologEpilogInserter, iterate
2899 // backwards, to pair up registers starting from lower numbered registers.
2900 RegInc = -1;
2901 FirstReg = Count - 1;
2902 }
2903 int ScalableByteOffset = AFI->getSVECalleeSavedStackSize();
2904 bool NeedGapToAlignStack = AFI->hasCalleeSaveStackFreeSpace();
2905
2906 // When iterating backwards, the loop condition relies on unsigned wraparound.
2907 for (unsigned i = FirstReg; i < Count; i += RegInc) {
2908 RegPairInfo RPI;
2909 RPI.Reg1 = CSI[i].getReg();
2910
2911 if (AArch64::GPR64RegClass.contains(RPI.Reg1))
2912 RPI.Type = RegPairInfo::GPR;
2913 else if (AArch64::FPR64RegClass.contains(RPI.Reg1))
2914 RPI.Type = RegPairInfo::FPR64;
2915 else if (AArch64::FPR128RegClass.contains(RPI.Reg1))
2916 RPI.Type = RegPairInfo::FPR128;
2917 else if (AArch64::ZPRRegClass.contains(RPI.Reg1))
2918 RPI.Type = RegPairInfo::ZPR;
2919 else if (AArch64::PPRRegClass.contains(RPI.Reg1))
2920 RPI.Type = RegPairInfo::PPR;
2921 else if (RPI.Reg1 == AArch64::VG)
2922 RPI.Type = RegPairInfo::VG;
2923 else
2924 llvm_unreachable("Unsupported register class.");
2925
2926 // Add the next reg to the pair if it is in the same register class.
2927 if (unsigned(i + RegInc) < Count) {
2928 Register NextReg = CSI[i + RegInc].getReg();
2929 bool IsFirst = i == FirstReg;
2930 switch (RPI.Type) {
2931 case RegPairInfo::GPR:
2932 if (AArch64::GPR64RegClass.contains(NextReg) &&
2933 !invalidateRegisterPairing(RPI.Reg1, NextReg, IsWindows,
2934 NeedsWinCFI, NeedsFrameRecord, IsFirst,
2935 TRI))
2936 RPI.Reg2 = NextReg;
2937 break;
2938 case RegPairInfo::FPR64:
2939 if (AArch64::FPR64RegClass.contains(NextReg) &&
2940 !invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI,
2941 IsFirst, TRI))
2942 RPI.Reg2 = NextReg;
2943 break;
2944 case RegPairInfo::FPR128:
2945 if (AArch64::FPR128RegClass.contains(NextReg))
2946 RPI.Reg2 = NextReg;
2947 break;
2948 case RegPairInfo::PPR:
2949 break;
2950 case RegPairInfo::ZPR:
2951 if (AFI->getPredicateRegForFillSpill() != 0)
2952 if (((RPI.Reg1 - AArch64::Z0) & 1) == 0 && (NextReg == RPI.Reg1 + 1))
2953 RPI.Reg2 = NextReg;
2954 break;
2955 case RegPairInfo::VG:
2956 break;
2957 }
2958 }
2959
2960 // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
2961 // list to come in sorted by frame index so that we can issue the store
2962 // pair instructions directly. Assert if we see anything otherwise.
2963 //
2964 // The order of the registers in the list is controlled by
2965 // getCalleeSavedRegs(), so they will always be in-order, as well.
2966 assert((!RPI.isPaired() ||
2967 (CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
2968 "Out of order callee saved regs!");
2969
2970 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
2971 RPI.Reg1 == AArch64::LR) &&
2972 "FrameRecord must be allocated together with LR");
2973
2974 // Windows AAPCS has FP and LR reversed.
2975 assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
2976 RPI.Reg2 == AArch64::LR) &&
2977 "FrameRecord must be allocated together with LR");
2978
2979 // MachO's compact unwind format relies on all registers being stored in
2980 // adjacent register pairs.
2984 (RPI.isPaired() &&
2985 ((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
2986 RPI.Reg1 + 1 == RPI.Reg2))) &&
2987 "Callee-save registers not saved as adjacent register pair!");
2988
2989 RPI.FrameIdx = CSI[i].getFrameIdx();
2990 if (NeedsWinCFI &&
2991 RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
2992 RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
2993 int Scale = RPI.getScale();
2994
2995 int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
2996 assert(OffsetPre % Scale == 0);
2997
2998 if (RPI.isScalable())
2999 ScalableByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
3000 else
3001 ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
3002
3003 // Swift's async context is directly before FP, so allocate an extra
3004 // 8 bytes for it.
3005 if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
3006 ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
3007 (IsWindows && RPI.Reg2 == AArch64::LR)))
3008 ByteOffset += StackFillDir * 8;
3009
3010 // Round up size of non-pair to pair size if we need to pad the
3011 // callee-save area to ensure 16-byte alignment.
3012 if (NeedGapToAlignStack && !NeedsWinCFI &&
3013 !RPI.isScalable() && RPI.Type != RegPairInfo::FPR128 &&
3014 !RPI.isPaired() && ByteOffset % 16 != 0) {
3015 ByteOffset += 8 * StackFillDir;
3016 assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
3017 // A stack frame with a gap looks like this, bottom up:
3018 // d9, d8. x21, gap, x20, x19.
3019 // Set extra alignment on the x21 object to create the gap above it.
3020 MFI.setObjectAlignment(RPI.FrameIdx, Align(16));
3021 NeedGapToAlignStack = false;
3022 }
3023
3024 int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
3025 assert(OffsetPost % Scale == 0);
3026 // If filling top down (default), we want the offset after incrementing it.
3027 // If filling bottom up (WinCFI) we need the original offset.
3028 int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
3029
3030 // The FP, LR pair goes 8 bytes into our expanded 24-byte slot so that the
3031 // Swift context can directly precede FP.
3032 if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
3033 ((!IsWindows && RPI.Reg2 == AArch64::FP) ||
3034 (IsWindows && RPI.Reg2 == AArch64::LR)))
3035 Offset += 8;
3036 RPI.Offset = Offset / Scale;
3037
3038 assert(((!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
3039 (RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
3040 "Offset out of bounds for LDP/STP immediate");
3041
3042 // Save the offset to frame record so that the FP register can point to the
3043 // innermost frame record (spilled FP and LR registers).
3044 if (NeedsFrameRecord && ((!IsWindows && RPI.Reg1 == AArch64::LR &&
3045 RPI.Reg2 == AArch64::FP) ||
3046 (IsWindows && RPI.Reg1 == AArch64::FP &&
3047 RPI.Reg2 == AArch64::LR)))
3049
3050 RegPairs.push_back(RPI);
3051 if (RPI.isPaired())
3052 i += RegInc;
3053 }
3054 if (NeedsWinCFI) {
3055 // If we need an alignment gap in the stack, align the topmost stack
3056 // object. A stack frame with a gap looks like this, bottom up:
3057 // x19, d8. d9, gap.
3058 // Set extra alignment on the topmost stack object (the first element in
3059 // CSI, which goes top down), to create the gap above it.
3060 if (AFI->hasCalleeSaveStackFreeSpace())
3061 MFI.setObjectAlignment(CSI[0].getFrameIdx(), Align(16));
3062 // We iterated bottom up over the registers; flip RegPairs back to top
3063 // down order.
3064 std::reverse(RegPairs.begin(), RegPairs.end());
3065 }
3066}
3067
3071 MachineFunction &MF = *MBB.getParent();
3074 bool NeedsWinCFI = needsWinCFI(MF);
3075 DebugLoc DL;
3077
3078 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF));
3079
3080 const MachineRegisterInfo &MRI = MF.getRegInfo();
3081 if (homogeneousPrologEpilog(MF)) {
3082 auto MIB = BuildMI(MBB, MI, DL, TII.get(AArch64::HOM_Prolog))
3084
3085 for (auto &RPI : RegPairs) {
3086 MIB.addReg(RPI.Reg1);
3087 MIB.addReg(RPI.Reg2);
3088
3089 // Update register live in.
3090 if (!MRI.isReserved(RPI.Reg1))
3091 MBB.addLiveIn(RPI.Reg1);
3092 if (RPI.isPaired() && !MRI.isReserved(RPI.Reg2))
3093 MBB.addLiveIn(RPI.Reg2);
3094 }
3095 return true;
3096 }
3097 bool PTrueCreated = false;
3098 for (const RegPairInfo &RPI : llvm::reverse(RegPairs)) {
3099 unsigned Reg1 = RPI.Reg1;
3100 unsigned Reg2 = RPI.Reg2;
3101 unsigned StrOpc;
3102
3103 // Issue sequence of spills for cs regs. The first spill may be converted
3104 // to a pre-decrement store later by emitPrologue if the callee-save stack
3105 // area allocation can't be combined with the local stack area allocation.
3106 // For example:
3107 // stp x22, x21, [sp, #0] // addImm(+0)
3108 // stp x20, x19, [sp, #16] // addImm(+2)
3109 // stp fp, lr, [sp, #32] // addImm(+4)
3110 // Rationale: This sequence saves uop updates compared to a sequence of
3111 // pre-increment spills like stp xi,xj,[sp,#-16]!
3112 // Note: Similar rationale and sequence for restores in epilog.
3113 unsigned Size;
3114 Align Alignment;
3115 switch (RPI.Type) {
3116 case RegPairInfo::GPR:
3117 StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
3118 Size = 8;
3119 Alignment = Align(8);
3120 break;
3121 case RegPairInfo::FPR64:
3122 StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
3123 Size = 8;
3124 Alignment = Align(8);
3125 break;
3126 case RegPairInfo::FPR128:
3127 StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
3128 Size = 16;
3129 Alignment = Align(16);
3130 break;
3131 case RegPairInfo::ZPR:
3132 StrOpc = RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI;
3133 Size = 16;
3134 Alignment = Align(16);
3135 break;
3136 case RegPairInfo::PPR:
3137 StrOpc = AArch64::STR_PXI;
3138 Size = 2;
3139 Alignment = Align(2);
3140 break;
3141 case RegPairInfo::VG:
3142 StrOpc = AArch64::STRXui;
3143 Size = 8;
3144 Alignment = Align(8);
3145 break;
3146 }
3147
3148 unsigned X0Scratch = AArch64::NoRegister;
3149 if (Reg1 == AArch64::VG) {
3150 // Find an available register to store value of VG to.
3152 assert(Reg1 != AArch64::NoRegister);
3153 SMEAttrs Attrs(MF.getFunction());
3154
3155 if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface() &&
3156 AFI->getStreamingVGIdx() == std::numeric_limits<int>::max()) {
3157 // For locally-streaming functions, we need to store both the streaming
3158 // & non-streaming VG. Spill the streaming value first.
3159 BuildMI(MBB, MI, DL, TII.get(AArch64::RDSVLI_XI), Reg1)
3160 .addImm(1)
3162 BuildMI(MBB, MI, DL, TII.get(AArch64::UBFMXri), Reg1)
3163 .addReg(Reg1)
3164 .addImm(3)
3165 .addImm(63)
3167
3168 AFI->setStreamingVGIdx(RPI.FrameIdx);
3169 } else if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
3170 BuildMI(MBB, MI, DL, TII.get(AArch64::CNTD_XPiI), Reg1)
3171 .addImm(31)
3172 .addImm(1)
3174 AFI->setVGIdx(RPI.FrameIdx);
3175 } else {
3177 if (llvm::any_of(
3178 MBB.liveins(),
3179 [&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) {
3180 return STI.getRegisterInfo()->isSuperOrSubRegisterEq(
3181 AArch64::X0, LiveIn.PhysReg);
3182 }))
3183 X0Scratch = Reg1;
3184
3185 if (X0Scratch != AArch64::NoRegister)
3186 BuildMI(MBB, MI, DL, TII.get(AArch64::ORRXrr), Reg1)
3187 .addReg(AArch64::XZR)
3188 .addReg(AArch64::X0, RegState::Undef)
3189 .addReg(AArch64::X0, RegState::Implicit)
3191
3192 const uint32_t *RegMask = TRI->getCallPreservedMask(
3193 MF,
3195 BuildMI(MBB, MI, DL, TII.get(AArch64::BL))
3196 .addExternalSymbol("__arm_get_current_vg")
3197 .addRegMask(RegMask)
3198 .addReg(AArch64::X0, RegState::ImplicitDefine)
3200 Reg1 = AArch64::X0;
3201 AFI->setVGIdx(RPI.FrameIdx);
3202 }
3203 }
3204
3205 LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
3206 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
3207 dbgs() << ") -> fi#(" << RPI.FrameIdx;
3208 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
3209 dbgs() << ")\n");
3210
3211 assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
3212 "Windows unwdinding requires a consecutive (FP,LR) pair");
3213 // Windows unwind codes require consecutive registers if registers are
3214 // paired. Make the switch here, so that the code below will save (x,x+1)
3215 // and not (x+1,x).
3216 unsigned FrameIdxReg1 = RPI.FrameIdx;
3217 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
3218 if (NeedsWinCFI && RPI.isPaired()) {
3219 std::swap(Reg1, Reg2);
3220 std::swap(FrameIdxReg1, FrameIdxReg2);
3221 }
3222
3223 if (RPI.isPaired() && RPI.isScalable()) {
3224 [[maybe_unused]] const AArch64Subtarget &Subtarget =
3227 unsigned PnReg = AFI->getPredicateRegForFillSpill();
3228 assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
3229 "Expects SVE2.1 or SME2 target and a predicate register");
3230#ifdef EXPENSIVE_CHECKS
3231 auto IsPPR = [](const RegPairInfo &c) {
3232 return c.Reg1 == RegPairInfo::PPR;
3233 };
3234 auto PPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsPPR);
3235 auto IsZPR = [](const RegPairInfo &c) {
3236 return c.Type == RegPairInfo::ZPR;
3237 };
3238 auto ZPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsZPR);
3239 assert(!(PPRBegin < ZPRBegin) &&
3240 "Expected callee save predicate to be handled first");
3241#endif
3242 if (!PTrueCreated) {
3243 PTrueCreated = true;
3244 BuildMI(MBB, MI, DL, TII.get(AArch64::PTRUE_C_B), PnReg)
3246 }
3247 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
3248 if (!MRI.isReserved(Reg1))
3249 MBB.addLiveIn(Reg1);
3250 if (!MRI.isReserved(Reg2))
3251 MBB.addLiveIn(Reg2);
3252 MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0));
3254 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
3255 MachineMemOperand::MOStore, Size, Alignment));
3256 MIB.addReg(PnReg);
3257 MIB.addReg(AArch64::SP)
3258 .addImm(RPI.Offset) // [sp, #offset*scale],
3259 // where factor*scale is implicit
3262 MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
3263 MachineMemOperand::MOStore, Size, Alignment));
3264 if (NeedsWinCFI)
3266 } else { // The code when the pair of ZReg is not present
3267 MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
3268 if (!MRI.isReserved(Reg1))
3269 MBB.addLiveIn(Reg1);
3270 if (RPI.isPaired()) {
3271 if (!MRI.isReserved(Reg2))
3272 MBB.addLiveIn(Reg2);
3273 MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
3275 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
3276 MachineMemOperand::MOStore, Size, Alignment));
3277 }
3278 MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
3279 .addReg(AArch64::SP)
3280 .addImm(RPI.Offset) // [sp, #offset*scale],
3281 // where factor*scale is implicit
3284 MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
3285 MachineMemOperand::MOStore, Size, Alignment));
3286 if (NeedsWinCFI)
3288 }
3289 // Update the StackIDs of the SVE stack slots.
3290 MachineFrameInfo &MFI = MF.getFrameInfo();
3291 if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR) {
3292 MFI.setStackID(FrameIdxReg1, TargetStackID::ScalableVector);
3293 if (RPI.isPaired())
3294 MFI.setStackID(FrameIdxReg2, TargetStackID::ScalableVector);
3295 }
3296
3297 if (X0Scratch != AArch64::NoRegister)
3298 BuildMI(MBB, MI, DL, TII.get(AArch64::ORRXrr), AArch64::X0)
3299 .addReg(AArch64::XZR)
3300 .addReg(X0Scratch, RegState::Undef)
3301 .addReg(X0Scratch, RegState::Implicit)
3303 }
3304 return true;
3305}
3306
3310 MachineFunction &MF = *MBB.getParent();
3312 DebugLoc DL;
3314 bool NeedsWinCFI = needsWinCFI(MF);
3315
3316 if (MBBI != MBB.end())
3317 DL = MBBI->getDebugLoc();
3318
3319 computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF));
3320 if (homogeneousPrologEpilog(MF, &MBB)) {
3321 auto MIB = BuildMI(MBB, MBBI, DL, TII.get(AArch64::HOM_Epilog))
3323 for (auto &RPI : RegPairs) {
3324 MIB.addReg(RPI.Reg1, RegState::Define);
3325 MIB.addReg(RPI.Reg2, RegState::Define);
3326 }
3327 return true;
3328 }
3329
3330 // For performance reasons restore SVE register in increasing order
3331 auto IsPPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::PPR; };
3332 auto PPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsPPR);
3333 auto PPREnd = std::find_if_not(PPRBegin, RegPairs.end(), IsPPR);
3334 std::reverse(PPRBegin, PPREnd);
3335 auto IsZPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::ZPR; };
3336 auto ZPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsZPR);
3337 auto ZPREnd = std::find_if_not(ZPRBegin, RegPairs.end(), IsZPR);
3338 std::reverse(ZPRBegin, ZPREnd);
3339
3340 bool PTrueCreated = false;
3341 for (const RegPairInfo &RPI : RegPairs) {
3342 unsigned Reg1 = RPI.Reg1;
3343 unsigned Reg2 = RPI.Reg2;
3344
3345 // Issue sequence of restores for cs regs. The last restore may be converted
3346 // to a post-increment load later by emitEpilogue if the callee-save stack
3347 // area allocation can't be combined with the local stack area allocation.
3348 // For example:
3349 // ldp fp, lr, [sp, #32] // addImm(+4)
3350 // ldp x20, x19, [sp, #16] // addImm(+2)
3351 // ldp x22, x21, [sp, #0] // addImm(+0)
3352 // Note: see comment in spillCalleeSavedRegisters()
3353 unsigned LdrOpc;
3354 unsigned Size;
3355 Align Alignment;
3356 switch (RPI.Type) {
3357 case RegPairInfo::GPR:
3358 LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
3359 Size = 8;
3360 Alignment = Align(8);
3361 break;
3362 case RegPairInfo::FPR64:
3363 LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
3364 Size = 8;
3365 Alignment = Align(8);
3366 break;
3367 case RegPairInfo::FPR128:
3368 LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
3369 Size = 16;
3370 Alignment = Align(16);
3371 break;
3372 case RegPairInfo::ZPR:
3373 LdrOpc = RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI;
3374 Size = 16;
3375 Alignment = Align(16);
3376 break;
3377 case RegPairInfo::PPR:
3378 LdrOpc = AArch64::LDR_PXI;
3379 Size = 2;
3380 Alignment = Align(2);
3381 break;
3382 case RegPairInfo::VG:
3383 continue;
3384 }
3385 LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
3386 if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
3387 dbgs() << ") -> fi#(" << RPI.FrameIdx;
3388 if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
3389 dbgs() << ")\n");
3390
3391 // Windows unwind codes require consecutive registers if registers are
3392 // paired. Make the switch here, so that the code below will save (x,x+1)
3393 // and not (x+1,x).
3394 unsigned FrameIdxReg1 = RPI.FrameIdx;
3395 unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
3396 if (NeedsWinCFI && RPI.isPaired()) {
3397 std::swap(Reg1, Reg2);
3398 std::swap(FrameIdxReg1, FrameIdxReg2);
3399 }
3400
3402 if (RPI.isPaired() && RPI.isScalable()) {
3403 [[maybe_unused]] const AArch64Subtarget &Subtarget =
3405 unsigned PnReg = AFI->getPredicateRegForFillSpill();
3406 assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
3407 "Expects SVE2.1 or SME2 target and a predicate register");
3408#ifdef EXPENSIVE_CHECKS
3409 assert(!(PPRBegin < ZPRBegin) &&
3410 "Expected callee save predicate to be handled first");
3411#endif
3412 if (!PTrueCreated) {
3413 PTrueCreated = true;
3414 BuildMI(MBB, MBBI, DL, TII.get(AArch64::PTRUE_C_B), PnReg)
3416 }
3417 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
3418 MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0),
3419 getDefRegState(true));
3421 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
3422 MachineMemOperand::MOLoad, Size, Alignment));
3423 MIB.addReg(PnReg);
3424 MIB.addReg(AArch64::SP)
3425 .addImm(RPI.Offset) // [sp, #offset*scale]
3426 // where factor*scale is implicit
3429 MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
3430 MachineMemOperand::MOLoad, Size, Alignment));
3431 if (NeedsWinCFI)
3433 } else {
3434 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
3435 if (RPI.isPaired()) {
3436 MIB.addReg(Reg2, getDefRegState(true));
3438 MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
3439 MachineMemOperand::MOLoad, Size, Alignment));
3440 }
3441 MIB.addReg(Reg1, getDefRegState(true));
3442 MIB.addReg(AArch64::SP)
3443 .addImm(RPI.Offset) // [sp, #offset*scale]
3444 // where factor*scale is implicit
3447 MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
3448 MachineMemOperand::MOLoad, Size, Alignment));
3449 if (NeedsWinCFI)
3451 }
3452 }
3453 return true;
3454}
3455
3457 BitVector &SavedRegs,
3458 RegScavenger *RS) const {
3459 // All calls are tail calls in GHC calling conv, and functions have no
3460 // prologue/epilogue.
3462 return;
3463
3465 const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
3467 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
3469 unsigned UnspilledCSGPR = AArch64::NoRegister;
3470 unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
3471
3472 MachineFrameInfo &MFI = MF.getFrameInfo();
3473 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
3474
3475 unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
3476 ? RegInfo->getBaseRegister()
3477 : (unsigned)AArch64::NoRegister;
3478
3479 unsigned ExtraCSSpill = 0;
3480 bool HasUnpairedGPR64 = false;
3481 bool HasPairZReg = false;
3482 // Figure out which callee-saved registers to save/restore.
3483 for (unsigned i = 0; CSRegs[i]; ++i) {
3484 const unsigned Reg = CSRegs[i];
3485
3486 // Add the base pointer register to SavedRegs if it is callee-save.
3487 if (Reg == BasePointerReg)
3488 SavedRegs.set(Reg);
3489
3490 bool RegUsed = SavedRegs.test(Reg);
3491 unsigned PairedReg = AArch64::NoRegister;
3492 const bool RegIsGPR64 = AArch64::GPR64RegClass.contains(Reg);
3493 if (RegIsGPR64 || AArch64::FPR64RegClass.contains(Reg) ||
3494 AArch64::FPR128RegClass.contains(Reg)) {
3495 // Compensate for odd numbers of GP CSRs.
3496 // For now, all the known cases of odd number of CSRs are of GPRs.
3497 if (HasUnpairedGPR64)
3498 PairedReg = CSRegs[i % 2 == 0 ? i - 1 : i + 1];
3499 else
3500 PairedReg = CSRegs[i ^ 1];
3501 }
3502
3503 // If the function requires all the GP registers to save (SavedRegs),
3504 // and there are an odd number of GP CSRs at the same time (CSRegs),
3505 // PairedReg could be in a different register class from Reg, which would
3506 // lead to a FPR (usually D8) accidentally being marked saved.
3507 if (RegIsGPR64 && !AArch64::GPR64RegClass.contains(PairedReg)) {
3508 PairedReg = AArch64::NoRegister;
3509 HasUnpairedGPR64 = true;
3510 }
3511 assert(PairedReg == AArch64::NoRegister ||
3512 AArch64::GPR64RegClass.contains(Reg, PairedReg) ||
3513 AArch64::FPR64RegClass.contains(Reg, PairedReg) ||
3514 AArch64::FPR128RegClass.contains(Reg, PairedReg));
3515
3516 if (!RegUsed) {
3517 if (AArch64::GPR64RegClass.contains(Reg) &&
3518 !RegInfo->isReservedReg(MF, Reg)) {
3519 UnspilledCSGPR = Reg;
3520 UnspilledCSGPRPaired = PairedReg;
3521 }
3522 continue;
3523 }
3524
3525 // MachO's compact unwind format relies on all registers being stored in
3526 // pairs.
3527 // FIXME: the usual format is actually better if unwinding isn't needed.
3528 if (producePairRegisters(MF) && PairedReg != AArch64::NoRegister &&
3529 !SavedRegs.test(PairedReg)) {
3530 SavedRegs.set(PairedReg);
3531 if (AArch64::GPR64RegClass.contains(PairedReg) &&
3532 !RegInfo->isReservedReg(MF, PairedReg))
3533 ExtraCSSpill = PairedReg;
3534 }
3535 // Check if there is a pair of ZRegs, so it can select PReg for spill/fill
3536 HasPairZReg |= (AArch64::ZPRRegClass.contains(Reg, CSRegs[i ^ 1]) &&
3537 SavedRegs.test(CSRegs[i ^ 1]));
3538 }
3539
3540 if (HasPairZReg && (Subtarget.hasSVE2p1() || Subtarget.hasSME2())) {
3542 // Find a suitable predicate register for the multi-vector spill/fill
3543 // instructions.
3544 unsigned PnReg = findFreePredicateReg(SavedRegs);
3545 if (PnReg != AArch64::NoRegister)
3546 AFI->setPredicateRegForFillSpill(PnReg);
3547 // If no free callee-save has been found assign one.
3548 if (!AFI->getPredicateRegForFillSpill() &&
3549 MF.getFunction().getCallingConv() ==
3551 SavedRegs.set(AArch64::P8);
3552 AFI->setPredicateRegForFillSpill(AArch64::PN8);
3553 }
3554
3555 assert(!RegInfo->isReservedReg(MF, AFI->getPredicateRegForFillSpill()) &&
3556 "Predicate cannot be a reserved register");
3557 }
3558
3560 !Subtarget.isTargetWindows()) {
3561 // For Windows calling convention on a non-windows OS, where X18 is treated
3562 // as reserved, back up X18 when entering non-windows code (marked with the
3563 // Windows calling convention) and restore when returning regardless of
3564 // whether the individual function uses it - it might call other functions
3565 // that clobber it.
3566 SavedRegs.set(AArch64::X18);
3567 }
3568
3569 // Calculates the callee saved stack size.
3570 unsigned CSStackSize = 0;
3571 unsigned SVECSStackSize = 0;
3573 const MachineRegisterInfo &MRI = MF.getRegInfo();
3574 for (unsigned Reg : SavedRegs.set_bits()) {
3575 auto RegSize = TRI->getRegSizeInBits(Reg, MRI) / 8;
3576 if (AArch64::PPRRegClass.contains(Reg) ||
3577 AArch64::ZPRRegClass.contains(Reg))
3578 SVECSStackSize += RegSize;
3579 else
3580 CSStackSize += RegSize;
3581 }
3582
3583 // Increase the callee-saved stack size if the function has streaming mode
3584 // changes, as we will need to spill the value of the VG register.
3585 // For locally streaming functions, we spill both the streaming and
3586 // non-streaming VG value.
3587 const Function &F = MF.getFunction();
3588 SMEAttrs Attrs(F);
3589 if (AFI->hasStreamingModeChanges()) {
3590 if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
3591 CSStackSize += 16;
3592 else
3593 CSStackSize += 8;
3594 }
3595
3596 // Save number of saved regs, so we can easily update CSStackSize later.
3597 unsigned NumSavedRegs = SavedRegs.count();
3598
3599 // The frame record needs to be created by saving the appropriate registers
3600 uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
3601 if (hasFP(MF) ||
3602 windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
3603 SavedRegs.set(AArch64::FP);
3604 SavedRegs.set(AArch64::LR);
3605 }
3606
3607 LLVM_DEBUG(dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
3608 for (unsigned Reg
3609 : SavedRegs.set_bits()) dbgs()
3610 << ' ' << printReg(Reg, RegInfo);
3611 dbgs() << "\n";);
3612
3613 // If any callee-saved registers are used, the frame cannot be eliminated.
3614 int64_t SVEStackSize =
3615 alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
3616 bool CanEliminateFrame = (SavedRegs.count() == 0) && !SVEStackSize;
3617
3618 // The CSR spill slots have not been allocated yet, so estimateStackSize
3619 // won't include them.
3620 unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
3621
3622 // We may address some of the stack above the canonical frame address, either
3623 // for our own arguments or during a call. Include that in calculating whether
3624 // we have complicated addressing concerns.
3625 int64_t CalleeStackUsed = 0;
3626 for (int I = MFI.getObjectIndexBegin(); I != 0; ++I) {
3627 int64_t FixedOff = MFI.getObjectOffset(I);
3628 if (FixedOff > CalleeStackUsed) CalleeStackUsed = FixedOff;
3629 }
3630
3631 // Conservatively always assume BigStack when there are SVE spills.
3632 bool BigStack = SVEStackSize || (EstimatedStackSize + CSStackSize +
3633 CalleeStackUsed) > EstimatedStackSizeLimit;
3634 if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
3635 AFI->setHasStackFrame(true);
3636
3637 // Estimate if we might need to scavenge a register at some point in order
3638 // to materialize a stack offset. If so, either spill one additional
3639 // callee-saved register or reserve a special spill slot to facilitate
3640 // register scavenging. If we already spilled an extra callee-saved register
3641 // above to keep the number of spills even, we don't need to do anything else
3642 // here.
3643 if (BigStack) {
3644 if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
3645 LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
3646 << " to get a scratch register.\n");
3647 SavedRegs.set(UnspilledCSGPR);
3648 ExtraCSSpill = UnspilledCSGPR;
3649
3650 // MachO's compact unwind format relies on all registers being stored in
3651 // pairs, so if we need to spill one extra for BigStack, then we need to
3652 // store the pair.
3653 if (producePairRegisters(MF)) {
3654 if (UnspilledCSGPRPaired == AArch64::NoRegister) {
3655 // Failed to make a pair for compact unwind format, revert spilling.
3656 if (produceCompactUnwindFrame(MF)) {
3657 SavedRegs.reset(UnspilledCSGPR);
3658 ExtraCSSpill = AArch64::NoRegister;
3659 }
3660 } else
3661 SavedRegs.set(UnspilledCSGPRPaired);
3662 }
3663 }
3664
3665 // If we didn't find an extra callee-saved register to spill, create
3666 // an emergency spill slot.
3667 if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
3669 const TargetRegisterClass &RC = AArch64::GPR64RegClass;
3670 unsigned Size = TRI->getSpillSize(RC);
3671 Align Alignment = TRI->getSpillAlign(RC);
3672 int FI = MFI.CreateStackObject(Size, Alignment, false);
3674 LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
3675 << " as the emergency spill slot.\n");
3676 }
3677 }
3678
3679 // Adding the size of additional 64bit GPR saves.
3680 CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
3681
3682 // A Swift asynchronous context extends the frame record with a pointer
3683 // directly before FP.
3684 if (hasFP(MF) && AFI->hasSwiftAsyncContext())
3685 CSStackSize += 8;
3686
3687 uint64_t AlignedCSStackSize = alignTo(CSStackSize, 16);
3688 LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
3689 << EstimatedStackSize + AlignedCSStackSize
3690 << " bytes.\n");
3691
3693 AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
3694 "Should not invalidate callee saved info");
3695
3696 // Round up to register pair alignment to avoid additional SP adjustment
3697 // instructions.
3698 AFI->setCalleeSavedStackSize(AlignedCSStackSize);
3699 AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
3700 AFI->setSVECalleeSavedStackSize(alignTo(SVECSStackSize, 16));
3701}
3702
3704 MachineFunction &MF, const TargetRegisterInfo *RegInfo,
3705 std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
3706 unsigned &MaxCSFrameIndex) const {
3707 bool NeedsWinCFI = needsWinCFI(MF);
3708 // To match the canonical windows frame layout, reverse the list of
3709 // callee saved registers to get them laid out by PrologEpilogInserter
3710 // in the right order. (PrologEpilogInserter allocates stack objects top
3711 // down. Windows canonical prologs store higher numbered registers at
3712 // the top, thus have the CSI array start from the highest registers.)
3713 if (NeedsWinCFI)
3714 std::reverse(CSI.begin(), CSI.end());
3715
3716 if (CSI.empty())
3717 return true; // Early exit if no callee saved registers are modified!
3718
3719 // Now that we know which registers need to be saved and restored, allocate
3720 // stack slots for them.
3721 MachineFrameInfo &MFI = MF.getFrameInfo();
3722 auto *AFI = MF.getInfo<AArch64FunctionInfo>();
3723
3724 bool UsesWinAAPCS = isTargetWindows(MF);
3725 if (UsesWinAAPCS && hasFP(MF) && AFI->hasSwiftAsyncContext()) {
3726 int FrameIdx = MFI.CreateStackObject(8, Align(16), true);
3727 AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
3728 if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3729 if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3730 }
3731
3732 // Insert VG into the list of CSRs, immediately before LR if saved.
3733 if (AFI->hasStreamingModeChanges()) {
3734 std::vector<CalleeSavedInfo> VGSaves;
3735 SMEAttrs Attrs(MF.getFunction());
3736
3737 auto VGInfo = CalleeSavedInfo(AArch64::VG);
3738 VGInfo.setRestored(false);
3739 VGSaves.push_back(VGInfo);
3740
3741 // Add VG again if the function is locally-streaming, as we will spill two
3742 // values.
3743 if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
3744 VGSaves.push_back(VGInfo);
3745
3746 bool InsertBeforeLR = false;
3747
3748 for (unsigned I = 0; I < CSI.size(); I++)
3749 if (CSI[I].getReg() == AArch64::LR) {
3750 InsertBeforeLR = true;
3751 CSI.insert(CSI.begin() + I, VGSaves.begin(), VGSaves.end());
3752 break;
3753 }
3754
3755 if (!InsertBeforeLR)
3756 CSI.insert(CSI.end(), VGSaves.begin(), VGSaves.end());
3757 }
3758
3759 for (auto &CS : CSI) {
3760 Register Reg = CS.getReg();
3761 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
3762
3763 unsigned Size = RegInfo->getSpillSize(*RC);
3764 Align Alignment(RegInfo->getSpillAlign(*RC));
3765 int FrameIdx = MFI.CreateStackObject(Size, Alignment, true);
3766 CS.setFrameIdx(FrameIdx);
3767
3768 if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3769 if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3770
3771 // Grab 8 bytes below FP for the extended asynchronous frame info.
3772 if (hasFP(MF) && AFI->hasSwiftAsyncContext() && !UsesWinAAPCS &&
3773 Reg == AArch64::FP) {
3774 FrameIdx = MFI.CreateStackObject(8, Alignment, true);
3775 AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
3776 if ((unsigned)FrameIdx < MinCSFrameIndex) MinCSFrameIndex = FrameIdx;
3777 if ((unsigned)FrameIdx > MaxCSFrameIndex) MaxCSFrameIndex = FrameIdx;
3778 }
3779 }
3780 return true;
3781}
3782
3784 const MachineFunction &MF) const {
3786 // If the function has streaming-mode changes, don't scavenge a
3787 // spillslot in the callee-save area, as that might require an
3788 // 'addvl' in the streaming-mode-changing call-sequence when the
3789 // function doesn't use a FP.
3790 if (AFI->hasStreamingModeChanges() && !hasFP(MF))
3791 return false;
3792 return AFI->hasCalleeSaveStackFreeSpace();
3793}
3794
3795/// returns true if there are any SVE callee saves.
3797 int &Min, int &Max) {
3798 Min = std::numeric_limits<int>::max();
3799 Max = std::numeric_limits<int>::min();
3800
3801 if (!MFI.isCalleeSavedInfoValid())
3802 return false;
3803
3804 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
3805 for (auto &CS : CSI) {
3806 if (AArch64::ZPRRegClass.contains(CS.getReg()) ||
3807 AArch64::PPRRegClass.contains(CS.getReg())) {
3808 assert((Max == std::numeric_limits<int>::min() ||
3809 Max + 1 == CS.getFrameIdx()) &&
3810 "SVE CalleeSaves are not consecutive");
3811
3812 Min = std::min(Min, CS.getFrameIdx());
3813 Max = std::max(Max, CS.getFrameIdx());
3814 }
3815 }
3816 return Min != std::numeric_limits<int>::max();
3817}
3818
3819// Process all the SVE stack objects and determine offsets for each
3820// object. If AssignOffsets is true, the offsets get assigned.
3821// Fills in the first and last callee-saved frame indices into
3822// Min/MaxCSFrameIndex, respectively.
3823// Returns the size of the stack.
3825 int &MinCSFrameIndex,
3826 int &MaxCSFrameIndex,
3827 bool AssignOffsets) {
3828#ifndef NDEBUG
3829 // First process all fixed stack objects.
3830 for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
3832 "SVE vectors should never be passed on the stack by value, only by "
3833 "reference.");
3834#endif
3835
3836 auto Assign = [&MFI](int FI, int64_t Offset) {
3837 LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
3838 MFI.setObjectOffset(FI, Offset);
3839 };
3840
3841 int64_t Offset = 0;
3842
3843 // Then process all callee saved slots.
3844 if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
3845 // Assign offsets to the callee save slots.
3846 for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
3847 Offset += MFI.getObjectSize(I);
3849 if (AssignOffsets)
3850 Assign(I, -Offset);
3851 }
3852 }
3853
3854 // Ensure that the Callee-save area is aligned to 16bytes.
3855 Offset = alignTo(Offset, Align(16U));
3856
3857 // Create a buffer of SVE objects to allocate and sort it.
3858 SmallVector<int, 8> ObjectsToAllocate;
3859 // If we have a stack protector, and we've previously decided that we have SVE
3860 // objects on the stack and thus need it to go in the SVE stack area, then it
3861 // needs to go first.
3862 int StackProtectorFI = -1;
3863 if (MFI.hasStackProtectorIndex()) {
3864 StackProtectorFI = MFI.getStackProtectorIndex();
3865 if (MFI.getStackID(StackProtectorFI) == TargetStackID::ScalableVector)
3866 ObjectsToAllocate.push_back(StackProtectorFI);
3867 }
3868 for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
3869 unsigned StackID = MFI.getStackID(I);
3870 if (StackID != TargetStackID::ScalableVector)
3871 continue;
3872 if (I == StackProtectorFI)
3873 continue;
3874 if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex)
3875 continue;
3876 if (MFI.isDeadObjectIndex(I))
3877 continue;
3878
3879 ObjectsToAllocate.push_back(I);
3880 }
3881
3882 // Allocate all SVE locals and spills
3883 for (unsigned FI : ObjectsToAllocate) {
3884 Align Alignment = MFI.getObjectAlign(FI);
3885 // FIXME: Given that the length of SVE vectors is not necessarily a power of
3886 // two, we'd need to align every object dynamically at runtime if the
3887 // alignment is larger than 16. This is not yet supported.
3888 if (Alignment > Align(16))
3890 "Alignment of scalable vectors > 16 bytes is not yet supported");
3891
3892 Offset = alignTo(Offset + MFI.getObjectSize(FI), Alignment);
3893 if (AssignOffsets)
3894 Assign(FI, -Offset);
3895 }
3896
3897 return Offset;
3898}
3899
3900int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
3901 MachineFrameInfo &MFI) const {
3902 int MinCSFrameIndex, MaxCSFrameIndex;
3903 return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex, false);
3904}
3905
3906int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
3907 MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex) const {
3908 return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex,
3909 true);
3910}
3911
3913 MachineFunction &MF, RegScavenger *RS) const {
3914 MachineFrameInfo &MFI = MF.getFrameInfo();
3915
3917 "Upwards growing stack unsupported");
3918
3919 int MinCSFrameIndex, MaxCSFrameIndex;
3920 int64_t SVEStackSize =
3921 assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
3922
3924 AFI->setStackSizeSVE(alignTo(SVEStackSize, 16U));
3925 AFI->setMinMaxSVECSFrameIndex(MinCSFrameIndex, MaxCSFrameIndex);
3926
3927 // If this function isn't doing Win64-style C++ EH, we don't need to do
3928 // anything.
3929 if (!MF.hasEHFunclets())
3930 return;
3932 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
3933
3934 MachineBasicBlock &MBB = MF.front();
3935 auto MBBI = MBB.begin();
3936 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
3937 ++MBBI;
3938
3939 // Create an UnwindHelp object.
3940 // The UnwindHelp object is allocated at the start of the fixed object area
3941 int64_t FixedObject =
3942 getFixedObjectSize(MF, AFI, /*IsWin64*/ true, /*IsFunclet*/ false);
3943 int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8,
3944 /*SPOffset*/ -FixedObject,
3945 /*IsImmutable=*/false);
3946 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
3947
3948 // We need to store -2 into the UnwindHelp object at the start of the
3949 // function.
3950 DebugLoc DL;
3952 RS->backward(MBBI);
3953 Register DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
3954 assert(DstReg && "There must be a free register after frame setup");
3955 BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
3956 BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
3957 .addReg(DstReg, getKillRegState(true))
3958 .addFrameIndex(UnwindHelpFI)
3959 .addImm(0);
3960}
3961
3962namespace {
3963struct TagStoreInstr {
3965 int64_t Offset, Size;
3966 explicit TagStoreInstr(MachineInstr *MI, int64_t Offset, int64_t Size)
3967 : MI(MI), Offset(Offset), Size(Size) {}
3968};
3969
3970class TagStoreEdit {
3971 MachineFunction *MF;
3974 // Tag store instructions that are being replaced.
3976 // Combined memref arguments of the above instructions.
3978
3979 // Replace allocation tags in [FrameReg + FrameRegOffset, FrameReg +
3980 // FrameRegOffset + Size) with the address tag of SP.
3981 Register FrameReg;
3982 StackOffset FrameRegOffset;
3983 int64_t Size;
3984 // If not std::nullopt, move FrameReg to (FrameReg + FrameRegUpdate) at the
3985 // end.
3986 std::optional<int64_t> FrameRegUpdate;
3987 // MIFlags for any FrameReg updating instructions.
3988 unsigned FrameRegUpdateFlags;
3989
3990 // Use zeroing instruction variants.
3991 bool ZeroData;
3992 DebugLoc DL;
3993
3994 void emitUnrolled(MachineBasicBlock::iterator InsertI);
3995 void emitLoop(MachineBasicBlock::iterator InsertI);
3996
3997public:
3998 TagStoreEdit(MachineBasicBlock *MBB, bool ZeroData)
3999 : MBB(MBB), ZeroData(ZeroData) {
4000 MF = MBB->getParent();
4001 MRI = &MF->getRegInfo();
4002 }
4003 // Add an instruction to be replaced. Instructions must be added in the
4004 // ascending order of Offset, and have to be adjacent.
4005 void addInstruction(TagStoreInstr I) {
4006 assert((TagStores.empty() ||
4007 TagStores.back().Offset + TagStores.back().Size == I.Offset) &&
4008 "Non-adjacent tag store instructions.");
4009 TagStores.push_back(I);
4010 }
4011 void clear() { TagStores.clear(); }
4012 // Emit equivalent code at the given location, and erase the current set of
4013 // instructions. May skip if the replacement is not profitable. May invalidate
4014 // the input iterator and replace it with a valid one.
4015 void emitCode(MachineBasicBlock::iterator &InsertI,
4016 const AArch64FrameLowering *TFI, bool TryMergeSPUpdate);
4017};
4018
4019void TagStoreEdit::emitUnrolled(MachineBasicBlock::iterator InsertI) {
4020 const AArch64InstrInfo *TII =
4021 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
4022
4023 const int64_t kMinOffset = -256 * 16;
4024 const int64_t kMaxOffset = 255 * 16;
4025
4026 Register BaseReg = FrameReg;
4027 int64_t BaseRegOffsetBytes = FrameRegOffset.getFixed();
4028 if (BaseRegOffsetBytes < kMinOffset ||
4029 BaseRegOffsetBytes + (Size - Size % 32) > kMaxOffset ||
4030 // BaseReg can be FP, which is not necessarily aligned to 16-bytes. In
4031 // that case, BaseRegOffsetBytes will not be aligned to 16 bytes, which
4032 // is required for the offset of ST2G.
4033 BaseRegOffsetBytes % 16 != 0) {
4034 Register ScratchReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
4035 emitFrameOffset(*MBB, InsertI, DL, ScratchReg, BaseReg,
4036 StackOffset::getFixed(BaseRegOffsetBytes), TII);
4037 BaseReg = ScratchReg;
4038 BaseRegOffsetBytes = 0;
4039 }
4040
4041 MachineInstr *LastI = nullptr;
4042 while (Size) {
4043 int64_t InstrSize = (Size > 16) ? 32 : 16;
4044 unsigned Opcode =
4045 InstrSize == 16
4046 ? (ZeroData ? AArch64::STZGi : AArch64::STGi)
4047 : (ZeroData ? AArch64::STZ2Gi : AArch64::ST2Gi);
4048 assert(BaseRegOffsetBytes % 16 == 0);
4049 MachineInstr *I = BuildMI(*MBB, InsertI, DL, TII->get(Opcode))
4050 .addReg(AArch64::SP)
4051 .addReg(BaseReg)
4052 .addImm(BaseRegOffsetBytes / 16)
4053 .setMemRefs(CombinedMemRefs);
4054 // A store to [BaseReg, #0] should go last for an opportunity to fold the
4055 // final SP adjustment in the epilogue.
4056 if (BaseRegOffsetBytes == 0)
4057 LastI = I;
4058 BaseRegOffsetBytes += InstrSize;
4059 Size -= InstrSize;
4060 }
4061
4062 if (LastI)
4063 MBB->splice(InsertI, MBB, LastI);
4064}
4065
4066void TagStoreEdit::emitLoop(MachineBasicBlock::iterator InsertI) {
4067 const AArch64InstrInfo *TII =
4068 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
4069
4070 Register BaseReg = FrameRegUpdate
4071 ? FrameReg
4072 : MRI->createVirtualRegister(&AArch64::GPR64RegClass);
4073 Register SizeReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
4074
4075 emitFrameOffset(*MBB, InsertI, DL, BaseReg, FrameReg, FrameRegOffset, TII);
4076
4077 int64_t LoopSize = Size;
4078 // If the loop size is not a multiple of 32, split off one 16-byte store at
4079 // the end to fold BaseReg update into.
4080 if (FrameRegUpdate && *FrameRegUpdate)
4081 LoopSize -= LoopSize % 32;
4082 MachineInstr *LoopI = BuildMI(*MBB, InsertI, DL,
4083 TII->get(ZeroData ? AArch64::STZGloop_wback
4084 : AArch64::STGloop_wback))
4085 .addDef(SizeReg)
4086 .addDef(BaseReg)
4087 .addImm(LoopSize)
4088 .addReg(BaseReg)
4089 .setMemRefs(CombinedMemRefs);
4090 if (FrameRegUpdate)
4091 LoopI->setFlags(FrameRegUpdateFlags);
4092
4093 int64_t ExtraBaseRegUpdate =
4094 FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.getFixed() - Size) : 0;
4095 if (LoopSize < Size) {
4096 assert(FrameRegUpdate);
4097 assert(Size - LoopSize == 16);
4098 // Tag 16 more bytes at BaseReg and update BaseReg.
4099 BuildMI(*MBB, InsertI, DL,
4100 TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
4101 .addDef(BaseReg)
4102 .addReg(BaseReg)
4103 .addReg(BaseReg)
4104 .addImm(1 + ExtraBaseRegUpdate / 16)
4105 .setMemRefs(CombinedMemRefs)
4106 .setMIFlags(FrameRegUpdateFlags);
4107 } else if (ExtraBaseRegUpdate) {
4108 // Update BaseReg.
4109 BuildMI(
4110 *MBB, InsertI, DL,
4111 TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
4112 .addDef(BaseReg)
4113 .addReg(BaseReg)
4114 .addImm(std::abs(ExtraBaseRegUpdate))
4115 .addImm(0)
4116 .setMIFlags(FrameRegUpdateFlags);
4117 }
4118}
4119
4120// Check if *II is a register update that can be merged into STGloop that ends
4121// at (Reg + Size). RemainingOffset is the required adjustment to Reg after the
4122// end of the loop.
4123bool canMergeRegUpdate(MachineBasicBlock::iterator II, unsigned Reg,
4124 int64_t Size, int64_t *TotalOffset) {
4125 MachineInstr &MI = *II;
4126 if ((MI.getOpcode() == AArch64::ADDXri ||
4127 MI.getOpcode() == AArch64::SUBXri) &&
4128 MI.getOperand(0).getReg() == Reg && MI.getOperand(1).getReg() == Reg) {
4129 unsigned Shift = AArch64_AM::getShiftValue(MI.getOperand(3).getImm());
4130 int64_t Offset = MI.getOperand(2).getImm() << Shift;
4131 if (MI.getOpcode() == AArch64::SUBXri)
4132 Offset = -Offset;
4133 int64_t AbsPostOffset = std::abs(Offset - Size);
4134 const int64_t kMaxOffset =
4135 0xFFF; // Max encoding for unshifted ADDXri / SUBXri
4136 if (AbsPostOffset <= kMaxOffset && AbsPostOffset % 16 == 0) {
4137 *TotalOffset = Offset;
4138 return true;
4139 }
4140 }
4141 return false;
4142}
4143
4144void mergeMemRefs(const SmallVectorImpl<TagStoreInstr> &TSE,
4146 MemRefs.clear();
4147 for (auto &TS : TSE) {
4148 MachineInstr *MI = TS.MI;
4149 // An instruction without memory operands may access anything. Be
4150 // conservative and return an empty list.
4151 if (MI->memoperands_empty()) {
4152 MemRefs.clear();
4153 return;
4154 }
4155 MemRefs.append(MI->memoperands_begin(), MI->memoperands_end());
4156 }
4157}
4158
4159void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
4160 const AArch64FrameLowering *TFI,
4161 bool TryMergeSPUpdate) {
4162 if (TagStores.empty())
4163 return;
4164 TagStoreInstr &FirstTagStore = TagStores[0];
4165 TagStoreInstr &LastTagStore = TagStores[TagStores.size() - 1];
4166 Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
4167 DL = TagStores[0].MI->getDebugLoc();
4168
4169 Register Reg;
4170 FrameRegOffset = TFI->resolveFrameOffsetReference(
4171 *MF, FirstTagStore.Offset, false /*isFixed*/, false /*isSVE*/, Reg,
4172 /*PreferFP=*/false, /*ForSimm=*/true);
4173 FrameReg = Reg;
4174 FrameRegUpdate = std::nullopt;
4175
4176 mergeMemRefs(TagStores, CombinedMemRefs);
4177
4178 LLVM_DEBUG(dbgs() << "Replacing adjacent STG instructions:\n";
4179 for (const auto &Instr
4180 : TagStores) { dbgs() << " " << *Instr.MI; });
4181
4182 // Size threshold where a loop becomes shorter than a linear sequence of
4183 // tagging instructions.
4184 const int kSetTagLoopThreshold = 176;
4185 if (Size < kSetTagLoopThreshold) {
4186 if (TagStores.size() < 2)
4187 return;
4188 emitUnrolled(InsertI);
4189 } else {
4190 MachineInstr *UpdateInstr = nullptr;
4191 int64_t TotalOffset = 0;
4192 if (TryMergeSPUpdate) {
4193 // See if we can merge base register update into the STGloop.
4194 // This is done in AArch64LoadStoreOptimizer for "normal" stores,
4195 // but STGloop is way too unusual for that, and also it only
4196 // realistically happens in function epilogue. Also, STGloop is expanded
4197 // before that pass.
4198 if (InsertI != MBB->end() &&
4199 canMergeRegUpdate(InsertI, FrameReg, FrameRegOffset.getFixed() + Size,
4200 &TotalOffset)) {
4201 UpdateInstr = &*InsertI++;
4202 LLVM_DEBUG(dbgs() << "Folding SP update into loop:\n "
4203 << *UpdateInstr);
4204 }
4205 }
4206
4207 if (!UpdateInstr && TagStores.size() < 2)
4208 return;
4209
4210 if (UpdateInstr) {
4211 FrameRegUpdate = TotalOffset;
4212 FrameRegUpdateFlags = UpdateInstr->getFlags();
4213 }
4214 emitLoop(InsertI);
4215 if (UpdateInstr)
4216 UpdateInstr->eraseFromParent();
4217 }
4218
4219 for (auto &TS : TagStores)
4220 TS.MI->eraseFromParent();
4221}
4222
4223bool isMergeableStackTaggingInstruction(MachineInstr &MI, int64_t &Offset,
4224 int64_t &Size, bool &ZeroData) {
4225 MachineFunction &MF = *MI.getParent()->getParent();
4226 const MachineFrameInfo &MFI = MF.getFrameInfo();
4227
4228 unsigned Opcode = MI.getOpcode();
4229 ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGi ||
4230 Opcode == AArch64::STZ2Gi);
4231
4232 if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
4233 if (!MI.getOperand(0).isDead() || !MI.getOperand(1).isDead())
4234 return false;
4235 if (!MI.getOperand(2).isImm() || !MI.getOperand(3).isFI())
4236 return false;
4237 Offset = MFI.getObjectOffset(MI.getOperand(3).getIndex());
4238 Size = MI.getOperand(2).getImm();
4239 return true;
4240 }
4241
4242 if (Opcode == AArch64::STGi || Opcode == AArch64::STZGi)
4243 Size = 16;
4244 else if (Opcode == AArch64::ST2Gi || Opcode == AArch64::STZ2Gi)
4245 Size = 32;
4246 else
4247 return false;
4248
4249 if (MI.getOperand(0).getReg() != AArch64::SP || !MI.getOperand(1).isFI())
4250 return false;
4251
4252 Offset = MFI.getObjectOffset(MI.getOperand(1).getIndex()) +
4253 16 * MI.getOperand(2).getImm();
4254 return true;
4255}
4256
4257// Detect a run of memory tagging instructions for adjacent stack frame slots,
4258// and replace them with a shorter instruction sequence:
4259// * replace STG + STG with ST2G
4260// * replace STGloop + STGloop with STGloop
4261// This code needs to run when stack slot offsets are already known, but before
4262// FrameIndex operands in STG instructions are eliminated.
4264 const AArch64FrameLowering *TFI,
4265 RegScavenger *RS) {
4266 bool FirstZeroData;
4267 int64_t Size, Offset;
4268 MachineInstr &MI = *II;
4269 MachineBasicBlock *MBB = MI.getParent();
4271 if (&MI == &MBB->instr_back())
4272 return II;
4273 if (!isMergeableStackTaggingInstruction(MI, Offset, Size, FirstZeroData))
4274 return II;
4275
4277 Instrs.emplace_back(&MI, Offset, Size);
4278
4279 constexpr int kScanLimit = 10;
4280 int Count = 0;
4282 NextI != E && Count < kScanLimit; ++NextI) {
4283 MachineInstr &MI = *NextI;
4284 bool ZeroData;
4285 int64_t Size, Offset;
4286 // Collect instructions that update memory tags with a FrameIndex operand
4287 // and (when applicable) constant size, and whose output registers are dead
4288 // (the latter is almost always the case in practice). Since these
4289 // instructions effectively have no inputs or outputs, we are free to skip
4290 // any non-aliasing instructions in between without tracking used registers.
4291 if (isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData)) {
4292 if (ZeroData != FirstZeroData)
4293 break;
4294 Instrs.emplace_back(&MI, Offset, Size);
4295 continue;
4296 }
4297
4298 // Only count non-transient, non-tagging instructions toward the scan
4299 // limit.
4300 if (!MI.isTransient())
4301 ++Count;
4302
4303 // Just in case, stop before the epilogue code starts.
4304 if (MI.getFlag(MachineInstr::FrameSetup) ||
4306 break;
4307
4308 // Reject anything that may alias the collected instructions.
4309 if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects())
4310 break;
4311 }
4312
4313 // New code will be inserted after the last tagging instruction we've found.
4314 MachineBasicBlock::iterator InsertI = Instrs.back().MI;
4315
4316 // All the gathered stack tag instructions are merged and placed after
4317 // last tag store in the list. The check should be made if the nzcv
4318 // flag is live at the point where we are trying to insert. Otherwise
4319 // the nzcv flag might get clobbered if any stg loops are present.
4320
4321 // FIXME : This approach of bailing out from merge is conservative in
4322 // some ways like even if stg loops are not present after merge the
4323 // insert list, this liveness check is done (which is not needed).
4325 LiveRegs.addLiveOuts(*MBB);
4326 for (auto I = MBB->rbegin();; ++I) {
4327 MachineInstr &MI = *I;
4328 if (MI == InsertI)
4329 break;
4330 LiveRegs.stepBackward(*I);
4331 }
4332 InsertI++;
4333 if (LiveRegs.contains(AArch64::NZCV))
4334 return InsertI;
4335
4336 llvm::stable_sort(Instrs,
4337 [](const TagStoreInstr &Left, const TagStoreInstr &Right) {
4338 return Left.Offset < Right.Offset;
4339 });
4340
4341 // Make sure that we don't have any overlapping stores.
4342 int64_t CurOffset = Instrs[0].Offset;
4343 for (auto &Instr : Instrs) {
4344 if (CurOffset > Instr.Offset)
4345 return NextI;
4346 CurOffset = Instr.Offset + Instr.Size;
4347 }
4348
4349 // Find contiguous runs of tagged memory and emit shorter instruction
4350 // sequencies for them when possible.
4351 TagStoreEdit TSE(MBB, FirstZeroData);
4352 std::optional<int64_t> EndOffset;
4353 for (auto &Instr : Instrs) {
4354 if (EndOffset && *EndOffset != Instr.Offset) {
4355 // Found a gap.
4356 TSE.emitCode(InsertI, TFI, /*TryMergeSPUpdate = */ false);
4357 TSE.clear();
4358 }
4359
4360 TSE.addInstruction(Instr);
4361 EndOffset = Instr.Offset + Instr.Size;
4362 }
4363
4364 const MachineFunction *MF = MBB->getParent();
4365 // Multiple FP/SP updates in a loop cannot be described by CFI instructions.
4366 TSE.emitCode(
4367 InsertI, TFI, /*TryMergeSPUpdate = */
4369
4370 return InsertI;
4371}
4372} // namespace
4373
4375 const AArch64FrameLowering *TFI) {
4376 MachineInstr &MI = *II;
4377 MachineBasicBlock *MBB = MI.getParent();
4378 MachineFunction *MF = MBB->getParent();
4379
4380 if (MI.getOpcode() != AArch64::VGSavePseudo &&
4381 MI.getOpcode() != AArch64::VGRestorePseudo)
4382 return II;
4383
4384 SMEAttrs FuncAttrs(MF->getFunction());
4385 bool LocallyStreaming =
4386 FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface();
4389 const AArch64InstrInfo *TII =
4390 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
4391
4392 int64_t VGFrameIdx =
4393 LocallyStreaming ? AFI->getStreamingVGIdx() : AFI->getVGIdx();
4394 assert(VGFrameIdx != std::numeric_limits<int>::max() &&
4395 "Expected FrameIdx for VG");
4396
4397 unsigned CFIIndex;
4398 if (MI.getOpcode() == AArch64::VGSavePseudo) {
4399 const MachineFrameInfo &MFI = MF->getFrameInfo();
4400 int64_t Offset =
4401 MFI.getObjectOffset(VGFrameIdx) - TFI->getOffsetOfLocalArea();
4403 nullptr, TRI->getDwarfRegNum(AArch64::VG, true), Offset));
4404 } else
4406 nullptr, TRI->getDwarfRegNum(AArch64::VG, true)));
4407
4408 MachineInstr *UnwindInst = BuildMI(*MBB, II, II->getDebugLoc(),
4409 TII->get(TargetOpcode::CFI_INSTRUCTION))
4410 .addCFIIndex(CFIIndex);
4411
4412 MI.eraseFromParent();
4413 return UnwindInst->getIterator();
4414}
4415
4417 MachineFunction &MF, RegScavenger *RS = nullptr) const {
4419 for (auto &BB : MF)
4420 for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();) {
4421 if (AFI->hasStreamingModeChanges())
4422 II = emitVGSaveRestore(II, this);
4424 II = tryMergeAdjacentSTG(II, this, RS);
4425 }
4426}
4427
4428/// For Win64 AArch64 EH, the offset to the Unwind object is from the SP
4429/// before the update. This is easily retrieved as it is exactly the offset
4430/// that is set in processFunctionBeforeFrameFinalized.
4432 const MachineFunction &MF, int FI, Register &FrameReg,
4433 bool IgnoreSPUpdates) const {
4434 const MachineFrameInfo &MFI = MF.getFrameInfo();
4435 if (IgnoreSPUpdates) {
4436 LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
4437 << MFI.getObjectOffset(FI) << "\n");
4438 FrameReg = AArch64::SP;
4439 return StackOffset::getFixed(MFI.getObjectOffset(FI));
4440 }
4441
4442 // Go to common code if we cannot provide sp + offset.
4443 if (MFI.hasVarSizedObjects() ||
4446 return getFrameIndexReference(MF, FI, FrameReg);
4447
4448 FrameReg = AArch64::SP;
4449 return getStackOffset(MF, MFI.getObjectOffset(FI));
4450}
4451
4452/// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
4453/// the parent's frame pointer
4455 const MachineFunction &MF) const {
4456 return 0;
4457}
4458
4459/// Funclets only need to account for space for the callee saved registers,
4460/// as the locals are accounted for in the parent's stack frame.
4462 const MachineFunction &MF) const {
4463 // This is the size of the pushed CSRs.
4464 unsigned CSSize =
4465 MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
4466 // This is the amount of stack a funclet needs to allocate.
4467 return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
4468 getStackAlign());
4469}
4470
4471namespace {
4472struct FrameObject {
4473 bool IsValid = false;
4474 // Index of the object in MFI.
4475 int ObjectIndex = 0;
4476 // Group ID this object belongs to.
4477 int GroupIndex = -1;
4478 // This object should be placed first (closest to SP).
4479 bool ObjectFirst = false;
4480 // This object's group (which always contains the object with
4481 // ObjectFirst==true) should be placed first.
4482 bool GroupFirst = false;
4483};
4484
4485class GroupBuilder {
4486 SmallVector<int, 8> CurrentMembers;
4487 int NextGroupIndex = 0;
4488 std::vector<FrameObject> &Objects;
4489
4490public:
4491 GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
4492 void AddMember(int Index) { CurrentMembers.push_back(Index); }
4493 void EndCurrentGroup() {
4494 if (CurrentMembers.size() > 1) {
4495 // Create a new group with the current member list. This might remove them
4496 // from their pre-existing groups. That's OK, dealing with overlapping
4497 // groups is too hard and unlikely to make a difference.
4498 LLVM_DEBUG(dbgs() << "group:");
4499 for (int Index : CurrentMembers) {
4500 Objects[Index].GroupIndex = NextGroupIndex;
4501 LLVM_DEBUG(dbgs() << " " << Index);
4502 }
4503 LLVM_DEBUG(dbgs() << "\n");
4504 NextGroupIndex++;
4505 }
4506 CurrentMembers.clear();
4507 }
4508};
4509
4510bool FrameObjectCompare(const FrameObject &A, const FrameObject &B) {
4511 // Objects at a lower index are closer to FP; objects at a higher index are
4512 // closer to SP.
4513 //
4514 // For consistency in our comparison, all invalid objects are placed
4515 // at the end. This also allows us to stop walking when we hit the
4516 // first invalid item after it's all sorted.
4517 //
4518 // The "first" object goes first (closest to SP), followed by the members of
4519 // the "first" group.
4520 //
4521 // The rest are sorted by the group index to keep the groups together.
4522 // Higher numbered groups are more likely to be around longer (i.e. untagged
4523 // in the function epilogue and not at some earlier point). Place them closer
4524 // to SP.
4525 //
4526 // If all else equal, sort by the object index to keep the objects in the
4527 // original order.
4528 return std::make_tuple(!A.IsValid, A.ObjectFirst, A.GroupFirst, A.GroupIndex,
4529 A.ObjectIndex) <
4530 std::make_tuple(!B.IsValid, B.ObjectFirst, B.GroupFirst, B.GroupIndex,
4531 B.ObjectIndex);
4532}
4533} // namespace
4534
4536 const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
4537 if (!OrderFrameObjects || ObjectsToAllocate.empty())
4538 return;
4539
4540 const MachineFrameInfo &MFI = MF.getFrameInfo();
4541 std::vector<FrameObject> FrameObjects(MFI.getObjectIndexEnd());
4542 for (auto &Obj : ObjectsToAllocate) {
4543 FrameObjects[Obj].IsValid = true;
4544 FrameObjects[Obj].ObjectIndex = Obj;
4545 }
4546
4547 // Identify stack slots that are tagged at the same time.
4548 GroupBuilder GB(FrameObjects);
4549 for (auto &MBB : MF) {
4550 for (auto &MI : MBB) {
4551 if (MI.isDebugInstr())
4552 continue;
4553 int OpIndex;
4554 switch (MI.getOpcode()) {
4555 case AArch64::STGloop:
4556 case AArch64::STZGloop:
4557 OpIndex = 3;
4558 break;
4559 case AArch64::STGi:
4560 case AArch64::STZGi:
4561 case AArch64::ST2Gi:
4562 case AArch64::STZ2Gi:
4563 OpIndex = 1;
4564 break;
4565 default:
4566 OpIndex = -1;
4567 }
4568
4569 int TaggedFI = -1;
4570 if (OpIndex >= 0) {
4571 const MachineOperand &MO =