LLVM 19.0.0git
AArch64CallLowering.cpp
Go to the documentation of this file.
1//===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the lowering of LLVM calls to machine code calls for
11/// GlobalISel.
12///
13//===----------------------------------------------------------------------===//
14
15#include "AArch64CallLowering.h"
17#include "AArch64ISelLowering.h"
19#include "AArch64RegisterInfo.h"
20#include "AArch64Subtarget.h"
21#include "llvm/ADT/ArrayRef.h"
41#include "llvm/IR/Argument.h"
42#include "llvm/IR/Attributes.h"
43#include "llvm/IR/Function.h"
44#include "llvm/IR/Type.h"
45#include "llvm/IR/Value.h"
46#include <algorithm>
47#include <cassert>
48#include <cstdint>
49#include <iterator>
50
51#define DEBUG_TYPE "aarch64-call-lowering"
52
53using namespace llvm;
54using namespace AArch64GISelUtils;
55
57
59 : CallLowering(&TLI) {}
60
61static void applyStackPassedSmallTypeDAGHack(EVT OrigVT, MVT &ValVT,
62 MVT &LocVT) {
63 // If ValVT is i1/i8/i16, we should set LocVT to i8/i8/i16. This is a legacy
64 // hack because the DAG calls the assignment function with pre-legalized
65 // register typed values, not the raw type.
66 //
67 // This hack is not applied to return values which are not passed on the
68 // stack.
69 if (OrigVT == MVT::i1 || OrigVT == MVT::i8)
70 ValVT = LocVT = MVT::i8;
71 else if (OrigVT == MVT::i16)
72 ValVT = LocVT = MVT::i16;
73}
74
75// Account for i1/i8/i16 stack passed value hack
77 const MVT ValVT = VA.getValVT();
78 return (ValVT == MVT::i8 || ValVT == MVT::i16) ? LLT(ValVT)
79 : LLT(VA.getLocVT());
80}
81
82namespace {
83
84struct AArch64IncomingValueAssigner
86 AArch64IncomingValueAssigner(CCAssignFn *AssignFn_,
87 CCAssignFn *AssignFnVarArg_)
88 : IncomingValueAssigner(AssignFn_, AssignFnVarArg_) {}
89
90 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
93 CCState &State) override {
94 applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
95 return IncomingValueAssigner::assignArg(ValNo, OrigVT, ValVT, LocVT,
96 LocInfo, Info, Flags, State);
97 }
98};
99
100struct AArch64OutgoingValueAssigner
102 const AArch64Subtarget &Subtarget;
103
104 /// Track if this is used for a return instead of function argument
105 /// passing. We apply a hack to i1/i8/i16 stack passed values, but do not use
106 /// stack passed returns for them and cannot apply the type adjustment.
107 bool IsReturn;
108
109 AArch64OutgoingValueAssigner(CCAssignFn *AssignFn_,
110 CCAssignFn *AssignFnVarArg_,
111 const AArch64Subtarget &Subtarget_,
112 bool IsReturn)
113 : OutgoingValueAssigner(AssignFn_, AssignFnVarArg_),
114 Subtarget(Subtarget_), IsReturn(IsReturn) {}
115
116 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
117 CCValAssign::LocInfo LocInfo,
119 CCState &State) override {
120 bool IsCalleeWin = Subtarget.isCallingConvWin64(State.getCallingConv());
121 bool UseVarArgsCCForFixed = IsCalleeWin && State.isVarArg();
122
123 bool Res;
124 if (Info.IsFixed && !UseVarArgsCCForFixed) {
125 if (!IsReturn)
126 applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
127 Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State);
128 } else
129 Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Flags, State);
130
131 StackSize = State.getStackSize();
132 return Res;
133 }
134};
135
136struct IncomingArgHandler : public CallLowering::IncomingValueHandler {
137 IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
138 : IncomingValueHandler(MIRBuilder, MRI) {}
139
140 Register getStackAddress(uint64_t Size, int64_t Offset,
142 ISD::ArgFlagsTy Flags) override {
143 auto &MFI = MIRBuilder.getMF().getFrameInfo();
144
145 // Byval is assumed to be writable memory, but other stack passed arguments
146 // are not.
147 const bool IsImmutable = !Flags.isByVal();
148
149 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable);
150 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
151 auto AddrReg = MIRBuilder.buildFrameIndex(LLT::pointer(0, 64), FI);
152 return AddrReg.getReg(0);
153 }
154
155 LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
156 ISD::ArgFlagsTy Flags) const override {
157 // For pointers, we just need to fixup the integer types reported in the
158 // CCValAssign.
159 if (Flags.isPointer())
162 }
163
164 void assignValueToReg(Register ValVReg, Register PhysReg,
165 const CCValAssign &VA) override {
166 markPhysRegUsed(PhysReg);
167 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
168 }
169
170 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
171 const MachinePointerInfo &MPO,
172 const CCValAssign &VA) override {
173 MachineFunction &MF = MIRBuilder.getMF();
174
175 LLT ValTy(VA.getValVT());
176 LLT LocTy(VA.getLocVT());
177
178 // Fixup the types for the DAG compatibility hack.
179 if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16)
180 std::swap(ValTy, LocTy);
181 else {
182 // The calling code knows if this is a pointer or not, we're only touching
183 // the LocTy for the i8/i16 hack.
184 assert(LocTy.getSizeInBits() == MemTy.getSizeInBits());
185 LocTy = MemTy;
186 }
187
188 auto MMO = MF.getMachineMemOperand(
190 inferAlignFromPtrInfo(MF, MPO));
191
192 switch (VA.getLocInfo()) {
193 case CCValAssign::LocInfo::ZExt:
194 MIRBuilder.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, ValVReg, Addr, *MMO);
195 return;
196 case CCValAssign::LocInfo::SExt:
197 MIRBuilder.buildLoadInstr(TargetOpcode::G_SEXTLOAD, ValVReg, Addr, *MMO);
198 return;
199 default:
200 MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
201 return;
202 }
203 }
204
205 /// How the physical register gets marked varies between formal
206 /// parameters (it's a basic-block live-in), and a call instruction
207 /// (it's an implicit-def of the BL).
208 virtual void markPhysRegUsed(MCRegister PhysReg) = 0;
209};
210
211struct FormalArgHandler : public IncomingArgHandler {
213 : IncomingArgHandler(MIRBuilder, MRI) {}
214
215 void markPhysRegUsed(MCRegister PhysReg) override {
216 MIRBuilder.getMRI()->addLiveIn(PhysReg);
217 MIRBuilder.getMBB().addLiveIn(PhysReg);
218 }
219};
220
221struct CallReturnHandler : public IncomingArgHandler {
222 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
224 : IncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {}
225
226 void markPhysRegUsed(MCRegister PhysReg) override {
227 MIB.addDef(PhysReg, RegState::Implicit);
228 }
229
231};
232
233/// A special return arg handler for "returned" attribute arg calls.
234struct ReturnedArgCallReturnHandler : public CallReturnHandler {
235 ReturnedArgCallReturnHandler(MachineIRBuilder &MIRBuilder,
238 : CallReturnHandler(MIRBuilder, MRI, MIB) {}
239
240 void markPhysRegUsed(MCRegister PhysReg) override {}
241};
242
243struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
244 OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
245 MachineInstrBuilder MIB, bool IsTailCall = false,
246 int FPDiff = 0)
247 : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB), IsTailCall(IsTailCall),
248 FPDiff(FPDiff),
249 Subtarget(MIRBuilder.getMF().getSubtarget<AArch64Subtarget>()) {}
250
251 Register getStackAddress(uint64_t Size, int64_t Offset,
253 ISD::ArgFlagsTy Flags) override {
254 MachineFunction &MF = MIRBuilder.getMF();
255 LLT p0 = LLT::pointer(0, 64);
256 LLT s64 = LLT::scalar(64);
257
258 if (IsTailCall) {
259 assert(!Flags.isByVal() && "byval unhandled with tail calls");
260
261 Offset += FPDiff;
262 int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
263 auto FIReg = MIRBuilder.buildFrameIndex(p0, FI);
265 return FIReg.getReg(0);
266 }
267
268 if (!SPReg)
269 SPReg = MIRBuilder.buildCopy(p0, Register(AArch64::SP)).getReg(0);
270
271 auto OffsetReg = MIRBuilder.buildConstant(s64, Offset);
272
273 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg);
274
276 return AddrReg.getReg(0);
277 }
278
279 /// We need to fixup the reported store size for certain value types because
280 /// we invert the interpretation of ValVT and LocVT in certain cases. This is
281 /// for compatability with the DAG call lowering implementation, which we're
282 /// currently building on top of.
283 LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA,
284 ISD::ArgFlagsTy Flags) const override {
285 if (Flags.isPointer())
288 }
289
290 void assignValueToReg(Register ValVReg, Register PhysReg,
291 const CCValAssign &VA) override {
292 MIB.addUse(PhysReg, RegState::Implicit);
293 Register ExtReg = extendRegister(ValVReg, VA);
294 MIRBuilder.buildCopy(PhysReg, ExtReg);
295 }
296
297 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
298 const MachinePointerInfo &MPO,
299 const CCValAssign &VA) override {
300 MachineFunction &MF = MIRBuilder.getMF();
301 auto MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy,
302 inferAlignFromPtrInfo(MF, MPO));
303 MIRBuilder.buildStore(ValVReg, Addr, *MMO);
304 }
305
306 void assignValueToAddress(const CallLowering::ArgInfo &Arg, unsigned RegIndex,
307 Register Addr, LLT MemTy,
308 const MachinePointerInfo &MPO,
309 const CCValAssign &VA) override {
310 unsigned MaxSize = MemTy.getSizeInBytes() * 8;
311 // For varargs, we always want to extend them to 8 bytes, in which case
312 // we disable setting a max.
313 if (!Arg.IsFixed)
314 MaxSize = 0;
315
316 Register ValVReg = Arg.Regs[RegIndex];
317 if (VA.getLocInfo() != CCValAssign::LocInfo::FPExt) {
318 MVT LocVT = VA.getLocVT();
319 MVT ValVT = VA.getValVT();
320
321 if (VA.getValVT() == MVT::i8 || VA.getValVT() == MVT::i16) {
322 std::swap(ValVT, LocVT);
323 MemTy = LLT(VA.getValVT());
324 }
325
326 ValVReg = extendRegister(ValVReg, VA, MaxSize);
327 } else {
328 // The store does not cover the full allocated stack slot.
329 MemTy = LLT(VA.getValVT());
330 }
331
332 assignValueToAddress(ValVReg, Addr, MemTy, MPO, VA);
333 }
334
336
337 bool IsTailCall;
338
339 /// For tail calls, the byte offset of the call's argument area from the
340 /// callee's. Unused elsewhere.
341 int FPDiff;
342
343 // Cache the SP register vreg if we need it more than once in this call site.
344 Register SPReg;
345
346 const AArch64Subtarget &Subtarget;
347};
348} // namespace
349
350static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt) {
351 return (CallConv == CallingConv::Fast && TailCallOpt) ||
352 CallConv == CallingConv::Tail || CallConv == CallingConv::SwiftTail;
353}
354
356 const Value *Val,
357 ArrayRef<Register> VRegs,
359 Register SwiftErrorVReg) const {
360 auto MIB = MIRBuilder.buildInstrNoInsert(AArch64::RET_ReallyLR);
361 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) &&
362 "Return value without a vreg");
363
364 bool Success = true;
365 if (!FLI.CanLowerReturn) {
366 insertSRetStores(MIRBuilder, Val->getType(), VRegs, FLI.DemoteRegister);
367 } else if (!VRegs.empty()) {
368 MachineFunction &MF = MIRBuilder.getMF();
369 const Function &F = MF.getFunction();
370 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
371
373 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
374 CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
375 auto &DL = F.getParent()->getDataLayout();
376 LLVMContext &Ctx = Val->getType()->getContext();
377
378 SmallVector<EVT, 4> SplitEVTs;
379 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs);
380 assert(VRegs.size() == SplitEVTs.size() &&
381 "For each split Type there should be exactly one VReg.");
382
383 SmallVector<ArgInfo, 8> SplitArgs;
384 CallingConv::ID CC = F.getCallingConv();
385
386 for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
387 Register CurVReg = VRegs[i];
388 ArgInfo CurArgInfo = ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Ctx), 0};
390
391 // i1 is a special case because SDAG i1 true is naturally zero extended
392 // when widened using ANYEXT. We need to do it explicitly here.
393 auto &Flags = CurArgInfo.Flags[0];
394 if (MRI.getType(CurVReg).getSizeInBits() == 1 && !Flags.isSExt() &&
395 !Flags.isZExt()) {
396 CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg).getReg(0);
397 } else if (TLI.getNumRegistersForCallingConv(Ctx, CC, SplitEVTs[i]) ==
398 1) {
399 // Some types will need extending as specified by the CC.
400 MVT NewVT = TLI.getRegisterTypeForCallingConv(Ctx, CC, SplitEVTs[i]);
401 if (EVT(NewVT) != SplitEVTs[i]) {
402 unsigned ExtendOp = TargetOpcode::G_ANYEXT;
403 if (F.getAttributes().hasRetAttr(Attribute::SExt))
404 ExtendOp = TargetOpcode::G_SEXT;
405 else if (F.getAttributes().hasRetAttr(Attribute::ZExt))
406 ExtendOp = TargetOpcode::G_ZEXT;
407
408 LLT NewLLT(NewVT);
409 LLT OldLLT = getLLTForType(*CurArgInfo.Ty, DL);
410 CurArgInfo.Ty = EVT(NewVT).getTypeForEVT(Ctx);
411 // Instead of an extend, we might have a vector type which needs
412 // padding with more elements, e.g. <2 x half> -> <4 x half>.
413 if (NewVT.isVector()) {
414 if (OldLLT.isVector()) {
415 if (NewLLT.getNumElements() > OldLLT.getNumElements()) {
416 CurVReg =
417 MIRBuilder.buildPadVectorWithUndefElements(NewLLT, CurVReg)
418 .getReg(0);
419 } else {
420 // Just do a vector extend.
421 CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg})
422 .getReg(0);
423 }
424 } else if (NewLLT.getNumElements() >= 2 &&
425 NewLLT.getNumElements() <= 8) {
426 // We need to pad a <1 x S> type to <2/4/8 x S>. Since we don't
427 // have <1 x S> vector types in GISel we use a build_vector
428 // instead of a vector merge/concat.
429 CurVReg =
430 MIRBuilder.buildPadVectorWithUndefElements(NewLLT, CurVReg)
431 .getReg(0);
432 } else {
433 LLVM_DEBUG(dbgs() << "Could not handle ret ty\n");
434 return false;
435 }
436 } else {
437 // If the split EVT was a <1 x T> vector, and NewVT is T, then we
438 // don't have to do anything since we don't distinguish between the
439 // two.
440 if (NewLLT != MRI.getType(CurVReg)) {
441 // A scalar extend.
442 CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg})
443 .getReg(0);
444 }
445 }
446 }
447 }
448 if (CurVReg != CurArgInfo.Regs[0]) {
449 CurArgInfo.Regs[0] = CurVReg;
450 // Reset the arg flags after modifying CurVReg.
452 }
453 splitToValueTypes(CurArgInfo, SplitArgs, DL, CC);
454 }
455
456 AArch64OutgoingValueAssigner Assigner(AssignFn, AssignFn, Subtarget,
457 /*IsReturn*/ true);
458 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB);
459 Success = determineAndHandleAssignments(Handler, Assigner, SplitArgs,
460 MIRBuilder, CC, F.isVarArg());
461 }
462
463 if (SwiftErrorVReg) {
464 MIB.addUse(AArch64::X21, RegState::Implicit);
465 MIRBuilder.buildCopy(AArch64::X21, SwiftErrorVReg);
466 }
467
468 MIRBuilder.insertInstr(MIB);
469 return Success;
470}
471
473 CallingConv::ID CallConv,
475 bool IsVarArg) const {
477 const auto &TLI = *getTLI<AArch64TargetLowering>();
478 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
479 MF.getFunction().getContext());
480
481 return checkReturn(CCInfo, Outs, TLI.CCAssignFnForReturn(CallConv));
482}
483
484/// Helper function to compute forwarded registers for musttail calls. Computes
485/// the forwarded registers, sets MBB liveness, and emits COPY instructions that
486/// can be used to save + restore registers later.
488 CCAssignFn *AssignFn) {
489 MachineBasicBlock &MBB = MIRBuilder.getMBB();
490 MachineFunction &MF = MIRBuilder.getMF();
491 MachineFrameInfo &MFI = MF.getFrameInfo();
492
493 if (!MFI.hasMustTailInVarArgFunc())
494 return;
495
497 const Function &F = MF.getFunction();
498 assert(F.isVarArg() && "Expected F to be vararg?");
499
500 // Compute the set of forwarded registers. The rest are scratch.
502 CCState CCInfo(F.getCallingConv(), /*IsVarArg=*/true, MF, ArgLocs,
503 F.getContext());
504 SmallVector<MVT, 2> RegParmTypes;
505 RegParmTypes.push_back(MVT::i64);
506 RegParmTypes.push_back(MVT::f128);
507
508 // Later on, we can use this vector to restore the registers if necessary.
511 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, AssignFn);
512
513 // Conservatively forward X8, since it might be used for an aggregate
514 // return.
515 if (!CCInfo.isAllocated(AArch64::X8)) {
516 Register X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
517 Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
518 }
519
520 // Add the forwards to the MachineBasicBlock and MachineFunction.
521 for (const auto &F : Forwards) {
522 MBB.addLiveIn(F.PReg);
523 MIRBuilder.buildCopy(Register(F.VReg), Register(F.PReg));
524 }
525}
526
528 auto &F = MF.getFunction();
529 if (!EnableSVEGISel && (F.getReturnType()->isScalableTy() ||
530 llvm::any_of(F.args(), [](const Argument &A) {
531 return A.getType()->isScalableTy();
532 })))
533 return true;
534 const auto &ST = MF.getSubtarget<AArch64Subtarget>();
535 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
536 LLVM_DEBUG(dbgs() << "Falling back to SDAG because we don't support no-NEON\n");
537 return true;
538 }
539
540 SMEAttrs Attrs(F);
541 if (Attrs.hasZAState() || Attrs.hasZT0State() ||
542 Attrs.hasStreamingInterfaceOrBody() ||
543 Attrs.hasStreamingCompatibleInterface())
544 return true;
545
546 return false;
547}
548
549void AArch64CallLowering::saveVarArgRegisters(
551 CCState &CCInfo) const {
554
555 MachineFunction &MF = MIRBuilder.getMF();
557 MachineFrameInfo &MFI = MF.getFrameInfo();
559 auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
560 bool IsWin64CC =
561 Subtarget.isCallingConvWin64(CCInfo.getCallingConv());
562 const LLT p0 = LLT::pointer(0, 64);
563 const LLT s64 = LLT::scalar(64);
564
565 unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(GPRArgRegs);
566 unsigned NumVariadicGPRArgRegs = GPRArgRegs.size() - FirstVariadicGPR + 1;
567
568 unsigned GPRSaveSize = 8 * (GPRArgRegs.size() - FirstVariadicGPR);
569 int GPRIdx = 0;
570 if (GPRSaveSize != 0) {
571 if (IsWin64CC) {
572 GPRIdx = MFI.CreateFixedObject(GPRSaveSize,
573 -static_cast<int>(GPRSaveSize), false);
574 if (GPRSaveSize & 15)
575 // The extra size here, if triggered, will always be 8.
576 MFI.CreateFixedObject(16 - (GPRSaveSize & 15),
577 -static_cast<int>(alignTo(GPRSaveSize, 16)),
578 false);
579 } else
580 GPRIdx = MFI.CreateStackObject(GPRSaveSize, Align(8), false);
581
582 auto FIN = MIRBuilder.buildFrameIndex(p0, GPRIdx);
583 auto Offset =
584 MIRBuilder.buildConstant(MRI.createGenericVirtualRegister(s64), 8);
585
586 for (unsigned i = FirstVariadicGPR; i < GPRArgRegs.size(); ++i) {
587 Register Val = MRI.createGenericVirtualRegister(s64);
588 Handler.assignValueToReg(
589 Val, GPRArgRegs[i],
591 GPRArgRegs[i], MVT::i64, CCValAssign::Full));
592 auto MPO = IsWin64CC ? MachinePointerInfo::getFixedStack(
593 MF, GPRIdx, (i - FirstVariadicGPR) * 8)
594 : MachinePointerInfo::getStack(MF, i * 8);
595 MIRBuilder.buildStore(Val, FIN, MPO, inferAlignFromPtrInfo(MF, MPO));
596
597 FIN = MIRBuilder.buildPtrAdd(MRI.createGenericVirtualRegister(p0),
598 FIN.getReg(0), Offset);
599 }
600 }
601 FuncInfo->setVarArgsGPRIndex(GPRIdx);
602 FuncInfo->setVarArgsGPRSize(GPRSaveSize);
603
604 if (Subtarget.hasFPARMv8() && !IsWin64CC) {
605 unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(FPRArgRegs);
606
607 unsigned FPRSaveSize = 16 * (FPRArgRegs.size() - FirstVariadicFPR);
608 int FPRIdx = 0;
609 if (FPRSaveSize != 0) {
610 FPRIdx = MFI.CreateStackObject(FPRSaveSize, Align(16), false);
611
612 auto FIN = MIRBuilder.buildFrameIndex(p0, FPRIdx);
613 auto Offset =
614 MIRBuilder.buildConstant(MRI.createGenericVirtualRegister(s64), 16);
615
616 for (unsigned i = FirstVariadicFPR; i < FPRArgRegs.size(); ++i) {
617 Register Val = MRI.createGenericVirtualRegister(LLT::scalar(128));
618 Handler.assignValueToReg(
619 Val, FPRArgRegs[i],
621 i + MF.getFunction().getNumOperands() + NumVariadicGPRArgRegs,
622 MVT::f128, FPRArgRegs[i], MVT::f128, CCValAssign::Full));
623
624 auto MPO = MachinePointerInfo::getStack(MF, i * 16);
625 MIRBuilder.buildStore(Val, FIN, MPO, inferAlignFromPtrInfo(MF, MPO));
626
627 FIN = MIRBuilder.buildPtrAdd(MRI.createGenericVirtualRegister(p0),
628 FIN.getReg(0), Offset);
629 }
630 }
631 FuncInfo->setVarArgsFPRIndex(FPRIdx);
632 FuncInfo->setVarArgsFPRSize(FPRSaveSize);
633 }
634}
635
637 MachineIRBuilder &MIRBuilder, const Function &F,
639 MachineFunction &MF = MIRBuilder.getMF();
640 MachineBasicBlock &MBB = MIRBuilder.getMBB();
642 auto &DL = F.getParent()->getDataLayout();
643 auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
644
645 // Arm64EC has extra requirements for varargs calls which are only implemented
646 // in SelectionDAG; bail out for now.
647 if (F.isVarArg() && Subtarget.isWindowsArm64EC())
648 return false;
649
650 // Arm64EC thunks have a special calling convention which is only implemented
651 // in SelectionDAG; bail out for now.
652 if (F.getCallingConv() == CallingConv::ARM64EC_Thunk_Native ||
653 F.getCallingConv() == CallingConv::ARM64EC_Thunk_X64)
654 return false;
655
656 bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv()) && !Subtarget.isWindowsArm64EC();
657
658 SmallVector<ArgInfo, 8> SplitArgs;
660
661 // Insert the hidden sret parameter if the return value won't fit in the
662 // return registers.
663 if (!FLI.CanLowerReturn)
665
666 unsigned i = 0;
667 for (auto &Arg : F.args()) {
668 if (DL.getTypeStoreSize(Arg.getType()).isZero())
669 continue;
670
671 ArgInfo OrigArg{VRegs[i], Arg, i};
673
674 // i1 arguments are zero-extended to i8 by the caller. Emit a
675 // hint to reflect this.
676 if (OrigArg.Ty->isIntegerTy(1)) {
677 assert(OrigArg.Regs.size() == 1 &&
678 MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 &&
679 "Unexpected registers used for i1 arg");
680
681 auto &Flags = OrigArg.Flags[0];
682 if (!Flags.isZExt() && !Flags.isSExt()) {
683 // Lower i1 argument as i8, and insert AssertZExt + Trunc later.
684 Register OrigReg = OrigArg.Regs[0];
685 Register WideReg = MRI.createGenericVirtualRegister(LLT::scalar(8));
686 OrigArg.Regs[0] = WideReg;
687 BoolArgs.push_back({OrigReg, WideReg});
688 }
689 }
690
691 if (Arg.hasAttribute(Attribute::SwiftAsync))
692 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
693
694 splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv());
695 ++i;
696 }
697
698 if (!MBB.empty())
699 MIRBuilder.setInstr(*MBB.begin());
700
701 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
702 CCAssignFn *AssignFn = TLI.CCAssignFnForCall(F.getCallingConv(), IsWin64 && F.isVarArg());
703
704 AArch64IncomingValueAssigner Assigner(AssignFn, AssignFn);
705 FormalArgHandler Handler(MIRBuilder, MRI);
707 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
708 if (!determineAssignments(Assigner, SplitArgs, CCInfo) ||
709 !handleAssignments(Handler, SplitArgs, CCInfo, ArgLocs, MIRBuilder))
710 return false;
711
712 if (!BoolArgs.empty()) {
713 for (auto &KV : BoolArgs) {
714 Register OrigReg = KV.first;
715 Register WideReg = KV.second;
716 LLT WideTy = MRI.getType(WideReg);
717 assert(MRI.getType(OrigReg).getScalarSizeInBits() == 1 &&
718 "Unexpected bit size of a bool arg");
719 MIRBuilder.buildTrunc(
720 OrigReg, MIRBuilder.buildAssertZExt(WideTy, WideReg, 1).getReg(0));
721 }
722 }
723
725 uint64_t StackSize = Assigner.StackSize;
726 if (F.isVarArg()) {
727 if ((!Subtarget.isTargetDarwin() && !Subtarget.isWindowsArm64EC()) || IsWin64) {
728 // The AAPCS variadic function ABI is identical to the non-variadic
729 // one. As a result there may be more arguments in registers and we should
730 // save them for future reference.
731 // Win64 variadic functions also pass arguments in registers, but all
732 // float arguments are passed in integer registers.
733 saveVarArgRegisters(MIRBuilder, Handler, CCInfo);
734 } else if (Subtarget.isWindowsArm64EC()) {
735 return false;
736 }
737
738 // We currently pass all varargs at 8-byte alignment, or 4 in ILP32.
739 StackSize = alignTo(Assigner.StackSize, Subtarget.isTargetILP32() ? 4 : 8);
740
741 auto &MFI = MIRBuilder.getMF().getFrameInfo();
742 FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackSize, true));
743 }
744
745 if (doesCalleeRestoreStack(F.getCallingConv(),
747 // We have a non-standard ABI, so why not make full use of the stack that
748 // we're going to pop? It must be aligned to 16 B in any case.
749 StackSize = alignTo(StackSize, 16);
750
751 // If we're expected to restore the stack (e.g. fastcc), then we'll be
752 // adding a multiple of 16.
753 FuncInfo->setArgumentStackToRestore(StackSize);
754
755 // Our own callers will guarantee that the space is free by giving an
756 // aligned value to CALLSEQ_START.
757 }
758
759 // When we tail call, we need to check if the callee's arguments
760 // will fit on the caller's stack. So, whenever we lower formal arguments,
761 // we should keep track of this information, since we might lower a tail call
762 // in this function later.
763 FuncInfo->setBytesInStackArgArea(StackSize);
764
765 if (Subtarget.hasCustomCallingConv())
766 Subtarget.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
767
768 handleMustTailForwardedRegisters(MIRBuilder, AssignFn);
769
770 // Move back to the end of the basic block.
771 MIRBuilder.setMBB(MBB);
772
773 return true;
774}
775
776/// Return true if the calling convention is one that we can guarantee TCO for.
777static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
778 return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
780}
781
782/// Return true if we might ever do TCO for calls with this calling convention.
784 switch (CC) {
785 case CallingConv::C:
793 return true;
794 default:
795 return false;
796 }
797}
798
799/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
800/// CC.
801static std::pair<CCAssignFn *, CCAssignFn *>
803 return {TLI.CCAssignFnForCall(CC, false), TLI.CCAssignFnForCall(CC, true)};
804}
805
806bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
807 CallLoweringInfo &Info, MachineFunction &MF,
808 SmallVectorImpl<ArgInfo> &InArgs) const {
809 const Function &CallerF = MF.getFunction();
810 CallingConv::ID CalleeCC = Info.CallConv;
811 CallingConv::ID CallerCC = CallerF.getCallingConv();
812
813 // If the calling conventions match, then everything must be the same.
814 if (CalleeCC == CallerCC)
815 return true;
816
817 // Check if the caller and callee will handle arguments in the same way.
818 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
819 CCAssignFn *CalleeAssignFnFixed;
820 CCAssignFn *CalleeAssignFnVarArg;
821 std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =
822 getAssignFnsForCC(CalleeCC, TLI);
823
824 CCAssignFn *CallerAssignFnFixed;
825 CCAssignFn *CallerAssignFnVarArg;
826 std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =
827 getAssignFnsForCC(CallerCC, TLI);
828
829 AArch64IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,
830 CalleeAssignFnVarArg);
831 AArch64IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,
832 CallerAssignFnVarArg);
833
834 if (!resultsCompatible(Info, MF, InArgs, CalleeAssigner, CallerAssigner))
835 return false;
836
837 // Make sure that the caller and callee preserve all of the same registers.
838 auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
839 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
840 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
842 TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
843 TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
844 }
845
846 return TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved);
847}
848
849bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
850 CallLoweringInfo &Info, MachineFunction &MF,
851 SmallVectorImpl<ArgInfo> &OrigOutArgs) const {
852 // If there are no outgoing arguments, then we are done.
853 if (OrigOutArgs.empty())
854 return true;
855
856 const Function &CallerF = MF.getFunction();
857 LLVMContext &Ctx = CallerF.getContext();
858 CallingConv::ID CalleeCC = Info.CallConv;
859 CallingConv::ID CallerCC = CallerF.getCallingConv();
860 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
861 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
862
863 CCAssignFn *AssignFnFixed;
864 CCAssignFn *AssignFnVarArg;
865 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
866
867 // We have outgoing arguments. Make sure that we can tail call with them.
869 CCState OutInfo(CalleeCC, false, MF, OutLocs, Ctx);
870
871 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
872 Subtarget, /*IsReturn*/ false);
873 // determineAssignments() may modify argument flags, so make a copy.
875 append_range(OutArgs, OrigOutArgs);
876 if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo)) {
877 LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n");
878 return false;
879 }
880
881 // Make sure that they can fit on the caller's stack.
882 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
883 if (OutInfo.getStackSize() > FuncInfo->getBytesInStackArgArea()) {
884 LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n");
885 return false;
886 }
887
888 // Verify that the parameters in callee-saved registers match.
889 // TODO: Port this over to CallLowering as general code once swiftself is
890 // supported.
891 auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
892 const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);
894
895 if (Info.IsVarArg) {
896 // Be conservative and disallow variadic memory operands to match SDAG's
897 // behaviour.
898 // FIXME: If the caller's calling convention is C, then we can
899 // potentially use its argument area. However, for cases like fastcc,
900 // we can't do anything.
901 for (unsigned i = 0; i < OutLocs.size(); ++i) {
902 auto &ArgLoc = OutLocs[i];
903 if (ArgLoc.isRegLoc())
904 continue;
905
907 dbgs()
908 << "... Cannot tail call vararg function with stack arguments\n");
909 return false;
910 }
911 }
912
913 return parametersInCSRMatch(MRI, CallerPreservedMask, OutLocs, OutArgs);
914}
915
917 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
919 SmallVectorImpl<ArgInfo> &OutArgs) const {
920
921 // Must pass all target-independent checks in order to tail call optimize.
922 if (!Info.IsTailCall)
923 return false;
924
925 CallingConv::ID CalleeCC = Info.CallConv;
926 MachineFunction &MF = MIRBuilder.getMF();
927 const Function &CallerF = MF.getFunction();
928
929 LLVM_DEBUG(dbgs() << "Attempting to lower call as tail call\n");
930
931 if (Info.SwiftErrorVReg) {
932 // TODO: We should handle this.
933 // Note that this is also handled by the check for no outgoing arguments.
934 // Proactively disabling this though, because the swifterror handling in
935 // lowerCall inserts a COPY *after* the location of the call.
936 LLVM_DEBUG(dbgs() << "... Cannot handle tail calls with swifterror yet.\n");
937 return false;
938 }
939
940 if (!mayTailCallThisCC(CalleeCC)) {
941 LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n");
942 return false;
943 }
944
945 // Byval parameters hand the function a pointer directly into the stack area
946 // we want to reuse during a tail call. Working around this *is* possible (see
947 // X86).
948 //
949 // FIXME: In AArch64ISelLowering, this isn't worked around. Can/should we try
950 // it?
951 //
952 // On Windows, "inreg" attributes signify non-aggregate indirect returns.
953 // In this case, it is necessary to save/restore X0 in the callee. Tail
954 // call opt interferes with this. So we disable tail call opt when the
955 // caller has an argument with "inreg" attribute.
956 //
957 // FIXME: Check whether the callee also has an "inreg" argument.
958 //
959 // When the caller has a swifterror argument, we don't want to tail call
960 // because would have to move into the swifterror register before the
961 // tail call.
962 if (any_of(CallerF.args(), [](const Argument &A) {
963 return A.hasByValAttr() || A.hasInRegAttr() || A.hasSwiftErrorAttr();
964 })) {
965 LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval, "
966 "inreg, or swifterror arguments\n");
967 return false;
968 }
969
970 // Externally-defined functions with weak linkage should not be
971 // tail-called on AArch64 when the OS does not support dynamic
972 // pre-emption of symbols, as the AAELF spec requires normal calls
973 // to undefined weak functions to be replaced with a NOP or jump to the
974 // next instruction. The behaviour of branch instructions in this
975 // situation (as used for tail calls) is implementation-defined, so we
976 // cannot rely on the linker replacing the tail call with a return.
977 if (Info.Callee.isGlobal()) {
978 const GlobalValue *GV = Info.Callee.getGlobal();
979 const Triple &TT = MF.getTarget().getTargetTriple();
980 if (GV->hasExternalWeakLinkage() &&
981 (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
982 TT.isOSBinFormatMachO())) {
983 LLVM_DEBUG(dbgs() << "... Cannot tail call externally-defined function "
984 "with weak linkage for this OS.\n");
985 return false;
986 }
987 }
988
989 // If we have -tailcallopt, then we're done.
991 return CalleeCC == CallerF.getCallingConv();
992
993 // We don't have -tailcallopt, so we're allowed to change the ABI (sibcall).
994 // Try to find cases where we can do that.
995
996 // I want anyone implementing a new calling convention to think long and hard
997 // about this assert.
998 assert((!Info.IsVarArg || CalleeCC == CallingConv::C) &&
999 "Unexpected variadic calling convention");
1000
1001 // Verify that the incoming and outgoing arguments from the callee are
1002 // safe to tail call.
1003 if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
1004 LLVM_DEBUG(
1005 dbgs()
1006 << "... Caller and callee have incompatible calling conventions.\n");
1007 return false;
1008 }
1009
1010 if (!areCalleeOutgoingArgsTailCallable(Info, MF, OutArgs))
1011 return false;
1012
1013 LLVM_DEBUG(
1014 dbgs() << "... Call is eligible for tail call optimization.\n");
1015 return true;
1016}
1017
1018static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,
1019 bool IsTailCall,
1020 std::optional<CallLowering::PtrAuthInfo> &PAI,
1022 const AArch64FunctionInfo *FuncInfo = CallerF.getInfo<AArch64FunctionInfo>();
1023
1024 if (!IsTailCall) {
1025 if (!PAI)
1026 return IsIndirect ? getBLRCallOpcode(CallerF) : (unsigned)AArch64::BL;
1027
1028 assert(IsIndirect && "Direct call should not be authenticated");
1029 assert((PAI->Key == AArch64PACKey::IA || PAI->Key == AArch64PACKey::IB) &&
1030 "Invalid auth call key");
1031 return AArch64::BLRA;
1032 }
1033
1034 if (!IsIndirect)
1035 return AArch64::TCRETURNdi;
1036
1037 // When BTI or PAuthLR are enabled, there are restrictions on using x16 and
1038 // x17 to hold the function pointer.
1039 if (FuncInfo->branchTargetEnforcement()) {
1040 if (FuncInfo->branchProtectionPAuthLR()) {
1041 assert(!PAI && "ptrauth tail-calls not yet supported with PAuthLR");
1042 return AArch64::TCRETURNrix17;
1043 }
1044 if (PAI)
1045 return AArch64::AUTH_TCRETURN_BTI;
1046 return AArch64::TCRETURNrix16x17;
1047 }
1048
1049 if (FuncInfo->branchProtectionPAuthLR()) {
1050 assert(!PAI && "ptrauth tail-calls not yet supported with PAuthLR");
1051 return AArch64::TCRETURNrinotx16;
1052 }
1053
1054 if (PAI)
1055 return AArch64::AUTH_TCRETURN;
1056 return AArch64::TCRETURNri;
1057}
1058
1059static const uint32_t *
1063 const uint32_t *Mask;
1064 if (!OutArgs.empty() && OutArgs[0].Flags[0].isReturned()) {
1065 // For 'this' returns, use the X0-preserving mask if applicable
1066 Mask = TRI.getThisReturnPreservedMask(MF, Info.CallConv);
1067 if (!Mask) {
1068 OutArgs[0].Flags[0].setReturned(false);
1069 Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1070 }
1071 } else {
1072 Mask = TRI.getCallPreservedMask(MF, Info.CallConv);
1073 }
1074 return Mask;
1075}
1076
1077bool AArch64CallLowering::lowerTailCall(
1078 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
1079 SmallVectorImpl<ArgInfo> &OutArgs) const {
1080 MachineFunction &MF = MIRBuilder.getMF();
1081 const Function &F = MF.getFunction();
1083 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1085
1086 // True when we're tail calling, but without -tailcallopt.
1087 bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt &&
1088 Info.CallConv != CallingConv::Tail &&
1089 Info.CallConv != CallingConv::SwiftTail;
1090
1091 // Find out which ABI gets to decide where things go.
1092 CallingConv::ID CalleeCC = Info.CallConv;
1093 CCAssignFn *AssignFnFixed;
1094 CCAssignFn *AssignFnVarArg;
1095 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
1096
1097 MachineInstrBuilder CallSeqStart;
1098 if (!IsSibCall)
1099 CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
1100
1101 unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), true, Info.PAI, MRI);
1102 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1103 MIB.add(Info.Callee);
1104
1105 // Tell the call which registers are clobbered.
1106 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1107 auto TRI = Subtarget.getRegisterInfo();
1108
1109 // Byte offset for the tail call. When we are sibcalling, this will always
1110 // be 0.
1111 MIB.addImm(0);
1112
1113 // Authenticated tail calls always take key/discriminator arguments.
1114 if (Opc == AArch64::AUTH_TCRETURN || Opc == AArch64::AUTH_TCRETURN_BTI) {
1115 assert((Info.PAI->Key == AArch64PACKey::IA ||
1116 Info.PAI->Key == AArch64PACKey::IB) &&
1117 "Invalid auth call key");
1118 MIB.addImm(Info.PAI->Key);
1119
1120 Register AddrDisc = 0;
1121 uint16_t IntDisc = 0;
1122 std::tie(IntDisc, AddrDisc) =
1123 extractPtrauthBlendDiscriminators(Info.PAI->Discriminator, MRI);
1124
1125 MIB.addImm(IntDisc);
1126 MIB.addUse(AddrDisc);
1127 if (AddrDisc != AArch64::NoRegister) {
1128 MIB->getOperand(4).setReg(constrainOperandRegClass(
1129 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
1130 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(),
1131 MIB->getOperand(4), 4));
1132 }
1133 }
1134
1135 // Tell the call which registers are clobbered.
1136 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);
1137 if (Subtarget.hasCustomCallingConv())
1138 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
1139 MIB.addRegMask(Mask);
1140
1141 if (Info.CFIType)
1142 MIB->setCFIType(MF, Info.CFIType->getZExtValue());
1143
1144 if (TRI->isAnyArgRegReserved(MF))
1145 TRI->emitReservedArgRegCallError(MF);
1146
1147 // FPDiff is the byte offset of the call's argument area from the callee's.
1148 // Stores to callee stack arguments will be placed in FixedStackSlots offset
1149 // by this amount for a tail call. In a sibling call it must be 0 because the
1150 // caller will deallocate the entire stack and the callee still expects its
1151 // arguments to begin at SP+0.
1152 int FPDiff = 0;
1153
1154 // This will be 0 for sibcalls, potentially nonzero for tail calls produced
1155 // by -tailcallopt. For sibcalls, the memory operands for the call are
1156 // already available in the caller's incoming argument space.
1157 unsigned NumBytes = 0;
1158 if (!IsSibCall) {
1159 // We aren't sibcalling, so we need to compute FPDiff. We need to do this
1160 // before handling assignments, because FPDiff must be known for memory
1161 // arguments.
1162 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
1164 CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
1165
1166 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
1167 Subtarget, /*IsReturn*/ false);
1168 if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo))
1169 return false;
1170
1171 // The callee will pop the argument stack as a tail call. Thus, we must
1172 // keep it 16-byte aligned.
1173 NumBytes = alignTo(OutInfo.getStackSize(), 16);
1174
1175 // FPDiff will be negative if this tail call requires more space than we
1176 // would automatically have in our incoming argument space. Positive if we
1177 // actually shrink the stack.
1178 FPDiff = NumReusableBytes - NumBytes;
1179
1180 // Update the required reserved area if this is the tail call requiring the
1181 // most argument stack space.
1182 if (FPDiff < 0 && FuncInfo->getTailCallReservedStack() < (unsigned)-FPDiff)
1183 FuncInfo->setTailCallReservedStack(-FPDiff);
1184
1185 // The stack pointer must be 16-byte aligned at all times it's used for a
1186 // memory operation, which in practice means at *all* times and in
1187 // particular across call boundaries. Therefore our own arguments started at
1188 // a 16-byte aligned SP and the delta applied for the tail call should
1189 // satisfy the same constraint.
1190 assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
1191 }
1192
1193 const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
1194
1195 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1196 Subtarget, /*IsReturn*/ false);
1197
1198 // Do the actual argument marshalling.
1199 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB,
1200 /*IsTailCall*/ true, FPDiff);
1201 if (!determineAndHandleAssignments(Handler, Assigner, OutArgs, MIRBuilder,
1202 CalleeCC, Info.IsVarArg))
1203 return false;
1204
1205 Mask = getMaskForArgs(OutArgs, Info, *TRI, MF);
1206
1207 if (Info.IsVarArg && Info.IsMustTailCall) {
1208 // Now we know what's being passed to the function. Add uses to the call for
1209 // the forwarded registers that we *aren't* passing as parameters. This will
1210 // preserve the copies we build earlier.
1211 for (const auto &F : Forwards) {
1212 Register ForwardedReg = F.PReg;
1213 // If the register is already passed, or aliases a register which is
1214 // already being passed, then skip it.
1215 if (any_of(MIB->uses(), [&ForwardedReg, &TRI](const MachineOperand &Use) {
1216 if (!Use.isReg())
1217 return false;
1218 return TRI->regsOverlap(Use.getReg(), ForwardedReg);
1219 }))
1220 continue;
1221
1222 // We aren't passing it already, so we should add it to the call.
1223 MIRBuilder.buildCopy(ForwardedReg, Register(F.VReg));
1224 MIB.addReg(ForwardedReg, RegState::Implicit);
1225 }
1226 }
1227
1228 // If we have -tailcallopt, we need to adjust the stack. We'll do the call
1229 // sequence start and end here.
1230 if (!IsSibCall) {
1231 MIB->getOperand(1).setImm(FPDiff);
1232 CallSeqStart.addImm(0).addImm(0);
1233 // End the call sequence *before* emitting the call. Normally, we would
1234 // tidy the frame up after the call. However, here, we've laid out the
1235 // parameters so that when SP is reset, they will be in the correct
1236 // location.
1237 MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP).addImm(0).addImm(0);
1238 }
1239
1240 // Now we can add the actual call instruction to the correct basic block.
1241 MIRBuilder.insertInstr(MIB);
1242
1243 // If Callee is a reg, since it is used by a target specific instruction,
1244 // it must have a register class matching the constraint of that instruction.
1245 if (MIB->getOperand(0).isReg())
1247 *MF.getSubtarget().getRegBankInfo(), *MIB,
1248 MIB->getDesc(), MIB->getOperand(0), 0);
1249
1251 Info.LoweredTailCall = true;
1252 return true;
1253}
1254
1256 CallLoweringInfo &Info) const {
1257 MachineFunction &MF = MIRBuilder.getMF();
1258 const Function &F = MF.getFunction();
1260 auto &DL = F.getParent()->getDataLayout();
1261 const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
1262 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
1263
1264 // Arm64EC has extra requirements for varargs calls; bail out for now.
1265 //
1266 // Arm64EC has special mangling rules for calls; bail out on all calls for
1267 // now.
1268 if (Subtarget.isWindowsArm64EC())
1269 return false;
1270
1271 // Arm64EC thunks have a special calling convention which is only implemented
1272 // in SelectionDAG; bail out for now.
1273 if (Info.CallConv == CallingConv::ARM64EC_Thunk_Native ||
1275 return false;
1276
1278 for (auto &OrigArg : Info.OrigArgs) {
1279 splitToValueTypes(OrigArg, OutArgs, DL, Info.CallConv);
1280 // AAPCS requires that we zero-extend i1 to 8 bits by the caller.
1281 auto &Flags = OrigArg.Flags[0];
1282 if (OrigArg.Ty->isIntegerTy(1) && !Flags.isSExt() && !Flags.isZExt()) {
1283 ArgInfo &OutArg = OutArgs.back();
1284 assert(OutArg.Regs.size() == 1 &&
1285 MRI.getType(OutArg.Regs[0]).getSizeInBits() == 1 &&
1286 "Unexpected registers used for i1 arg");
1287
1288 // We cannot use a ZExt ArgInfo flag here, because it will
1289 // zero-extend the argument to i32 instead of just i8.
1290 OutArg.Regs[0] =
1291 MIRBuilder.buildZExt(LLT::scalar(8), OutArg.Regs[0]).getReg(0);
1292 LLVMContext &Ctx = MF.getFunction().getContext();
1293 OutArg.Ty = Type::getInt8Ty(Ctx);
1294 }
1295 }
1296
1298 if (!Info.OrigRet.Ty->isVoidTy())
1299 splitToValueTypes(Info.OrigRet, InArgs, DL, Info.CallConv);
1300
1301 // If we can lower as a tail call, do that instead.
1302 bool CanTailCallOpt =
1303 isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
1304
1305 // We must emit a tail call if we have musttail.
1306 if (Info.IsMustTailCall && !CanTailCallOpt) {
1307 // There are types of incoming/outgoing arguments we can't handle yet, so
1308 // it doesn't make sense to actually die here like in ISelLowering. Instead,
1309 // fall back to SelectionDAG and let it try to handle this.
1310 LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
1311 return false;
1312 }
1313
1314 Info.IsTailCall = CanTailCallOpt;
1315 if (CanTailCallOpt)
1316 return lowerTailCall(MIRBuilder, Info, OutArgs);
1317
1318 // Find out which ABI gets to decide where things go.
1319 CCAssignFn *AssignFnFixed;
1320 CCAssignFn *AssignFnVarArg;
1321 std::tie(AssignFnFixed, AssignFnVarArg) =
1322 getAssignFnsForCC(Info.CallConv, TLI);
1323
1324 MachineInstrBuilder CallSeqStart;
1325 CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
1326
1327 // Create a temporarily-floating call instruction so we can add the implicit
1328 // uses of arg registers.
1329
1330 unsigned Opc = 0;
1331 // Calls with operand bundle "clang.arc.attachedcall" are special. They should
1332 // be expanded to the call, directly followed by a special marker sequence and
1333 // a call to an ObjC library function.
1335 Opc = Info.PAI ? AArch64::BLRA_RVMARKER : AArch64::BLR_RVMARKER;
1336 // A call to a returns twice function like setjmp must be followed by a bti
1337 // instruction.
1338 else if (Info.CB && Info.CB->hasFnAttr(Attribute::ReturnsTwice) &&
1339 !Subtarget.noBTIAtReturnTwice() &&
1341 Opc = AArch64::BLR_BTI;
1342 else {
1343 // For an intrinsic call (e.g. memset), use GOT if "RtLibUseGOT" (-fno-plt)
1344 // is set.
1345 if (Info.Callee.isSymbol() && F.getParent()->getRtLibUseGOT()) {
1346 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_GLOBAL_VALUE);
1347 DstOp(getLLTForType(*F.getType(), DL)).addDefToMIB(MRI, MIB);
1348 MIB.addExternalSymbol(Info.Callee.getSymbolName(), AArch64II::MO_GOT);
1349 Info.Callee = MachineOperand::CreateReg(MIB.getReg(0), false);
1350 }
1351 Opc = getCallOpcode(MF, Info.Callee.isReg(), false, Info.PAI, MRI);
1352 }
1353
1354 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
1355 unsigned CalleeOpNo = 0;
1356
1357 if (Opc == AArch64::BLR_RVMARKER || Opc == AArch64::BLRA_RVMARKER) {
1358 // Add a target global address for the retainRV/claimRV runtime function
1359 // just before the call target.
1361 MIB.addGlobalAddress(ARCFn);
1362 ++CalleeOpNo;
1363 } else if (Info.CFIType) {
1364 MIB->setCFIType(MF, Info.CFIType->getZExtValue());
1365 }
1366
1367 MIB.add(Info.Callee);
1368
1369 // Tell the call which registers are clobbered.
1370 const uint32_t *Mask;
1371 const auto *TRI = Subtarget.getRegisterInfo();
1372
1373 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1374 Subtarget, /*IsReturn*/ false);
1375 // Do the actual argument marshalling.
1376 OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, /*IsReturn*/ false);
1377 if (!determineAndHandleAssignments(Handler, Assigner, OutArgs, MIRBuilder,
1378 Info.CallConv, Info.IsVarArg))
1379 return false;
1380
1381 Mask = getMaskForArgs(OutArgs, Info, *TRI, MF);
1382
1383 if (Opc == AArch64::BLRA || Opc == AArch64::BLRA_RVMARKER) {
1384 assert((Info.PAI->Key == AArch64PACKey::IA ||
1385 Info.PAI->Key == AArch64PACKey::IB) &&
1386 "Invalid auth call key");
1387 MIB.addImm(Info.PAI->Key);
1388
1389 Register AddrDisc = 0;
1390 uint16_t IntDisc = 0;
1391 std::tie(IntDisc, AddrDisc) =
1392 extractPtrauthBlendDiscriminators(Info.PAI->Discriminator, MRI);
1393
1394 MIB.addImm(IntDisc);
1395 MIB.addUse(AddrDisc);
1396 if (AddrDisc != AArch64::NoRegister) {
1398 *MF.getSubtarget().getRegBankInfo(), *MIB,
1399 MIB->getDesc(), MIB->getOperand(CalleeOpNo + 3),
1400 CalleeOpNo + 3);
1401 }
1402 }
1403
1404 // Tell the call which registers are clobbered.
1406 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
1407 MIB.addRegMask(Mask);
1408
1409 if (TRI->isAnyArgRegReserved(MF))
1410 TRI->emitReservedArgRegCallError(MF);
1411
1412 // Now we can add the actual call instruction to the correct basic block.
1413 MIRBuilder.insertInstr(MIB);
1414
1415 uint64_t CalleePopBytes =
1418 ? alignTo(Assigner.StackSize, 16)
1419 : 0;
1420
1421 CallSeqStart.addImm(Assigner.StackSize).addImm(0);
1422 MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP)
1423 .addImm(Assigner.StackSize)
1424 .addImm(CalleePopBytes);
1425
1426 // If Callee is a reg, since it is used by a target specific
1427 // instruction, it must have a register class matching the
1428 // constraint of that instruction.
1429 if (MIB->getOperand(CalleeOpNo).isReg())
1430 constrainOperandRegClass(MF, *TRI, MRI, *Subtarget.getInstrInfo(),
1431 *Subtarget.getRegBankInfo(), *MIB, MIB->getDesc(),
1432 MIB->getOperand(CalleeOpNo), CalleeOpNo);
1433
1434 // Finally we can copy the returned value back into its virtual-register. In
1435 // symmetry with the arguments, the physical register must be an
1436 // implicit-define of the call instruction.
1437 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) {
1438 CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(Info.CallConv);
1439 CallReturnHandler Handler(MIRBuilder, MRI, MIB);
1440 bool UsingReturnedArg =
1441 !OutArgs.empty() && OutArgs[0].Flags[0].isReturned();
1442
1443 AArch64OutgoingValueAssigner Assigner(RetAssignFn, RetAssignFn, Subtarget,
1444 /*IsReturn*/ false);
1445 ReturnedArgCallReturnHandler ReturnedArgHandler(MIRBuilder, MRI, MIB);
1447 UsingReturnedArg ? ReturnedArgHandler : Handler, Assigner, InArgs,
1448 MIRBuilder, Info.CallConv, Info.IsVarArg,
1449 UsingReturnedArg ? ArrayRef(OutArgs[0].Regs) : std::nullopt))
1450 return false;
1451 }
1452
1453 if (Info.SwiftErrorVReg) {
1454 MIB.addDef(AArch64::X21, RegState::Implicit);
1455 MIRBuilder.buildCopy(Info.SwiftErrorVReg, Register(AArch64::X21));
1456 }
1457
1458 if (!Info.CanLowerReturn) {
1459 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs,
1460 Info.DemoteRegister, Info.DemoteStackIndex);
1461 }
1462 return true;
1463}
1464
1466 return Ty.getSizeInBits() == 64;
1467}
unsigned const MachineRegisterInfo * MRI
static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder, CCAssignFn *AssignFn)
Helper function to compute forwarded registers for musttail calls.
cl::opt< bool > EnableSVEGISel
static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect, bool IsTailCall, std::optional< CallLowering::PtrAuthInfo > &PAI, MachineRegisterInfo &MRI)
static LLT getStackValueStoreTypeHack(const CCValAssign &VA)
static const uint32_t * getMaskForArgs(SmallVectorImpl< AArch64CallLowering::ArgInfo > &OutArgs, AArch64CallLowering::CallLoweringInfo &Info, const AArch64RegisterInfo &TRI, MachineFunction &MF)
static void applyStackPassedSmallTypeDAGHack(EVT OrigVT, MVT &ValVT, MVT &LocVT)
static std::pair< CCAssignFn *, CCAssignFn * > getAssignFnsForCC(CallingConv::ID CC, const AArch64TargetLowering &TLI)
Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for CC.
static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt)
This file describes how to lower LLVM calls to machine code calls.
#define Success
static std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG)
static const MCPhysReg GPRArgRegs[]
static const MCPhysReg FPRArgRegs[]
cl::opt< bool > EnableSVEGISel("aarch64-enable-gisel-sve", cl::Hidden, cl::desc("Enable / disable SVE scalable vectors in Global ISel"), cl::init(false))
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
static bool mayTailCallThisCC(CallingConv::ID CC)
Return true if we might ever do TCO for calls with this calling convention.
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
uint64_t Size
Implement a low-level type suitable for MachineInstr level instruction selection.
#define F(x, y, z)
Definition: MD5.cpp:55
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
This file defines ARC utility functions which are used by various parts of the compiler.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const override
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
bool canLowerReturn(MachineFunction &MF, CallingConv::ID CallConv, SmallVectorImpl< BaseArgInfo > &Outs, bool IsVarArg) const override
This hook must be implemented to check whether the return values described by Outs can fit into the r...
bool fallBackToDAGISel(const MachineFunction &MF) const override
bool isTypeIsValidForThisReturn(EVT Ty) const override
For targets which support the "returned" parameter attribute, returns true if the given type is a val...
bool isEligibleForTailCallOptimization(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info, SmallVectorImpl< ArgInfo > &InArgs, SmallVectorImpl< ArgInfo > &OutArgs) const
Returns true if the call can be lowered as a tail call.
AArch64CallLowering(const AArch64TargetLowering &TLI)
bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const override
This hook must be implemented to lower the given call instruction, including argument and return valu...
bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const override
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
void setTailCallReservedStack(unsigned bytes)
SmallVectorImpl< ForwardedRegister > & getForwardedMustTailRegParms()
void setBytesInStackArgArea(unsigned bytes)
void setArgumentStackToRestore(unsigned bytes)
const AArch64RegisterInfo * getRegisterInfo() const override
const AArch64InstrInfo * getInstrInfo() const override
bool isCallingConvWin64(CallingConv::ID CC) const
const RegisterBankInfo * getRegBankInfo() const override
bool hasCustomCallingConv() const
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC) const
Selects the correct CCAssignFn for a given CallingConvention value.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const
Selects the correct CCAssignFn for a given CallingConvention value.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
void analyzeMustTailForwardedRegisters(SmallVectorImpl< ForwardedRegister > &Forwards, ArrayRef< MVT > RegParmTypes, CCAssignFn Fn)
Compute the set of registers that need to be preserved and forwarded to any musttail calls.
CallingConv::ID getCallingConv() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
bool isVarArg() const
bool isAllocated(MCRegister Reg) const
isAllocated - Return true if the specified register (or an alias) is allocated.
CCValAssign - Represent assignment of one arg/retval to a location.
LocInfo getLocInfo() const
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
bool handleAssignments(ValueHandler &Handler, SmallVectorImpl< ArgInfo > &Args, CCState &CCState, SmallVectorImpl< CCValAssign > &ArgLocs, MachineIRBuilder &MIRBuilder, ArrayRef< Register > ThisReturnRegs=std::nullopt) const
Use Handler to insert code to handle the argument/return values represented by Args.
void insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy, ArrayRef< Register > VRegs, Register DemoteReg, int FI) const
Load the returned value from the stack into virtual registers in VRegs.
bool determineAndHandleAssignments(ValueHandler &Handler, ValueAssigner &Assigner, SmallVectorImpl< ArgInfo > &Args, MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, bool IsVarArg, ArrayRef< Register > ThisReturnRegs=std::nullopt) const
Invoke ValueAssigner::assignArg on each of the given Args and then use Handler to move them to the as...
bool resultsCompatible(CallLoweringInfo &Info, MachineFunction &MF, SmallVectorImpl< ArgInfo > &InArgs, ValueAssigner &CalleeAssigner, ValueAssigner &CallerAssigner) const
void splitToValueTypes(const ArgInfo &OrigArgInfo, SmallVectorImpl< ArgInfo > &SplitArgs, const DataLayout &DL, CallingConv::ID CallConv, SmallVectorImpl< uint64_t > *Offsets=nullptr) const
Break OrigArgInfo into one or more pieces the calling convention can process, returned in SplitArgs.
void insertSRetIncomingArgument(const Function &F, SmallVectorImpl< ArgInfo > &SplitArgs, Register &DemoteReg, MachineRegisterInfo &MRI, const DataLayout &DL) const
Insert the hidden sret ArgInfo to the beginning of SplitArgs.
void insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy, ArrayRef< Register > VRegs, Register DemoteReg) const
Store the return value given by VRegs into stack starting at the offset specified in DemoteReg.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< ArgInfo > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
bool determineAssignments(ValueAssigner &Assigner, SmallVectorImpl< ArgInfo > &Args, CCState &CCInfo) const
Analyze the argument list in Args, using Assigner to populate CCInfo.
bool checkReturn(CCState &CCInfo, SmallVectorImpl< BaseArgInfo > &Outs, CCAssignFn *Fn) const
void setArgFlags(ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
void addDefToMIB(MachineRegisterInfo &MRI, MachineInstrBuilder &MIB) const
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
iterator_range< arg_iterator > args()
Definition: Function.h:845
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:264
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:358
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:528
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelType.h:159
constexpr bool isVector() const
Definition: LowLevelType.h:148
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelType.h:57
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:193
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
Definition: LowLevelType.h:203
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:33
Machine Value Type.
bool isVector() const
Return true if this is a vector value type.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setHasTailCall(bool V=true)
bool hasMustTailInVarArgFunc() const
Returns true if the function is variadic and contains a musttail call.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
Helper class to build MachineInstr.
MachineInstrBuilder insertInstr(MachineInstrBuilder MIB)
Insert an existing instruction at the insertion point.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ZEXT Op.
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
MachineInstrBuilder buildAssertZExt(const DstOp &Res, const SrcOp &Op, unsigned Size)
Build and insert Res = G_ASSERT_ZEXT Op, Size.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildPadVectorWithUndefElements(const DstOp &Res, const SrcOp &Op0)
Build and insert a, b, ..., x = G_UNMERGE_VALUES Op0 Res = G_BUILD_VECTOR a, b, .....
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_TRUNC Op.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:576
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
const Triple & getTargetTriple() const
TargetOptions Options
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
virtual const RegisterBankInfo * getRegBankInfo() const
If the information for the register banks is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static IntegerType * getInt8Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
unsigned getNumOperands() const
Definition: User.h:191
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
ArrayRef< MCPhysReg > getFPRArgRegs()
ArrayRef< MCPhysReg > getGPRArgRegs()
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ ARM64EC_Thunk_Native
Calling convention used in the ARM64EC ABI to implement calls between ARM64 code and thunks.
Definition: CallingConv.h:265
@ Swift
Calling convention for Swift.
Definition: CallingConv.h:69
@ PreserveMost
Used for runtime calls that preserves most registers.
Definition: CallingConv.h:63
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
Definition: CallingConv.h:66
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ PreserveNone
Used for runtime calls that preserves none general registers.
Definition: CallingConv.h:90
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition: CallingConv.h:76
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
Definition: CallingConv.h:87
@ ARM64EC_Thunk_X64
Calling convention used in the ARM64EC ABI to implement calls between x64 code and thunks.
Definition: CallingConv.h:260
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ Implicit
Not emitted register (e.g. carry, or temporary result).
std::optional< Function * > getAttachedARCFunction(const CallBase *CB)
This function returns operand bundle clang_arc_attachedcall's argument, which is the address of the A...
Definition: ObjCARCUtil.h:43
bool hasAttachedCallOpBundle(const CallBase *CB)
Definition: ObjCARCUtil.h:29
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
Definition: Utils.cpp:56
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2067
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:79
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
Align inferAlignFromPtrInfo(MachineFunction &MF, const MachinePointerInfo &MPO)
Definition: Utils.cpp:880
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
SmallVector< Register, 4 > Regs
Definition: CallLowering.h:63
SmallVector< ISD::ArgFlagsTy, 4 > Flags
Definition: CallLowering.h:51
Base class for ValueHandlers used for arguments coming into the current function, or for return value...
Definition: CallLowering.h:331
void assignValueToReg(Register ValVReg, Register PhysReg, const CCValAssign &VA) override
Provides a default implementation for argument handling.
Base class for ValueHandlers used for arguments passed to a function call, or for return values.
Definition: CallLowering.h:347
MachineRegisterInfo & MRI
Definition: CallLowering.h:244
virtual LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const
Return the in-memory size to write for the argument at VA.
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:203
Describes a register that needs to be forwarded from the prologue to a musttail call.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.