50#define DEBUG_TYPE "aarch64-call-lowering"
65 if (OrigVT == MVT::i1 || OrigVT == MVT::i8)
66 ValVT = LocVT = MVT::i8;
67 else if (OrigVT == MVT::i16)
68 ValVT = LocVT = MVT::i16;
74 return (ValVT == MVT::i8 || ValVT == MVT::i16) ?
LLT(ValVT)
80struct AArch64IncomingValueAssigner
82 AArch64IncomingValueAssigner(
CCAssignFn *AssignFn_,
84 : IncomingValueAssigner(AssignFn_, AssignFnVarArg_) {}
86 bool assignArg(
unsigned ValNo,
EVT OrigVT,
MVT ValVT,
MVT LocVT,
91 return IncomingValueAssigner::assignArg(ValNo, OrigVT, ValVT, LocVT,
92 LocInfo,
Info, Flags, State);
96struct AArch64OutgoingValueAssigner
105 AArch64OutgoingValueAssigner(
CCAssignFn *AssignFn_,
109 : OutgoingValueAssigner(AssignFn_, AssignFnVarArg_),
110 Subtarget(Subtarget_), IsReturn(IsReturn) {}
112 bool assignArg(
unsigned ValNo,
EVT OrigVT,
MVT ValVT,
MVT LocVT,
117 bool UseVarArgsCCForFixed = IsCalleeWin && State.
isVarArg();
120 if (
Info.IsFixed && !UseVarArgsCCForFixed) {
123 Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State);
125 Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Flags, State);
134 : IncomingValueHandler(MIRBuilder,
MRI) {}
139 auto &MFI = MIRBuilder.getMF().getFrameInfo();
143 const bool IsImmutable = !
Flags.isByVal();
145 int FI = MFI.CreateFixedObject(
Size,
Offset, IsImmutable);
147 auto AddrReg = MIRBuilder.buildFrameIndex(
LLT::pointer(0, 64), FI);
148 return AddrReg.getReg(0);
155 if (
Flags.isPointer())
162 markPhysRegUsed(PhysReg);
163 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);
189 case CCValAssign::LocInfo::ZExt:
190 MIRBuilder.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, ValVReg,
Addr, *MMO);
192 case CCValAssign::LocInfo::SExt:
193 MIRBuilder.buildLoadInstr(TargetOpcode::G_SEXTLOAD, ValVReg,
Addr, *MMO);
196 MIRBuilder.buildLoad(ValVReg,
Addr, *MMO);
204 virtual void markPhysRegUsed(
MCRegister PhysReg) = 0;
211 void markPhysRegUsed(
MCRegister PhysReg)
override {
217struct CallReturnHandler :
public IncomingArgHandler {
220 : IncomingArgHandler(MIRBuilder,
MRI), MIB(MIB) {}
222 void markPhysRegUsed(
MCRegister PhysReg)
override {
230struct ReturnedArgCallReturnHandler :
public CallReturnHandler {
234 : CallReturnHandler(MIRBuilder,
MRI, MIB) {}
236 void markPhysRegUsed(
MCRegister PhysReg)
override {}
243 : OutgoingValueHandler(MIRBuilder,
MRI), MIB(MIB), IsTailCall(IsTailCall),
255 assert(!
Flags.isByVal() &&
"byval unhandled with tail calls");
259 auto FIReg = MIRBuilder.buildFrameIndex(p0, FI);
261 return FIReg.getReg(0);
265 SPReg = MIRBuilder.buildCopy(p0,
Register(AArch64::SP)).getReg(0);
267 auto OffsetReg = MIRBuilder.buildConstant(s64,
Offset);
269 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg);
272 return AddrReg.getReg(0);
281 if (
Flags.isPointer())
289 Register ExtReg = extendRegister(ValVReg, VA);
290 MIRBuilder.buildCopy(PhysReg, ExtReg);
299 MIRBuilder.buildStore(ValVReg,
Addr, *MMO);
313 if (VA.
getLocInfo() != CCValAssign::LocInfo::FPExt) {
322 ValVReg = extendRegister(ValVReg, VA, MaxSize);
328 assignValueToAddress(ValVReg,
Addr, MemTy, MPO, VA);
358 "Return value without a vreg");
363 }
else if (!VRegs.
empty()) {
371 auto &
DL =
F.getParent()->getDataLayout();
377 "For each split Type there should be exactly one VReg.");
382 for (
unsigned i = 0; i < SplitEVTs.
size(); ++i) {
384 ArgInfo CurArgInfo =
ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Ctx), 0};
389 auto &Flags = CurArgInfo.
Flags[0];
390 if (
MRI.getType(CurVReg).getSizeInBits() == 1 && !Flags.isSExt() &&
397 if (
EVT(NewVT) != SplitEVTs[i]) {
398 unsigned ExtendOp = TargetOpcode::G_ANYEXT;
399 if (
F.getAttributes().hasRetAttr(Attribute::SExt))
400 ExtendOp = TargetOpcode::G_SEXT;
401 else if (
F.getAttributes().hasRetAttr(Attribute::ZExt))
402 ExtendOp = TargetOpcode::G_ZEXT;
418 CurVReg = MIRBuilder.
buildInstr(ExtendOp, {NewLLT}, {CurVReg})
437 if (NewLLT !=
MRI.getType(CurVReg)) {
439 CurVReg = MIRBuilder.
buildInstr(ExtendOp, {NewLLT}, {CurVReg})
445 if (CurVReg != CurArgInfo.
Regs[0]) {
446 CurArgInfo.
Regs[0] = CurVReg;
453 AArch64OutgoingValueAssigner Assigner(AssignFn, AssignFn, Subtarget,
455 OutgoingArgHandler Handler(MIRBuilder,
MRI, MIB);
457 MIRBuilder,
CC,
F.isVarArg());
460 if (SwiftErrorVReg) {
462 MIRBuilder.
buildCopy(AArch64::X21, SwiftErrorVReg);
472 bool IsVarArg)
const {
474 const auto &TLI = *getTLI<AArch64TargetLowering>();
475 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,
478 return checkReturn(CCInfo, Outs, TLI.CCAssignFnForReturn(CallConv));
495 assert(
F.isVarArg() &&
"Expected F to be vararg?");
499 CCState CCInfo(
F.getCallingConv(),
true, MF, ArgLocs,
518 for (
const auto &
F : Forwards) {
526 if (
F.getReturnType()->isScalableTy() ||
528 return A.getType()->isScalableTy();
532 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
533 LLVM_DEBUG(
dbgs() <<
"Falling back to SDAG because we don't support no-NEON\n");
538 if (Attrs.hasZAState() || Attrs.hasZT0State() ||
539 Attrs.hasStreamingInterfaceOrBody() ||
540 Attrs.hasStreamingCompatibleInterface())
546void AArch64CallLowering::saveVarArgRegisters(
563 unsigned NumVariadicGPRArgRegs =
GPRArgRegs.size() - FirstVariadicGPR + 1;
565 unsigned GPRSaveSize = 8 * (
GPRArgRegs.size() - FirstVariadicGPR);
567 if (GPRSaveSize != 0) {
570 -
static_cast<int>(GPRSaveSize),
false);
571 if (GPRSaveSize & 15)
574 -
static_cast<int>(
alignTo(GPRSaveSize, 16)),
583 for (
unsigned i = FirstVariadicGPR; i <
GPRArgRegs.size(); ++i) {
584 Register Val =
MRI.createGenericVirtualRegister(s64);
590 MF, GPRIdx, (i - FirstVariadicGPR) * 8)
594 FIN = MIRBuilder.
buildPtrAdd(
MRI.createGenericVirtualRegister(p0),
601 if (Subtarget.hasFPARMv8() && !IsWin64CC) {
604 unsigned FPRSaveSize = 16 * (
FPRArgRegs.size() - FirstVariadicFPR);
606 if (FPRSaveSize != 0) {
613 for (
unsigned i = FirstVariadicFPR; i <
FPRArgRegs.size(); ++i) {
624 FIN = MIRBuilder.
buildPtrAdd(
MRI.createGenericVirtualRegister(p0),
639 auto &
DL =
F.getParent()->getDataLayout();
644 if (
F.isVarArg() && Subtarget.isWindowsArm64EC())
653 bool IsWin64 = Subtarget.isCallingConvWin64(
F.getCallingConv()) && !Subtarget.isWindowsArm64EC();
664 for (
auto &Arg :
F.args()) {
665 if (
DL.getTypeStoreSize(Arg.getType()).isZero())
668 ArgInfo OrigArg{VRegs[i], Arg, i};
673 if (OrigArg.Ty->isIntegerTy(1)) {
674 assert(OrigArg.Regs.size() == 1 &&
675 MRI.getType(OrigArg.Regs[0]).getSizeInBits() == 1 &&
676 "Unexpected registers used for i1 arg");
678 auto &Flags = OrigArg.Flags[0];
679 if (!Flags.isZExt() && !Flags.isSExt()) {
683 OrigArg.Regs[0] = WideReg;
688 if (Arg.hasAttribute(Attribute::SwiftAsync))
701 AArch64IncomingValueAssigner Assigner(AssignFn, AssignFn);
704 CCState CCInfo(
F.getCallingConv(),
F.isVarArg(), MF, ArgLocs,
F.getContext());
709 if (!BoolArgs.
empty()) {
710 for (
auto &KV : BoolArgs) {
713 LLT WideTy =
MRI.getType(WideReg);
714 assert(
MRI.getType(OrigReg).getScalarSizeInBits() == 1 &&
715 "Unexpected bit size of a bool arg");
722 uint64_t StackSize = Assigner.StackSize;
724 if ((!Subtarget.isTargetDarwin() && !Subtarget.isWindowsArm64EC()) || IsWin64) {
730 saveVarArgRegisters(MIRBuilder, Handler, CCInfo);
731 }
else if (Subtarget.isWindowsArm64EC()) {
736 StackSize =
alignTo(Assigner.StackSize, Subtarget.isTargetILP32() ? 4 : 8);
746 StackSize =
alignTo(StackSize, 16);
762 if (Subtarget.hasCustomCallingConv())
763 Subtarget.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
797static std::pair<CCAssignFn *, CCAssignFn *>
802bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
810 if (CalleeCC == CallerCC)
817 std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =
822 std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =
825 AArch64IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,
826 CalleeAssignFnVarArg);
827 AArch64IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,
828 CallerAssignFnVarArg);
835 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
836 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
838 TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
839 TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
842 return TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved);
845bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
849 if (OrigOutArgs.
empty())
865 CCState OutInfo(CalleeCC,
false, MF, OutLocs, Ctx);
867 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
880 LLVM_DEBUG(
dbgs() <<
"... Cannot fit call operands on caller's stack.\n");
888 const uint32_t *CallerPreservedMask =
TRI->getCallPreservedMask(MF, CallerCC);
897 for (
unsigned i = 0; i < OutLocs.
size(); ++i) {
898 auto &ArgLoc = OutLocs[i];
899 if (ArgLoc.isRegLoc())
904 <<
"... Cannot tail call vararg function with stack arguments\n");
918 if (!
Info.IsTailCall)
927 if (
Info.SwiftErrorVReg) {
932 LLVM_DEBUG(
dbgs() <<
"... Cannot handle tail calls with swifterror yet.\n");
937 LLVM_DEBUG(
dbgs() <<
"... Calling convention cannot be tail called.\n");
959 return A.hasByValAttr() || A.hasInRegAttr() || A.hasSwiftErrorAttr();
961 LLVM_DEBUG(
dbgs() <<
"... Cannot tail call from callers with byval, "
962 "inreg, or swifterror arguments\n");
973 if (
Info.Callee.isGlobal()) {
977 (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
978 TT.isOSBinFormatMachO())) {
979 LLVM_DEBUG(
dbgs() <<
"... Cannot tail call externally-defined function "
980 "with weak linkage for this OS.\n");
995 "Unexpected variadic calling convention");
999 if (!doCallerAndCalleePassArgsTheSameWay(
Info, MF, InArgs)) {
1002 <<
"... Caller and callee have incompatible calling conventions.\n");
1006 if (!areCalleeOutgoingArgsTailCallable(
Info, MF, OutArgs))
1010 dbgs() <<
"... Call is eligible for tail call optimization.\n");
1022 return AArch64::TCRETURNdi;
1028 return AArch64::TCRETURNrix17;
1030 return AArch64::TCRETURNrix16x17;
1032 return AArch64::TCRETURNrinotx16;
1034 return AArch64::TCRETURNri;
1042 if (!OutArgs.
empty() && OutArgs[0].Flags[0].isReturned()) {
1044 Mask =
TRI.getThisReturnPreservedMask(MF,
Info.CallConv);
1046 OutArgs[0].Flags[0].setReturned(
false);
1047 Mask =
TRI.getCallPreservedMask(MF,
Info.CallConv);
1050 Mask =
TRI.getCallPreservedMask(MF,
Info.CallConv);
1055bool AArch64CallLowering::lowerTailCall(
1073 dbgs() <<
"Cannot lower indirect tail calls with BTI enabled yet.\n");
1085 CallSeqStart = MIRBuilder.
buildInstr(AArch64::ADJCALLSTACKDOWN);
1100 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
1101 MIB.addRegMask(Mask);
1104 MIB->setCFIType(MF,
Info.CFIType->getZExtValue());
1106 if (
TRI->isAnyArgRegReserved(MF))
1107 TRI->emitReservedArgRegCallError(MF);
1119 unsigned NumBytes = 0;
1126 CCState OutInfo(CalleeCC,
false, MF, OutLocs,
F.getContext());
1128 AArch64OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg,
1135 NumBytes =
alignTo(OutInfo.getStackSize(), 16);
1140 FPDiff = NumReusableBytes - NumBytes;
1144 if (FPDiff < 0 && FuncInfo->getTailCallReservedStack() < (
unsigned)-FPDiff)
1152 assert(FPDiff % 16 == 0 &&
"unaligned stack on tail call");
1157 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1161 OutgoingArgHandler Handler(MIRBuilder,
MRI, MIB,
1164 CalleeCC,
Info.IsVarArg))
1169 if (
Info.IsVarArg &&
Info.IsMustTailCall) {
1173 for (
const auto &
F : Forwards) {
1180 return TRI->regsOverlap(Use.getReg(), ForwardedReg);
1207 if (MIB->getOperand(0).isReg())
1210 MIB->getDesc(), MIB->getOperand(0), 0);
1213 Info.LoweredTailCall =
true;
1222 auto &
DL =
F.getParent()->getDataLayout();
1240 for (
auto &OrigArg :
Info.OrigArgs) {
1243 auto &Flags = OrigArg.Flags[0];
1244 if (OrigArg.Ty->isIntegerTy(1) && !Flags.isSExt() && !Flags.isZExt()) {
1247 MRI.getType(OutArg.
Regs[0]).getSizeInBits() == 1 &&
1248 "Unexpected registers used for i1 arg");
1260 if (!
Info.OrigRet.Ty->isVoidTy())
1264 bool CanTailCallOpt =
1268 if (
Info.IsMustTailCall && !CanTailCallOpt) {
1272 LLVM_DEBUG(
dbgs() <<
"Failed to lower musttail call as tail call\n");
1276 Info.IsTailCall = CanTailCallOpt;
1278 return lowerTailCall(MIRBuilder,
Info, OutArgs);
1283 std::tie(AssignFnFixed, AssignFnVarArg) =
1287 CallSeqStart = MIRBuilder.
buildInstr(AArch64::ADJCALLSTACKDOWN);
1297 Opc = AArch64::BLR_RVMARKER;
1300 else if (
Info.CB &&
Info.CB->hasFnAttr(Attribute::ReturnsTwice) &&
1301 !Subtarget.noBTIAtReturnTwice() &&
1303 Opc = AArch64::BLR_BTI;
1307 if (
Info.Callee.isSymbol() &&
F.getParent()->getRtLibUseGOT()) {
1308 auto MIB = MIRBuilder.
buildInstr(TargetOpcode::G_GLOBAL_VALUE);
1317 unsigned CalleeOpNo = 0;
1319 if (Opc == AArch64::BLR_RVMARKER) {
1323 MIB.addGlobalAddress(ARCFn);
1325 }
else if (
Info.CFIType) {
1326 MIB->setCFIType(MF,
Info.CFIType->getZExtValue());
1329 MIB.add(
Info.Callee);
1335 AArch64OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg,
1338 OutgoingArgHandler Handler(MIRBuilder,
MRI, MIB,
false);
1346 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
1347 MIB.addRegMask(Mask);
1349 if (
TRI->isAnyArgRegReserved(MF))
1350 TRI->emitReservedArgRegCallError(MF);
1358 ?
alignTo(Assigner.StackSize, 16)
1362 MIRBuilder.
buildInstr(AArch64::ADJCALLSTACKUP)
1363 .
addImm(Assigner.StackSize)
1369 if (MIB->getOperand(CalleeOpNo).isReg())
1372 MIB->getOperand(CalleeOpNo), CalleeOpNo);
1377 if (
Info.CanLowerReturn && !
Info.OrigRet.Ty->isVoidTy()) {
1379 CallReturnHandler Handler(MIRBuilder,
MRI, MIB);
1380 bool UsingReturnedArg =
1381 !OutArgs.
empty() && OutArgs[0].Flags[0].isReturned();
1383 AArch64OutgoingValueAssigner Assigner(RetAssignFn, RetAssignFn, Subtarget,
1385 ReturnedArgCallReturnHandler ReturnedArgHandler(MIRBuilder,
MRI, MIB);
1387 UsingReturnedArg ? ReturnedArgHandler : Handler, Assigner, InArgs,
1388 MIRBuilder,
Info.CallConv,
Info.IsVarArg,
1389 UsingReturnedArg ?
ArrayRef(OutArgs[0].Regs) : std::nullopt))
1393 if (
Info.SwiftErrorVReg) {
1398 if (!
Info.CanLowerReturn) {
1400 Info.DemoteRegister,
Info.DemoteStackIndex);
unsigned const MachineRegisterInfo * MRI
static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder, CCAssignFn *AssignFn)
Helper function to compute forwarded registers for musttail calls.
static LLT getStackValueStoreTypeHack(const CCValAssign &VA)
static const uint32_t * getMaskForArgs(SmallVectorImpl< AArch64CallLowering::ArgInfo > &OutArgs, AArch64CallLowering::CallLoweringInfo &Info, const AArch64RegisterInfo &TRI, MachineFunction &MF)
static void applyStackPassedSmallTypeDAGHack(EVT OrigVT, MVT &ValVT, MVT &LocVT)
static std::pair< CCAssignFn *, CCAssignFn * > getAssignFnsForCC(CallingConv::ID CC, const AArch64TargetLowering &TLI)
Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for CC.
static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect, bool IsTailCall)
static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt)
This file describes how to lower LLVM calls to machine code calls.
static const MCPhysReg GPRArgRegs[]
static const MCPhysReg FPRArgRegs[]
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
static bool mayTailCallThisCC(CallingConv::ID CC)
Return true if we might ever do TCO for calls with this calling convention.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
Implement a low-level type suitable for MachineInstr level instruction selection.
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
This file defines ARC utility functions which are used by various parts of the compiler.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const override
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
bool canLowerReturn(MachineFunction &MF, CallingConv::ID CallConv, SmallVectorImpl< BaseArgInfo > &Outs, bool IsVarArg) const override
This hook must be implemented to check whether the return values described by Outs can fit into the r...
bool fallBackToDAGISel(const MachineFunction &MF) const override
bool isTypeIsValidForThisReturn(EVT Ty) const override
For targets which support the "returned" parameter attribute, returns true if the given type is a val...
bool isEligibleForTailCallOptimization(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info, SmallVectorImpl< ArgInfo > &InArgs, SmallVectorImpl< ArgInfo > &OutArgs) const
Returns true if the call can be lowered as a tail call.
AArch64CallLowering(const AArch64TargetLowering &TLI)
bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const override
This hook must be implemented to lower the given call instruction, including argument and return valu...
bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const override
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool branchTargetEnforcement() const
void setVarArgsStackIndex(int Index)
void setTailCallReservedStack(unsigned bytes)
SmallVectorImpl< ForwardedRegister > & getForwardedMustTailRegParms()
void setBytesInStackArgArea(unsigned bytes)
void setVarArgsGPRIndex(int Index)
bool branchProtectionPAuthLR() const
void setVarArgsFPRSize(unsigned Size)
unsigned getBytesInStackArgArea() const
void setVarArgsFPRIndex(int Index)
void setVarArgsGPRSize(unsigned Size)
void setArgumentStackToRestore(unsigned bytes)
const AArch64RegisterInfo * getRegisterInfo() const override
const AArch64InstrInfo * getInstrInfo() const override
bool isCallingConvWin64(CallingConv::ID CC) const
bool isWindowsArm64EC() const
const RegisterBankInfo * getRegBankInfo() const override
bool hasCustomCallingConv() const
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC) const
Selects the correct CCAssignFn for a given CallingConvention value.
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const
Selects the correct CCAssignFn for a given CallingConvention value.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
void analyzeMustTailForwardedRegisters(SmallVectorImpl< ForwardedRegister > &Forwards, ArrayRef< MVT > RegParmTypes, CCAssignFn Fn)
Compute the set of registers that need to be preserved and forwarded to any musttail calls.
CallingConv::ID getCallingConv() const
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
bool isAllocated(MCRegister Reg) const
isAllocated - Return true if the specified register (or an alias) is allocated.
CCValAssign - Represent assignment of one arg/retval to a location.
LocInfo getLocInfo() const
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
bool handleAssignments(ValueHandler &Handler, SmallVectorImpl< ArgInfo > &Args, CCState &CCState, SmallVectorImpl< CCValAssign > &ArgLocs, MachineIRBuilder &MIRBuilder, ArrayRef< Register > ThisReturnRegs=std::nullopt) const
Use Handler to insert code to handle the argument/return values represented by Args.
void insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy, ArrayRef< Register > VRegs, Register DemoteReg, int FI) const
Load the returned value from the stack into virtual registers in VRegs.
bool determineAndHandleAssignments(ValueHandler &Handler, ValueAssigner &Assigner, SmallVectorImpl< ArgInfo > &Args, MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, bool IsVarArg, ArrayRef< Register > ThisReturnRegs=std::nullopt) const
Invoke ValueAssigner::assignArg on each of the given Args and then use Handler to move them to the as...
bool resultsCompatible(CallLoweringInfo &Info, MachineFunction &MF, SmallVectorImpl< ArgInfo > &InArgs, ValueAssigner &CalleeAssigner, ValueAssigner &CallerAssigner) const
void splitToValueTypes(const ArgInfo &OrigArgInfo, SmallVectorImpl< ArgInfo > &SplitArgs, const DataLayout &DL, CallingConv::ID CallConv, SmallVectorImpl< uint64_t > *Offsets=nullptr) const
Break OrigArgInfo into one or more pieces the calling convention can process, returned in SplitArgs.
void insertSRetIncomingArgument(const Function &F, SmallVectorImpl< ArgInfo > &SplitArgs, Register &DemoteReg, MachineRegisterInfo &MRI, const DataLayout &DL) const
Insert the hidden sret ArgInfo to the beginning of SplitArgs.
void insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy, ArrayRef< Register > VRegs, Register DemoteReg) const
Store the return value given by VRegs into stack starting at the offset specified in DemoteReg.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< ArgInfo > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
bool determineAssignments(ValueAssigner &Assigner, SmallVectorImpl< ArgInfo > &Args, CCState &CCInfo) const
Analyze the argument list in Args, using Assigner to populate CCInfo.
bool checkReturn(CCState &CCInfo, SmallVectorImpl< BaseArgInfo > &Outs, CCAssignFn *Fn) const
void setArgFlags(ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const
A parsed version of the target data layout string in and methods for querying it.
void addDefToMIB(MachineRegisterInfo &MRI, MachineInstrBuilder &MIB) const
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
iterator_range< arg_iterator > args()
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasExternalWeakLinkage() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
This is an important class for using LLVM in a threaded context.
Wrapper class representing physical registers. Should be passed by value.
bool isVector() const
Return true if this is a vector value type.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setHasTailCall(bool V=true)
bool hasMustTailInVarArgFunc() const
Returns true if the function is variadic and contains a musttail call.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
Helper class to build MachineInstr.
MachineInstrBuilder insertInstr(MachineInstrBuilder MIB)
Insert an existing instruction at the insertion point.
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
MachineInstrBuilder buildAssertZExt(const DstOp &Res, const SrcOp &Op, unsigned Size)
Build and insert Res = G_ASSERT_ZEXT Op, Size.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildPadVectorWithUndefElements(const DstOp &Res, const SrcOp &Op0)
Build and insert a, b, ..., x = G_UNMERGE_VALUES Op0 Res = G_BUILD_VECTOR a, b, .....
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
MachineFunction & getMF()
Getter for the function we currently build.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineOperand & getOperand(unsigned i) const
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
Wrapper class representing virtual and physical registers.
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
const Triple & getTargetTriple() const
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
virtual const RegisterBankInfo * getRegBankInfo() const
If the information for the register banks is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Triple - Helper class for working with autoconf configuration names.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
ArrayRef< MCPhysReg > getFPRArgRegs()
ArrayRef< MCPhysReg > getGPRArgRegs()
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ ARM64EC_Thunk_Native
Calling convention used in the ARM64EC ABI to implement calls between ARM64 code and thunks.
@ Swift
Calling convention for Swift.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ ARM64EC_Thunk_X64
Calling convention used in the ARM64EC ABI to implement calls between x64 code and thunks.
@ C
The default llvm calling convention, compatible with C.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
std::optional< Function * > getAttachedARCFunction(const CallBase *CB)
This function returns operand bundle clang_arc_attachedcall's argument, which is the address of the A...
bool hasAttachedCallOpBundle(const CallBase *CB)
This is an optimization pass for GlobalISel generic memory operations.
Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
unsigned getBLRCallOpcode(const MachineFunction &MF)
Return opcode to be used for indirect calls.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
Align inferAlignFromPtrInfo(MachineFunction &MF, const MachinePointerInfo &MPO)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
SmallVector< Register, 4 > Regs
SmallVector< ISD::ArgFlagsTy, 4 > Flags
Base class for ValueHandlers used for arguments coming into the current function, or for return value...
void assignValueToReg(Register ValVReg, Register PhysReg, const CCValAssign &VA) override
Provides a default implementation for argument handling.
Base class for ValueHandlers used for arguments passed to a function call, or for return values.
MachineIRBuilder & MIRBuilder
MachineRegisterInfo & MRI
virtual LLT getStackValueStoreType(const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const
Return the in-memory size to write for the argument at VA.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Describes a register that needs to be forwarded from the prologue to a musttail call.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.