Line data Source code
1 : //===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : ///
10 : /// \file
11 : /// This file implements the lowering of LLVM calls to machine code calls for
12 : /// GlobalISel.
13 : ///
14 : //===----------------------------------------------------------------------===//
15 :
16 : #include "AArch64CallLowering.h"
17 : #include "AArch64ISelLowering.h"
18 : #include "AArch64MachineFunctionInfo.h"
19 : #include "AArch64Subtarget.h"
20 : #include "llvm/ADT/ArrayRef.h"
21 : #include "llvm/ADT/SmallVector.h"
22 : #include "llvm/CodeGen/Analysis.h"
23 : #include "llvm/CodeGen/CallingConvLower.h"
24 : #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
25 : #include "llvm/CodeGen/GlobalISel/Utils.h"
26 : #include "llvm/CodeGen/LowLevelType.h"
27 : #include "llvm/CodeGen/MachineBasicBlock.h"
28 : #include "llvm/CodeGen/MachineFrameInfo.h"
29 : #include "llvm/CodeGen/MachineFunction.h"
30 : #include "llvm/CodeGen/MachineInstrBuilder.h"
31 : #include "llvm/CodeGen/MachineMemOperand.h"
32 : #include "llvm/CodeGen/MachineOperand.h"
33 : #include "llvm/CodeGen/MachineRegisterInfo.h"
34 : #include "llvm/CodeGen/TargetRegisterInfo.h"
35 : #include "llvm/CodeGen/TargetSubtargetInfo.h"
36 : #include "llvm/CodeGen/ValueTypes.h"
37 : #include "llvm/IR/Argument.h"
38 : #include "llvm/IR/Attributes.h"
39 : #include "llvm/IR/Function.h"
40 : #include "llvm/IR/Type.h"
41 : #include "llvm/IR/Value.h"
42 : #include "llvm/Support/MachineValueType.h"
43 : #include <algorithm>
44 : #include <cassert>
45 : #include <cstdint>
46 : #include <iterator>
47 :
48 : using namespace llvm;
49 :
50 1570 : AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI)
51 3140 : : CallLowering(&TLI) {}
52 :
53 : namespace {
54 : struct IncomingArgHandler : public CallLowering::ValueHandler {
55 : IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
56 : CCAssignFn *AssignFn)
57 323 : : ValueHandler(MIRBuilder, MRI, AssignFn), StackUsed(0) {}
58 :
59 11 : unsigned getStackAddress(uint64_t Size, int64_t Offset,
60 : MachinePointerInfo &MPO) override {
61 11 : auto &MFI = MIRBuilder.getMF().getFrameInfo();
62 11 : int FI = MFI.CreateFixedObject(Size, Offset, true);
63 11 : MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
64 22 : unsigned AddrReg = MRI.createGenericVirtualRegister(LLT::pointer(0, 64));
65 11 : MIRBuilder.buildFrameIndex(AddrReg, FI);
66 11 : StackUsed = std::max(StackUsed, Size + Offset);
67 11 : return AddrReg;
68 : }
69 :
70 426 : void assignValueToReg(unsigned ValVReg, unsigned PhysReg,
71 : CCValAssign &VA) override {
72 426 : markPhysRegUsed(PhysReg);
73 426 : switch (VA.getLocInfo()) {
74 397 : default:
75 397 : MIRBuilder.buildCopy(ValVReg, PhysReg);
76 397 : break;
77 29 : case CCValAssign::LocInfo::SExt:
78 : case CCValAssign::LocInfo::ZExt:
79 : case CCValAssign::LocInfo::AExt: {
80 29 : auto Copy = MIRBuilder.buildCopy(LLT{VA.getLocVT()}, PhysReg);
81 29 : MIRBuilder.buildTrunc(ValVReg, Copy);
82 : break;
83 : }
84 : }
85 426 : }
86 :
87 11 : void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size,
88 : MachinePointerInfo &MPO, CCValAssign &VA) override {
89 22 : auto MMO = MIRBuilder.getMF().getMachineMemOperand(
90 : MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size,
91 : 0);
92 11 : MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
93 11 : }
94 :
95 : /// How the physical register gets marked varies between formal
96 : /// parameters (it's a basic-block live-in), and a call instruction
97 : /// (it's an implicit-def of the BL).
98 : virtual void markPhysRegUsed(unsigned PhysReg) = 0;
99 :
100 : uint64_t StackUsed;
101 : };
102 :
103 323 : struct FormalArgHandler : public IncomingArgHandler {
104 : FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
105 : CCAssignFn *AssignFn)
106 323 : : IncomingArgHandler(MIRBuilder, MRI, AssignFn) {}
107 :
108 406 : void markPhysRegUsed(unsigned PhysReg) override {
109 406 : MIRBuilder.getMBB().addLiveIn(PhysReg);
110 406 : }
111 : };
112 :
113 19 : struct CallReturnHandler : public IncomingArgHandler {
114 : CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
115 : MachineInstrBuilder MIB, CCAssignFn *AssignFn)
116 19 : : IncomingArgHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
117 :
118 20 : void markPhysRegUsed(unsigned PhysReg) override {
119 20 : MIB.addDef(PhysReg, RegState::Implicit);
120 20 : }
121 :
122 : MachineInstrBuilder MIB;
123 : };
124 :
125 47 : struct OutgoingArgHandler : public CallLowering::ValueHandler {
126 : OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
127 : MachineInstrBuilder MIB, CCAssignFn *AssignFn,
128 : CCAssignFn *AssignFnVarArg)
129 227 : : ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB),
130 227 : AssignFnVarArg(AssignFnVarArg), StackSize(0) {}
131 :
132 17 : unsigned getStackAddress(uint64_t Size, int64_t Offset,
133 : MachinePointerInfo &MPO) override {
134 17 : LLT p0 = LLT::pointer(0, 64);
135 17 : LLT s64 = LLT::scalar(64);
136 34 : unsigned SPReg = MRI.createGenericVirtualRegister(p0);
137 17 : MIRBuilder.buildCopy(SPReg, AArch64::SP);
138 :
139 34 : unsigned OffsetReg = MRI.createGenericVirtualRegister(s64);
140 17 : MIRBuilder.buildConstant(OffsetReg, Offset);
141 :
142 34 : unsigned AddrReg = MRI.createGenericVirtualRegister(p0);
143 17 : MIRBuilder.buildGEP(AddrReg, SPReg, OffsetReg);
144 :
145 17 : MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset);
146 17 : return AddrReg;
147 : }
148 :
149 290 : void assignValueToReg(unsigned ValVReg, unsigned PhysReg,
150 : CCValAssign &VA) override {
151 290 : MIB.addUse(PhysReg, RegState::Implicit);
152 290 : unsigned ExtReg = extendRegister(ValVReg, VA);
153 290 : MIRBuilder.buildCopy(PhysReg, ExtReg);
154 290 : }
155 :
156 17 : void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size,
157 : MachinePointerInfo &MPO, CCValAssign &VA) override {
158 17 : if (VA.getLocInfo() == CCValAssign::LocInfo::AExt) {
159 4 : Size = VA.getLocVT().getSizeInBits() / 8;
160 8 : ValVReg = MIRBuilder.buildAnyExt(LLT::scalar(Size * 8), ValVReg)
161 4 : ->getOperand(0)
162 4 : .getReg();
163 : }
164 34 : auto MMO = MIRBuilder.getMF().getMachineMemOperand(
165 : MPO, MachineMemOperand::MOStore, Size, 0);
166 17 : MIRBuilder.buildStore(ValVReg, Addr, *MMO);
167 17 : }
168 :
169 311 : bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT,
170 : CCValAssign::LocInfo LocInfo,
171 : const CallLowering::ArgInfo &Info,
172 : CCState &State) override {
173 : bool Res;
174 311 : if (Info.IsFixed)
175 291 : Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Info.Flags, State);
176 : else
177 20 : Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Info.Flags, State);
178 :
179 311 : StackSize = State.getNextStackOffset();
180 311 : return Res;
181 : }
182 :
183 : MachineInstrBuilder MIB;
184 : CCAssignFn *AssignFnVarArg;
185 : uint64_t StackSize;
186 : };
187 : } // namespace
188 :
189 679 : void AArch64CallLowering::splitToValueTypes(
190 : const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs,
191 : const DataLayout &DL, MachineRegisterInfo &MRI, CallingConv::ID CallConv,
192 : const SplitArgTy &PerformArgSplit) const {
193 679 : const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
194 679 : LLVMContext &Ctx = OrigArg.Ty->getContext();
195 :
196 679 : if (OrigArg.Ty->isVoidTy())
197 658 : return;
198 :
199 : SmallVector<EVT, 4> SplitVTs;
200 : SmallVector<uint64_t, 4> Offsets;
201 677 : ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0);
202 :
203 677 : if (SplitVTs.size() == 1) {
204 : // No splitting to do, but we want to replace the original type (e.g. [1 x
205 : // double] -> double).
206 656 : SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx),
207 656 : OrigArg.Flags, OrigArg.IsFixed);
208 : return;
209 : }
210 :
211 21 : unsigned FirstRegIdx = SplitArgs.size();
212 21 : bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
213 21 : OrigArg.Ty, CallConv, false);
214 116 : for (auto SplitVT : SplitVTs) {
215 95 : Type *SplitTy = SplitVT.getTypeForEVT(Ctx);
216 95 : SplitArgs.push_back(
217 95 : ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*SplitTy, DL)),
218 95 : SplitTy, OrigArg.Flags, OrigArg.IsFixed});
219 95 : if (NeedsRegBlock)
220 : SplitArgs.back().Flags.setInConsecutiveRegs();
221 : }
222 :
223 : SplitArgs.back().Flags.setInConsecutiveRegsLast();
224 :
225 116 : for (unsigned i = 0; i < Offsets.size(); ++i)
226 285 : PerformArgSplit(SplitArgs[FirstRegIdx + i].Reg, Offsets[i] * 8);
227 : }
228 :
229 319 : bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
230 : const Value *Val,
231 : ArrayRef<unsigned> VRegs) const {
232 319 : auto MIB = MIRBuilder.buildInstrNoInsert(AArch64::RET_ReallyLR);
233 : assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) &&
234 : "Return value without a vreg");
235 :
236 : bool Success = true;
237 319 : if (!VRegs.empty()) {
238 180 : MachineFunction &MF = MIRBuilder.getMF();
239 180 : const Function &F = MF.getFunction();
240 :
241 180 : MachineRegisterInfo &MRI = MF.getRegInfo();
242 180 : const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
243 180 : CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
244 180 : auto &DL = F.getParent()->getDataLayout();
245 180 : LLVMContext &Ctx = Val->getType()->getContext();
246 :
247 : SmallVector<EVT, 4> SplitEVTs;
248 180 : ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs);
249 : assert(VRegs.size() == SplitEVTs.size() &&
250 : "For each split Type there should be exactly one VReg.");
251 :
252 : SmallVector<ArgInfo, 8> SplitArgs;
253 364 : for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
254 : // We zero-extend i1s to i8.
255 184 : unsigned CurVReg = VRegs[i];
256 184 : if (MRI.getType(VRegs[i]).getSizeInBits() == 1) {
257 2 : CurVReg = MIRBuilder.buildZExt(LLT::scalar(8), CurVReg)
258 2 : ->getOperand(0)
259 2 : .getReg();
260 : }
261 :
262 368 : ArgInfo CurArgInfo = ArgInfo{CurVReg, SplitEVTs[i].getTypeForEVT(Ctx)};
263 184 : setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F);
264 368 : splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, F.getCallingConv(),
265 : [&](unsigned Reg, uint64_t Offset) {
266 0 : MIRBuilder.buildExtract(Reg, CurVReg, Offset);
267 : });
268 : }
269 :
270 : OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFn, AssignFn);
271 360 : Success = handleAssignments(MIRBuilder, SplitArgs, Handler);
272 : }
273 :
274 319 : MIRBuilder.insertInstr(MIB);
275 319 : return Success;
276 : }
277 :
278 323 : bool AArch64CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
279 : const Function &F,
280 : ArrayRef<unsigned> VRegs) const {
281 323 : MachineFunction &MF = MIRBuilder.getMF();
282 323 : MachineBasicBlock &MBB = MIRBuilder.getMBB();
283 323 : MachineRegisterInfo &MRI = MF.getRegInfo();
284 323 : auto &DL = F.getParent()->getDataLayout();
285 :
286 : SmallVector<ArgInfo, 8> SplitArgs;
287 : unsigned i = 0;
288 701 : for (auto &Arg : F.args()) {
289 378 : if (DL.getTypeStoreSize(Arg.getType()) == 0)
290 1 : continue;
291 754 : ArgInfo OrigArg{VRegs[i], Arg.getType()};
292 377 : setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F);
293 377 : bool Split = false;
294 377 : LLT Ty = MRI.getType(VRegs[i]);
295 377 : unsigned Dst = VRegs[i];
296 :
297 377 : splitToValueTypes(OrigArg, SplitArgs, DL, MRI, F.getCallingConv(),
298 : [&](unsigned Reg, uint64_t Offset) {
299 : if (!Split) {
300 : Split = true;
301 : Dst = MRI.createGenericVirtualRegister(Ty);
302 : MIRBuilder.buildUndef(Dst);
303 : }
304 : unsigned Tmp = MRI.createGenericVirtualRegister(Ty);
305 : MIRBuilder.buildInsert(Tmp, Dst, Reg, Offset);
306 : Dst = Tmp;
307 : });
308 :
309 377 : if (Dst != VRegs[i])
310 12 : MIRBuilder.buildCopy(VRegs[i], Dst);
311 : ++i;
312 : }
313 :
314 323 : if (!MBB.empty())
315 22 : MIRBuilder.setInstr(*MBB.begin());
316 :
317 323 : const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
318 : CCAssignFn *AssignFn =
319 323 : TLI.CCAssignFnForCall(F.getCallingConv(), /*IsVarArg=*/false);
320 :
321 : FormalArgHandler Handler(MIRBuilder, MRI, AssignFn);
322 646 : if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
323 : return false;
324 :
325 320 : if (F.isVarArg()) {
326 1 : if (!MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) {
327 : // FIXME: we need to reimplement saveVarArgsRegisters from
328 : // AArch64ISelLowering.
329 : return false;
330 : }
331 :
332 : // We currently pass all varargs at 8-byte alignment.
333 1 : uint64_t StackOffset = alignTo(Handler.StackUsed, 8);
334 :
335 1 : auto &MFI = MIRBuilder.getMF().getFrameInfo();
336 1 : AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
337 1 : FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
338 : }
339 :
340 320 : auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
341 320 : if (Subtarget.hasCustomCallingConv())
342 3 : Subtarget.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
343 :
344 : // Move back to the end of the basic block.
345 320 : MIRBuilder.setMBB(MBB);
346 :
347 320 : return true;
348 : }
349 :
350 47 : bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
351 : CallingConv::ID CallConv,
352 : const MachineOperand &Callee,
353 : const ArgInfo &OrigRet,
354 : ArrayRef<ArgInfo> OrigArgs) const {
355 47 : MachineFunction &MF = MIRBuilder.getMF();
356 47 : const Function &F = MF.getFunction();
357 47 : MachineRegisterInfo &MRI = MF.getRegInfo();
358 47 : auto &DL = F.getParent()->getDataLayout();
359 :
360 : SmallVector<ArgInfo, 8> SplitArgs;
361 146 : for (auto &OrigArg : OrigArgs) {
362 198 : splitToValueTypes(OrigArg, SplitArgs, DL, MRI, CallConv,
363 : [&](unsigned Reg, uint64_t Offset) {
364 36 : MIRBuilder.buildExtract(Reg, OrigArg.Reg, Offset);
365 : });
366 : }
367 :
368 : // Find out which ABI gets to decide where things go.
369 47 : const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
370 : CCAssignFn *AssignFnFixed =
371 47 : TLI.CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
372 : CCAssignFn *AssignFnVarArg =
373 47 : TLI.CCAssignFnForCall(CallConv, /*IsVarArg=*/true);
374 :
375 47 : auto CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
376 :
377 : // Create a temporarily-floating call instruction so we can add the implicit
378 : // uses of arg registers.
379 : auto MIB = MIRBuilder.buildInstrNoInsert(Callee.isReg() ? AArch64::BLR
380 90 : : AArch64::BL);
381 : MIB.add(Callee);
382 :
383 : // Tell the call which registers are clobbered.
384 47 : auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
385 47 : const uint32_t *Mask = TRI->getCallPreservedMask(MF, F.getCallingConv());
386 94 : if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv())
387 1 : TRI->UpdateCustomCallPreservedMask(MF, &Mask);
388 47 : MIB.addRegMask(Mask);
389 :
390 47 : if (TRI->isAnyArgRegReserved(MF))
391 2 : TRI->emitReservedArgRegCallError(MF);
392 :
393 : // Do the actual argument marshalling.
394 : SmallVector<unsigned, 8> PhysRegs;
395 : OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFnFixed,
396 : AssignFnVarArg);
397 94 : if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
398 : return false;
399 :
400 : // Now we can add the actual call instruction to the correct basic block.
401 45 : MIRBuilder.insertInstr(MIB);
402 :
403 : // If Callee is a reg, since it is used by a target specific
404 : // instruction, it must have a register class matching the
405 : // constraint of that instruction.
406 45 : if (Callee.isReg())
407 2 : MIB->getOperand(0).setReg(constrainOperandRegClass(
408 2 : MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
409 2 : *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Callee, 0));
410 :
411 : // Finally we can copy the returned value back into its virtual-register. In
412 : // symmetry with the arugments, the physical register must be an
413 : // implicit-define of the call instruction.
414 45 : CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
415 45 : if (OrigRet.Reg) {
416 : SplitArgs.clear();
417 :
418 : SmallVector<uint64_t, 8> RegOffsets;
419 : SmallVector<unsigned, 8> SplitRegs;
420 38 : splitToValueTypes(OrigRet, SplitArgs, DL, MRI, F.getCallingConv(),
421 : [&](unsigned Reg, uint64_t Offset) {
422 4 : RegOffsets.push_back(Offset);
423 4 : SplitRegs.push_back(Reg);
424 : });
425 :
426 : CallReturnHandler Handler(MIRBuilder, MRI, MIB, RetAssignFn);
427 19 : if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
428 : return false;
429 :
430 19 : if (!RegOffsets.empty())
431 1 : MIRBuilder.buildSequence(OrigRet.Reg, SplitRegs, RegOffsets);
432 : }
433 :
434 45 : CallSeqStart.addImm(Handler.StackSize).addImm(0);
435 45 : MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP)
436 45 : .addImm(Handler.StackSize)
437 : .addImm(0);
438 :
439 45 : return true;
440 : }
|
