LLVM 18.0.0git
SIMachineFunctionInfo.cpp
Go to the documentation of this file.
1//===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10#include "AMDGPUSubtarget.h"
11#include "AMDGPUTargetMachine.h"
12#include "GCNSubtarget.h"
14#include "SIRegisterInfo.h"
22#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Function.h"
25#include <cassert>
26#include <optional>
27#include <vector>
28
29#define MAX_LANES 64
30
31using namespace llvm;
32
34 const SITargetLowering *TLI = STI->getTargetLowering();
35 return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine());
36}
37
39 const GCNSubtarget *STI)
40 : AMDGPUMachineFunction(F, *STI), Mode(F), GWSResourcePSV(getTM(STI)),
41 UserSGPRInfo(F, *STI), WorkGroupIDX(false), WorkGroupIDY(false),
42 WorkGroupIDZ(false), WorkGroupInfo(false), LDSKernelId(false),
43 PrivateSegmentWaveByteOffset(false), WorkItemIDX(false),
44 WorkItemIDY(false), WorkItemIDZ(false), ImplicitArgPtr(false),
45 GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0) {
46 const GCNSubtarget &ST = *static_cast<const GCNSubtarget *>(STI);
47 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
48 WavesPerEU = ST.getWavesPerEU(F);
49
50 Occupancy = ST.computeOccupancy(F, getLDSSize());
51 CallingConv::ID CC = F.getCallingConv();
52
53 VRegFlags.reserve(1024);
54
55 const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||
57
58 if (IsKernel) {
59 WorkGroupIDX = true;
60 WorkItemIDX = true;
61 } else if (CC == CallingConv::AMDGPU_PS) {
62 PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
63 }
64
65 MayNeedAGPRs = ST.hasMAIInsts();
66
67 if (AMDGPU::isChainCC(CC)) {
68 // Chain functions don't receive an SP from their caller, but are free to
69 // set one up. For now, we can use s32 to match what amdgpu_gfx functions
70 // would use if called, but this can be revisited.
71 // FIXME: Only reserve this if we actually need it.
72 StackPtrOffsetReg = AMDGPU::SGPR32;
73
74 ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51;
75
76 ArgInfo.PrivateSegmentBuffer =
77 ArgDescriptor::createRegister(ScratchRSrcReg);
78
79 ImplicitArgPtr = false;
80 } else if (!isEntryFunction()) {
83
84 // TODO: Pick a high register, and shift down, similar to a kernel.
85 FrameOffsetReg = AMDGPU::SGPR33;
86 StackPtrOffsetReg = AMDGPU::SGPR32;
87
88 if (!ST.enableFlatScratch()) {
89 // Non-entry functions have no special inputs for now, other registers
90 // required for scratch access.
91 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
92
93 ArgInfo.PrivateSegmentBuffer =
94 ArgDescriptor::createRegister(ScratchRSrcReg);
95 }
96
97 if (!F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))
98 ImplicitArgPtr = true;
99 } else {
100 ImplicitArgPtr = false;
101 MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(),
103
104 if (ST.hasGFX90AInsts() &&
105 ST.getMaxNumVGPRs(F) <= AMDGPU::VGPR_32RegClass.getNumRegs() &&
106 !mayUseAGPRs(F))
107 MayNeedAGPRs = false; // We will select all MAI with VGPR operands.
108 }
109
110 if (!AMDGPU::isGraphics(CC) ||
111 (CC == CallingConv::AMDGPU_CS && ST.hasArchitectedSGPRs())) {
112 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x"))
113 WorkGroupIDX = true;
114
115 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-y"))
116 WorkGroupIDY = true;
117
118 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-z"))
119 WorkGroupIDZ = true;
120 }
121
122 if (!AMDGPU::isGraphics(CC)) {
123 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workitem-id-x"))
124 WorkItemIDX = true;
125
126 if (!F.hasFnAttribute("amdgpu-no-workitem-id-y") &&
127 ST.getMaxWorkitemID(F, 1) != 0)
128 WorkItemIDY = true;
129
130 if (!F.hasFnAttribute("amdgpu-no-workitem-id-z") &&
131 ST.getMaxWorkitemID(F, 2) != 0)
132 WorkItemIDZ = true;
133
134 if (!IsKernel && !F.hasFnAttribute("amdgpu-no-lds-kernel-id"))
135 LDSKernelId = true;
136 }
137
138 if (isEntryFunction()) {
139 // X, XY, and XYZ are the only supported combinations, so make sure Y is
140 // enabled if Z is.
141 if (WorkItemIDZ)
142 WorkItemIDY = true;
143
144 if (!ST.flatScratchIsArchitected()) {
145 PrivateSegmentWaveByteOffset = true;
146
147 // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
148 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
150 ArgInfo.PrivateSegmentWaveByteOffset =
151 ArgDescriptor::createRegister(AMDGPU::SGPR5);
152 }
153 }
154
155 Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");
156 StringRef S = A.getValueAsString();
157 if (!S.empty())
158 S.consumeInteger(0, GITPtrHigh);
159
160 A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
161 S = A.getValueAsString();
162 if (!S.empty())
163 S.consumeInteger(0, HighBitsOf32BitAddress);
164
165 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch
166 // VGPR available at all times. For now, reserve highest available VGPR. After
167 // RA, shift it to the lowest available unused VGPR if the one exist.
168 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
169 VGPRForAGPRCopy =
170 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(F) - 1);
171 }
172}
173
175 BumpPtrAllocator &Allocator, MachineFunction &DestMF,
177 const {
178 return DestMF.cloneInfo<SIMachineFunctionInfo>(*this);
179}
180
183 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
184 limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
185 MF.getFunction()));
186}
187
189 const SIRegisterInfo &TRI) {
190 ArgInfo.PrivateSegmentBuffer =
191 ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
192 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
193 NumUserSGPRs += 4;
194 return ArgInfo.PrivateSegmentBuffer.getRegister();
195}
196
198 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
199 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
200 NumUserSGPRs += 2;
201 return ArgInfo.DispatchPtr.getRegister();
202}
203
205 ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
206 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
207 NumUserSGPRs += 2;
208 return ArgInfo.QueuePtr.getRegister();
209}
210
212 ArgInfo.KernargSegmentPtr
213 = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
214 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
215 NumUserSGPRs += 2;
216 return ArgInfo.KernargSegmentPtr.getRegister();
217}
218
220 ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
221 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
222 NumUserSGPRs += 2;
223 return ArgInfo.DispatchID.getRegister();
224}
225
227 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
228 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
229 NumUserSGPRs += 2;
230 return ArgInfo.FlatScratchInit.getRegister();
231}
232
234 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
235 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
236 NumUserSGPRs += 2;
237 return ArgInfo.ImplicitBufferPtr.getRegister();
238}
239
241 ArgInfo.LDSKernelId = ArgDescriptor::createRegister(getNextUserSGPR());
242 NumUserSGPRs += 1;
243 return ArgInfo.LDSKernelId.getRegister();
244}
245
247 const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
248 unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs) {
249 assert(!ArgInfo.PreloadKernArgs.count(KernArgIdx) &&
250 "Preload kernel argument allocated twice.");
251 NumUserSGPRs += PaddingSGPRs;
252 // If the available register tuples are aligned with the kernarg to be
253 // preloaded use that register, otherwise we need to use a set of SGPRs and
254 // merge them.
255 Register PreloadReg =
256 TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC);
257 if (PreloadReg &&
258 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {
259 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(PreloadReg);
260 NumUserSGPRs += AllocSizeDWord;
261 } else {
262 for (unsigned I = 0; I < AllocSizeDWord; ++I) {
263 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(getNextUserSGPR());
264 NumUserSGPRs++;
265 }
266 }
267
268 // Track the actual number of SGPRs that HW will preload to.
269 UserSGPRInfo.allocKernargPreloadSGPRs(AllocSizeDWord + PaddingSGPRs);
270 return &ArgInfo.PreloadKernArgs[KernArgIdx].Regs;
271}
272
274 uint64_t Size, Align Alignment) {
275 // Skip if it is an entry function or the register is already added.
276 if (isEntryFunction() || WWMSpills.count(VGPR))
277 return;
278
279 WWMSpills.insert(std::make_pair(
280 VGPR, MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));
281}
282
283// Separate out the callee-saved and scratch registers.
285 MachineFunction &MF,
286 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
287 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const {
288 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
289 for (auto &Reg : WWMSpills) {
290 if (isCalleeSavedReg(CSRegs, Reg.first))
291 CalleeSavedRegs.push_back(Reg);
292 else
293 ScratchRegs.push_back(Reg);
294 }
295}
296
298 MCPhysReg Reg) const {
299 for (unsigned I = 0; CSRegs[I]; ++I) {
300 if (CSRegs[I] == Reg)
301 return true;
302 }
303
304 return false;
305}
306
307bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
308 MachineFunction &MF, int FI, unsigned LaneIndex) {
310 Register LaneVGPR;
311 if (!LaneIndex) {
312 LaneVGPR = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
313 SpillVGPRs.push_back(LaneVGPR);
314 } else {
315 LaneVGPR = SpillVGPRs.back();
316 }
317
318 SGPRSpillsToVirtualVGPRLanes[FI].push_back(
319 SIRegisterInfo::SpilledReg(LaneVGPR, LaneIndex));
320 return true;
321}
322
323bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
324 MachineFunction &MF, int FI, unsigned LaneIndex) {
326 const SIRegisterInfo *TRI = ST.getRegisterInfo();
328 Register LaneVGPR;
329 if (!LaneIndex) {
330 LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
331 if (LaneVGPR == AMDGPU::NoRegister) {
332 // We have no VGPRs left for spilling SGPRs. Reset because we will not
333 // partially spill the SGPR to VGPRs.
334 SGPRSpillsToPhysicalVGPRLanes.erase(FI);
335 return false;
336 }
337
338 allocateWWMSpill(MF, LaneVGPR);
339 reserveWWMRegister(LaneVGPR);
340 for (MachineBasicBlock &MBB : MF) {
341 MBB.addLiveIn(LaneVGPR);
343 }
344 } else {
345 LaneVGPR = WWMReservedRegs.back();
346 }
347
348 SGPRSpillsToPhysicalVGPRLanes[FI].push_back(
349 SIRegisterInfo::SpilledReg(LaneVGPR, LaneIndex));
350 return true;
351}
352
354 int FI,
355 bool IsPrologEpilog) {
356 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
357 IsPrologEpilog ? SGPRSpillsToPhysicalVGPRLanes[FI]
358 : SGPRSpillsToVirtualVGPRLanes[FI];
359
360 // This has already been allocated.
361 if (!SpillLanes.empty())
362 return true;
363
364 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
365 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
366 unsigned WaveSize = ST.getWavefrontSize();
367
368 unsigned Size = FrameInfo.getObjectSize(FI);
369 unsigned NumLanes = Size / 4;
370
371 if (NumLanes > WaveSize)
372 return false;
373
374 assert(Size >= 4 && "invalid sgpr spill size");
375 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
376 "not spilling SGPRs to VGPRs");
377
378 unsigned &NumSpillLanes =
379 IsPrologEpilog ? NumPhysicalVGPRSpillLanes : NumVirtualVGPRSpillLanes;
380
381 for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) {
382 unsigned LaneIndex = (NumSpillLanes % WaveSize);
383
384 bool Allocated = IsPrologEpilog
385 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex)
386 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
387 if (!Allocated) {
388 NumSpillLanes -= I;
389 return false;
390 }
391 }
392
393 return true;
394}
395
396/// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
397/// Either AGPR is spilled to VGPR to vice versa.
398/// Returns true if a \p FI can be eliminated completely.
400 int FI,
401 bool isAGPRtoVGPR) {
403 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
404 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
405
406 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
407
408 auto &Spill = VGPRToAGPRSpills[FI];
409
410 // This has already been allocated.
411 if (!Spill.Lanes.empty())
412 return Spill.FullyAllocated;
413
414 unsigned Size = FrameInfo.getObjectSize(FI);
415 unsigned NumLanes = Size / 4;
416 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
417
418 const TargetRegisterClass &RC =
419 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
420 auto Regs = RC.getRegisters();
421
422 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
423 const SIRegisterInfo *TRI = ST.getRegisterInfo();
424 Spill.FullyAllocated = true;
425
426 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
427 // once.
428 BitVector OtherUsedRegs;
429 OtherUsedRegs.resize(TRI->getNumRegs());
430
431 const uint32_t *CSRMask =
432 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
433 if (CSRMask)
434 OtherUsedRegs.setBitsInMask(CSRMask);
435
436 // TODO: Should include register tuples, but doesn't matter with current
437 // usage.
438 for (MCPhysReg Reg : SpillAGPR)
439 OtherUsedRegs.set(Reg);
440 for (MCPhysReg Reg : SpillVGPR)
441 OtherUsedRegs.set(Reg);
442
443 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
444 for (int I = NumLanes - 1; I >= 0; --I) {
445 NextSpillReg = std::find_if(
446 NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
447 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
448 !OtherUsedRegs[Reg];
449 });
450
451 if (NextSpillReg == Regs.end()) { // Registers exhausted
452 Spill.FullyAllocated = false;
453 break;
454 }
455
456 OtherUsedRegs.set(*NextSpillReg);
457 SpillRegs.push_back(*NextSpillReg);
458 MRI.reserveReg(*NextSpillReg, TRI);
459 Spill.Lanes[I] = *NextSpillReg++;
460 }
461
462 return Spill.FullyAllocated;
463}
464
466 MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) {
467 // Remove dead frame indices from function frame, however keep FP & BP since
468 // spills for them haven't been inserted yet. And also make sure to remove the
469 // frame indices from `SGPRSpillsToVirtualVGPRLanes` data structure,
470 // otherwise, it could result in an unexpected side effect and bug, in case of
471 // any re-mapping of freed frame indices by later pass(es) like "stack slot
472 // coloring".
473 for (auto &R : make_early_inc_range(SGPRSpillsToVirtualVGPRLanes)) {
474 MFI.RemoveStackObject(R.first);
475 SGPRSpillsToVirtualVGPRLanes.erase(R.first);
476 }
477
478 // Remove the dead frame indices of CSR SGPRs which are spilled to physical
479 // VGPR lanes during SILowerSGPRSpills pass.
480 if (!ResetSGPRSpillStackIDs) {
481 for (auto &R : make_early_inc_range(SGPRSpillsToPhysicalVGPRLanes)) {
482 MFI.RemoveStackObject(R.first);
483 SGPRSpillsToPhysicalVGPRLanes.erase(R.first);
484 }
485 }
486 bool HaveSGPRToMemory = false;
487
488 if (ResetSGPRSpillStackIDs) {
489 // All other SGPRs must be allocated on the default stack, so reset the
490 // stack ID.
491 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E;
492 ++I) {
496 HaveSGPRToMemory = true;
497 }
498 }
499 }
500 }
501
502 for (auto &R : VGPRToAGPRSpills) {
503 if (R.second.IsDead)
504 MFI.RemoveStackObject(R.first);
505 }
506
507 return HaveSGPRToMemory;
508}
509
511 const SIRegisterInfo &TRI) {
512 if (ScavengeFI)
513 return *ScavengeFI;
514 if (isEntryFunction()) {
515 ScavengeFI = MFI.CreateFixedObject(
516 TRI.getSpillSize(AMDGPU::SGPR_32RegClass), 0, false);
517 } else {
518 ScavengeFI = MFI.CreateStackObject(
519 TRI.getSpillSize(AMDGPU::SGPR_32RegClass),
520 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass), false);
521 }
522 return *ScavengeFI;
523}
524
525MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
526 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
527 return AMDGPU::SGPR0 + NumUserSGPRs;
528}
529
530MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
531 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
532}
533
534void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) {
535 VRegFlags.grow(Reg);
536}
537
538void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg,
539 Register SrcReg) {
540 VRegFlags.grow(NewReg);
541 VRegFlags[NewReg] = VRegFlags[SrcReg];
542}
543
546 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
547 if (!ST.isAmdPalOS())
548 return Register();
549 Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
550 if (ST.hasMergedShaders()) {
551 switch (MF.getFunction().getCallingConv()) {
554 // Low GIT address is passed in s8 rather than s0 for an LS+HS or
555 // ES+GS merged shader on gfx9+.
556 GitPtrLo = AMDGPU::SGPR8;
557 return GitPtrLo;
558 default:
559 return GitPtrLo;
560 }
561 }
562 return GitPtrLo;
563}
564
566 const TargetRegisterInfo &TRI) {
568 {
570 OS << printReg(Reg, &TRI);
571 }
572 return Dest;
573}
574
575static std::optional<yaml::SIArgumentInfo>
577 const TargetRegisterInfo &TRI) {
579
580 auto convertArg = [&](std::optional<yaml::SIArgument> &A,
581 const ArgDescriptor &Arg) {
582 if (!Arg)
583 return false;
584
585 // Create a register or stack argument.
587 if (Arg.isRegister()) {
589 OS << printReg(Arg.getRegister(), &TRI);
590 } else
591 SA.StackOffset = Arg.getStackOffset();
592 // Check and update the optional mask.
593 if (Arg.isMasked())
594 SA.Mask = Arg.getMask();
595
596 A = SA;
597 return true;
598 };
599
600 // TODO: Need to serialize kernarg preloads.
601 bool Any = false;
602 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
603 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
604 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
605 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
606 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
607 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
608 Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);
609 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
610 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
611 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
612 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
613 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
614 Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
615 ArgInfo.PrivateSegmentWaveByteOffset);
616 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
617 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
618 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
619 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
620 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
621
622 if (Any)
623 return AI;
624
625 return std::nullopt;
626}
627
630 const llvm::MachineFunction &MF)
631 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
632 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
633 GDSSize(MFI.getGDSSize()),
634 DynLDSAlign(MFI.getDynLDSAlign()), IsEntryFunction(MFI.isEntryFunction()),
635 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
636 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
637 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
638 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
639 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
640 Occupancy(MFI.getOccupancy()),
641 ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
642 FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),
643 StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),
644 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
645 ReturnsVoid(MFI.returnsVoid()),
646 ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)),
647 PSInputAddr(MFI.getPSInputAddr()),
648 PSInputEnable(MFI.getPSInputEnable()),
649 Mode(MFI.getMode()) {
650 for (Register Reg : MFI.getWWMReservedRegs())
651 WWMReservedRegs.push_back(regToString(Reg, TRI));
652
653 if (MFI.getLongBranchReservedReg())
655 if (MFI.getVGPRForAGPRCopy())
657
658 if (MFI.getSGPRForEXECCopy())
660
661 auto SFI = MFI.getOptionalScavengeFI();
662 if (SFI)
664}
665
668}
669
671 const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF,
675 LDSSize = YamlMFI.LDSSize;
676 GDSSize = YamlMFI.GDSSize;
677 DynLDSAlign = YamlMFI.DynLDSAlign;
678 PSInputAddr = YamlMFI.PSInputAddr;
681 Occupancy = YamlMFI.Occupancy;
684 MemoryBound = YamlMFI.MemoryBound;
685 WaveLimiter = YamlMFI.WaveLimiter;
689 ReturnsVoid = YamlMFI.ReturnsVoid;
690
691 if (YamlMFI.ScavengeFI) {
692 auto FIOrErr = YamlMFI.ScavengeFI->getFI(MF.getFrameInfo());
693 if (!FIOrErr) {
694 // Create a diagnostic for a the frame index.
695 const MemoryBuffer &Buffer =
696 *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());
697
698 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1,
699 SourceMgr::DK_Error, toString(FIOrErr.takeError()),
700 "", std::nullopt, std::nullopt);
701 SourceRange = YamlMFI.ScavengeFI->SourceRange;
702 return true;
703 }
704 ScavengeFI = *FIOrErr;
705 } else {
706 ScavengeFI = std::nullopt;
707 }
708 return false;
709}
710
712 for (const BasicBlock &BB : F) {
713 for (const Instruction &I : BB) {
714 const auto *CB = dyn_cast<CallBase>(&I);
715 if (!CB)
716 continue;
717
718 if (CB->isInlineAsm()) {
719 const InlineAsm *IA = dyn_cast<InlineAsm>(CB->getCalledOperand());
720 for (const auto &CI : IA->ParseConstraints()) {
721 for (StringRef Code : CI.Codes) {
722 Code.consume_front("{");
723 if (Code.startswith("a"))
724 return true;
725 }
726 }
727 continue;
728 }
729
730 const Function *Callee =
731 dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());
732 if (!Callee)
733 return true;
734
735 if (Callee->getIntrinsicID() == Intrinsic::not_intrinsic)
736 return true;
737 }
738 }
739
740 return false;
741}
742
744 if (UsesAGPRs)
745 return *UsesAGPRs;
746
747 if (!mayNeedAGPRs()) {
748 UsesAGPRs = false;
749 return false;
750 }
751
753 MF.getFrameInfo().hasCalls()) {
754 UsesAGPRs = true;
755 return true;
756 }
757
758 const MachineRegisterInfo &MRI = MF.getRegInfo();
759
760 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
762 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
763 if (RC && SIRegisterInfo::isAGPRClass(RC)) {
764 UsesAGPRs = true;
765 return true;
766 } else if (!RC && !MRI.use_empty(Reg) && MRI.getType(Reg).isValid()) {
767 // Defer caching UsesAGPRs, function might not yet been regbank selected.
768 return true;
769 }
770 }
771
772 for (MCRegister Reg : AMDGPU::AGPR_32RegClass) {
773 if (MRI.isPhysRegUsed(Reg)) {
774 UsesAGPRs = true;
775 return true;
776 }
777 }
778
779 UsesAGPRs = false;
780 return false;
781}
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock & MBB
Provides AMDGPU specific target descriptions.
Base class for AMDGPU specific classes of TargetSubtarget.
The AMDGPU TargetMachine interface definition for hw codegen targets.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
uint64_t Size
IO & YamlIO
Definition: ELFYAML.cpp:1271
AMD GCN specific subclass of TargetSubtarget.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const GCNTargetMachine & getTM(const GCNSubtarget *STI)
static std::optional< yaml::SIArgumentInfo > convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo, const TargetRegisterInfo &TRI)
static yaml::StringValue regToString(Register Reg, const TargetRegisterInfo &TRI)
Interface definition for SIRegisterInfo.
raw_pwrite_stream & OS
static const AMDGPUFunctionArgInfo FixedABIFunctionInfo
Definition: Any.h:28
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
BitVector & set()
Definition: BitVector.h:351
void setBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
setBitsInMask - Add '1' bits from Mask to this vector.
Definition: BitVector.h:707
void push_back(bool Val)
Definition: BitVector.h:466
Allocate memory in an ever growing pool, as if by bump-pointer.
Definition: Allocator.h:66
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:239
const SITargetLowering * getTargetLowering() const override
Definition: GCNSubtarget.h:241
void allocKernargPreloadSGPRs(unsigned NumSGPRs)
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:33
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasCalls() const
Return true if the current function has any function calls.
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
void setStackID(int ObjectIdx, uint8_t ID)
bool isSpillSlotObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a spill slot.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
uint8_t getStackID(int ObjectIdx) const
int getObjectIndexBegin() const
Return the minimum frame object index.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * cloneInfo(const Ty &Old)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
size_type count(const KeyT &Key) const
Definition: MapVector.h:144
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: MapVector.h:117
This interface provides simple read-only access to a block of memory, and provides simple methods for...
Definition: MemoryBuffer.h:51
virtual StringRef getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
Definition: MemoryBuffer.h:76
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
Definition: Register.h:84
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool usesAGPRs(const MachineFunction &MF) const
bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange)
void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size=4, Align Alignment=Align(4))
Register addDispatchPtr(const SIRegisterInfo &TRI)
Register addFlatScratchInit(const SIRegisterInfo &TRI)
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI)
Register addQueuePtr(const SIRegisterInfo &TRI)
bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, bool IsPrologEpilog=false)
SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI)=default
Register getGITPtrLoReg(const MachineFunction &MF) const
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR)
Reserve AGPRs or VGPRs to support spilling for FrameIndex FI.
void splitWWMSpillRegisters(MachineFunction &MF, SmallVectorImpl< std::pair< Register, int > > &CalleeSavedRegs, SmallVectorImpl< std::pair< Register, int > > &ScratchRegs) const
bool mayUseAGPRs(const Function &F) const
bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const
Register addKernargSegmentPtr(const SIRegisterInfo &TRI)
Register addDispatchID(const SIRegisterInfo &TRI)
bool removeDeadFrameIndices(MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs)
If ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill to the default stack.
MachineFunctionInfo * clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF, const DenseMap< MachineBasicBlock *, MachineBasicBlock * > &Src2DstMBB) const override
Make a functionally equivalent copy of this MachineFunctionInfo in MF.
bool checkIndexInPrologEpilogSGPRSpills(int FI) const
Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI)
const ReservedRegSet & getWWMReservedRegs() const
std::optional< int > getOptionalScavengeFI() const
Register addImplicitBufferPtr(const SIRegisterInfo &TRI)
void limitOccupancy(const MachineFunction &MF)
SmallVectorImpl< MCRegister > * addPreloadedKernArg(const SIRegisterInfo &TRI, const TargetRegisterClass *RC, unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs)
static bool isAGPRClass(const TargetRegisterClass *RC)
Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...
Definition: SourceMgr.h:281
Represents a location in source code.
Definition: SMLoc.h:23
Represents a range in source code.
Definition: SMLoc.h:48
const value_type & back() const
Return the last element of the SetVector.
Definition: SetVector.h:149
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
typename SuperClass::const_iterator const_iterator
Definition: SmallVector.h:582
unsigned getMainFileID() const
Definition: SourceMgr.h:132
const MemoryBuffer * getMemoryBuffer(unsigned i) const
Definition: SourceMgr.h:125
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool consumeInteger(unsigned Radix, T &Result)
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:503
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
const TargetMachine & getTargetMachine() const
iterator_range< SmallVectorImpl< MCPhysReg >::const_iterator > getRegisters() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
bool isEntryFunctionCC(CallingConv::ID CC)
bool isChainCC(CallingConv::ID CC)
unsigned getInitialPSInputAddr(const Function &F)
bool isGraphics(CallingConv::ID cc)
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
Definition: CallingConv.h:194
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
Definition: CallingConv.h:197
@ AMDGPU_Gfx
Used for AMD graphics targets.
Definition: CallingConv.h:229
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
Definition: CallingConv.h:203
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
Definition: CallingConv.h:188
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
Definition: CallingConv.h:191
@ SPIR_KERNEL
Used for SPIR kernel functions.
Definition: CallingConv.h:141
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:21
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:666
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
Helper struct shared between Function Specialization and SCCP Solver.
Definition: SCCPSolver.h:41
MachineFunctionInfo - This class can be derived from and used by targets to hold private target-speci...
A serializaable representation of a reference to a stack object or fixed stack object.
std::optional< SIArgument > PrivateSegmentWaveByteOffset
std::optional< SIArgument > WorkGroupIDY
std::optional< SIArgument > FlatScratchInit
std::optional< SIArgument > DispatchPtr
std::optional< SIArgument > DispatchID
std::optional< SIArgument > WorkItemIDY
std::optional< SIArgument > WorkGroupIDX
std::optional< SIArgument > ImplicitArgPtr
std::optional< SIArgument > QueuePtr
std::optional< SIArgument > WorkGroupInfo
std::optional< SIArgument > LDSKernelId
std::optional< SIArgument > ImplicitBufferPtr
std::optional< SIArgument > WorkItemIDX
std::optional< SIArgument > KernargSegmentPtr
std::optional< SIArgument > WorkItemIDZ
std::optional< SIArgument > PrivateSegmentSize
std::optional< SIArgument > PrivateSegmentBuffer
std::optional< SIArgument > WorkGroupIDZ
std::optional< unsigned > Mask
static SIArgument createArgument(bool IsReg)
SmallVector< StringValue > WWMReservedRegs
void mappingImpl(yaml::IO &YamlIO) override
std::optional< FrameIndex > ScavengeFI
A wrapper around std::string which contains a source range that's being set during parsing.