41 UserSGPRInfo(
F, *STI), WorkGroupIDX(
false), WorkGroupIDY(
false),
43 PrivateSegmentWaveByteOffset(
false), WorkItemIDX(
false),
45 GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0) {
47 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(
F);
48 WavesPerEU = ST.getWavesPerEU(
F);
53 VRegFlags.reserve(1024);
65 MayNeedAGPRs = ST.hasMAIInsts();
72 StackPtrOffsetReg = AMDGPU::SGPR32;
74 ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51;
79 ImplicitArgPtr =
false;
85 FrameOffsetReg = AMDGPU::SGPR33;
86 StackPtrOffsetReg = AMDGPU::SGPR32;
88 if (!ST.enableFlatScratch()) {
91 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
97 if (!
F.hasFnAttribute(
"amdgpu-no-implicitarg-ptr"))
98 ImplicitArgPtr =
true;
100 ImplicitArgPtr =
false;
104 if (ST.hasGFX90AInsts() &&
105 ST.getMaxNumVGPRs(
F) <= AMDGPU::VGPR_32RegClass.getNumRegs() &&
107 MayNeedAGPRs =
false;
112 if (IsKernel || !
F.hasFnAttribute(
"amdgpu-no-workgroup-id-x"))
115 if (!
F.hasFnAttribute(
"amdgpu-no-workgroup-id-y"))
118 if (!
F.hasFnAttribute(
"amdgpu-no-workgroup-id-z"))
123 if (IsKernel || !
F.hasFnAttribute(
"amdgpu-no-workitem-id-x"))
126 if (!
F.hasFnAttribute(
"amdgpu-no-workitem-id-y") &&
127 ST.getMaxWorkitemID(
F, 1) != 0)
130 if (!
F.hasFnAttribute(
"amdgpu-no-workitem-id-z") &&
131 ST.getMaxWorkitemID(
F, 2) != 0)
134 if (!IsKernel && !
F.hasFnAttribute(
"amdgpu-no-lds-kernel-id"))
144 if (!ST.flatScratchIsArchitected()) {
145 PrivateSegmentWaveByteOffset =
true;
150 ArgInfo.PrivateSegmentWaveByteOffset =
155 Attribute A =
F.getFnAttribute(
"amdgpu-git-ptr-high");
160 A =
F.getFnAttribute(
"amdgpu-32bit-address-high-bits");
161 S =
A.getValueAsString();
168 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
170 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(
F) - 1);
192 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
194 return ArgInfo.PrivateSegmentBuffer.getRegister();
199 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
201 return ArgInfo.DispatchPtr.getRegister();
206 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
208 return ArgInfo.QueuePtr.getRegister();
214 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
216 return ArgInfo.KernargSegmentPtr.getRegister();
221 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
223 return ArgInfo.DispatchID.getRegister();
228 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
230 return ArgInfo.FlatScratchInit.getRegister();
235 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
237 return ArgInfo.ImplicitBufferPtr.getRegister();
243 return ArgInfo.LDSKernelId.getRegister();
248 unsigned AllocSizeDWord,
int KernArgIdx,
int PaddingSGPRs) {
250 "Preload kernel argument allocated twice.");
251 NumUserSGPRs += PaddingSGPRs;
256 TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC);
258 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {
259 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(PreloadReg);
260 NumUserSGPRs += AllocSizeDWord;
262 for (
unsigned I = 0;
I < AllocSizeDWord; ++
I) {
263 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(getNextUserSGPR());
270 return &
ArgInfo.PreloadKernArgs[KernArgIdx].Regs;
279 WWMSpills.
insert(std::make_pair(
289 for (
auto &Reg : WWMSpills) {
291 CalleeSavedRegs.push_back(Reg);
293 ScratchRegs.push_back(Reg);
299 for (
unsigned I = 0; CSRegs[
I]; ++
I) {
300 if (CSRegs[
I] == Reg)
307bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
312 LaneVGPR =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
313 SpillVGPRs.push_back(LaneVGPR);
315 LaneVGPR = SpillVGPRs.back();
318 SGPRSpillsToVirtualVGPRLanes[FI].push_back(
323bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
330 LaneVGPR =
TRI->findUnusedRegister(
MRI, &AMDGPU::VGPR_32RegClass, MF);
331 if (LaneVGPR == AMDGPU::NoRegister) {
334 SGPRSpillsToPhysicalVGPRLanes.erase(FI);
345 LaneVGPR = WWMReservedRegs.
back();
348 SGPRSpillsToPhysicalVGPRLanes[FI].push_back(
355 bool IsPrologEpilog) {
356 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
357 IsPrologEpilog ? SGPRSpillsToPhysicalVGPRLanes[FI]
358 : SGPRSpillsToVirtualVGPRLanes[FI];
361 if (!SpillLanes.empty())
366 unsigned WaveSize = ST.getWavefrontSize();
368 unsigned Size = FrameInfo.getObjectSize(FI);
369 unsigned NumLanes =
Size / 4;
371 if (NumLanes > WaveSize)
374 assert(
Size >= 4 &&
"invalid sgpr spill size");
375 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
376 "not spilling SGPRs to VGPRs");
378 unsigned &NumSpillLanes =
379 IsPrologEpilog ? NumPhysicalVGPRSpillLanes : NumVirtualVGPRSpillLanes;
381 for (
unsigned I = 0;
I < NumLanes; ++
I, ++NumSpillLanes) {
382 unsigned LaneIndex = (NumSpillLanes % WaveSize);
384 bool Allocated = IsPrologEpilog
385 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex)
386 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
408 auto &Spill = VGPRToAGPRSpills[FI];
411 if (!Spill.Lanes.empty())
412 return Spill.FullyAllocated;
415 unsigned NumLanes =
Size / 4;
416 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
419 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
422 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
424 Spill.FullyAllocated =
true;
439 OtherUsedRegs.
set(Reg);
441 OtherUsedRegs.
set(Reg);
444 for (
int I = NumLanes - 1;
I >= 0; --
I) {
445 NextSpillReg = std::find_if(
447 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
451 if (NextSpillReg == Regs.
end()) {
452 Spill.FullyAllocated =
false;
456 OtherUsedRegs.
set(*NextSpillReg);
458 MRI.reserveReg(*NextSpillReg,
TRI);
459 Spill.Lanes[
I] = *NextSpillReg++;
462 return Spill.FullyAllocated;
475 SGPRSpillsToVirtualVGPRLanes.erase(R.first);
480 if (!ResetSGPRSpillStackIDs) {
483 SGPRSpillsToPhysicalVGPRLanes.erase(R.first);
486 bool HaveSGPRToMemory =
false;
488 if (ResetSGPRSpillStackIDs) {
496 HaveSGPRToMemory =
true;
502 for (
auto &R : VGPRToAGPRSpills) {
507 return HaveSGPRToMemory;
516 TRI.getSpillSize(AMDGPU::SGPR_32RegClass), 0,
false);
519 TRI.getSpillSize(AMDGPU::SGPR_32RegClass),
520 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass),
false);
525MCPhysReg SIMachineFunctionInfo::getNextUserSGPR()
const {
526 assert(NumSystemSGPRs == 0 &&
"System SGPRs must be added after user SGPRs");
527 return AMDGPU::SGPR0 + NumUserSGPRs;
530MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR()
const {
531 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
534void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(
Register Reg) {
538void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(
Register NewReg,
540 VRegFlags.grow(NewReg);
541 VRegFlags[NewReg] = VRegFlags[SrcReg];
547 if (!ST.isAmdPalOS())
550 if (ST.hasMergedShaders()) {
556 GitPtrLo = AMDGPU::SGPR8;
575static std::optional<yaml::SIArgumentInfo>
580 auto convertArg = [&](std::optional<yaml::SIArgument> &
A,
587 if (Arg.isRegister()) {
594 SA.
Mask = Arg.getMask();
615 ArgInfo.PrivateSegmentWaveByteOffset);
631 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
632 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
633 GDSSize(MFI.getGDSSize()),
634 DynLDSAlign(MFI.getDynLDSAlign()), IsEntryFunction(MFI.isEntryFunction()),
635 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
636 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
637 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
638 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
639 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
640 Occupancy(MFI.getOccupancy()),
644 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
645 ReturnsVoid(MFI.returnsVoid()),
647 PSInputAddr(MFI.getPSInputAddr()),
648 PSInputEnable(MFI.getPSInputEnable()),
649 Mode(MFI.getMode()) {
700 "", std::nullopt, std::nullopt);
701 SourceRange = YamlMFI.
ScavengeFI->SourceRange;
714 const auto *CB = dyn_cast<CallBase>(&
I);
718 if (CB->isInlineAsm()) {
719 const InlineAsm *IA = dyn_cast<InlineAsm>(CB->getCalledOperand());
720 for (
const auto &CI : IA->ParseConstraints()) {
722 Code.consume_front(
"{");
723 if (Code.startswith(
"a"))
731 dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());
747 if (!mayNeedAGPRs()) {
760 for (
unsigned I = 0,
E =
MRI.getNumVirtRegs();
I !=
E; ++
I) {
766 }
else if (!RC && !
MRI.use_empty(Reg) &&
MRI.getType(Reg).isValid()) {
772 for (
MCRegister Reg : AMDGPU::AGPR_32RegClass) {
773 if (
MRI.isPhysRegUsed(Reg)) {
unsigned const MachineRegisterInfo * MRI
Provides AMDGPU specific target descriptions.
Base class for AMDGPU specific classes of TargetSubtarget.
The AMDGPU TargetMachine interface definition for hw codegen targets.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
AMD GCN specific subclass of TargetSubtarget.
unsigned const TargetRegisterInfo * TRI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const GCNTargetMachine & getTM(const GCNSubtarget *STI)
static std::optional< yaml::SIArgumentInfo > convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo, const TargetRegisterInfo &TRI)
static yaml::StringValue regToString(Register Reg, const TargetRegisterInfo &TRI)
Interface definition for SIRegisterInfo.
static const AMDGPUFunctionArgInfo FixedABIFunctionInfo
uint32_t getLDSSize() const
bool isEntryFunction() const
LLVM Basic Block Representation.
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
void setBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
setBitsInMask - Add '1' bits from Mask to this vector.
Allocate memory in an ever growing pool, as if by bump-pointer.
Lightweight error class with error context and mandatory checking.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
const SITargetLowering * getTargetLowering() const override
void allocKernargPreloadSGPRs(unsigned NumSGPRs)
Wrapper class representing physical registers. Should be passed by value.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasCalls() const
Return true if the current function has any function calls.
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
void setStackID(int ObjectIdx, uint8_t ID)
bool isSpillSlotObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a spill slot.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
uint8_t getStackID(int ObjectIdx) const
int getObjectIndexBegin() const
Return the minimum frame object index.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * cloneInfo(const Ty &Old)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
size_type count(const KeyT &Key) const
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This interface provides simple read-only access to a block of memory, and provides simple methods for...
virtual StringRef getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool usesAGPRs(const MachineFunction &MF) const
bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange)
void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size=4, Align Alignment=Align(4))
Register addDispatchPtr(const SIRegisterInfo &TRI)
Register getLongBranchReservedReg() const
Register addFlatScratchInit(const SIRegisterInfo &TRI)
unsigned getMaxWavesPerEU() const
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI)
Register addQueuePtr(const SIRegisterInfo &TRI)
bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, bool IsPrologEpilog=false)
SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI)=default
Register getGITPtrLoReg(const MachineFunction &MF) const
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR)
Reserve AGPRs or VGPRs to support spilling for FrameIndex FI.
void splitWWMSpillRegisters(MachineFunction &MF, SmallVectorImpl< std::pair< Register, int > > &CalleeSavedRegs, SmallVectorImpl< std::pair< Register, int > > &ScratchRegs) const
Register getSGPRForEXECCopy() const
bool mayUseAGPRs(const Function &F) const
bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const
Register addLDSKernelId()
Register getVGPRForAGPRCopy() const
Register addKernargSegmentPtr(const SIRegisterInfo &TRI)
Register addDispatchID(const SIRegisterInfo &TRI)
bool removeDeadFrameIndices(MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs)
If ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill to the default stack.
MachineFunctionInfo * clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF, const DenseMap< MachineBasicBlock *, MachineBasicBlock * > &Src2DstMBB) const override
Make a functionally equivalent copy of this MachineFunctionInfo in MF.
bool checkIndexInPrologEpilogSGPRSpills(int FI) const
Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI)
const ReservedRegSet & getWWMReservedRegs() const
std::optional< int > getOptionalScavengeFI() const
Register addImplicitBufferPtr(const SIRegisterInfo &TRI)
void limitOccupancy(const MachineFunction &MF)
SmallVectorImpl< MCRegister > * addPreloadedKernArg(const SIRegisterInfo &TRI, const TargetRegisterClass *RC, unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs)
void reserveWWMRegister(Register Reg)
static bool isAGPRClass(const TargetRegisterClass *RC)
Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...
Represents a location in source code.
Represents a range in source code.
const value_type & back() const
Return the last element of the SetVector.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
typename SuperClass::const_iterator const_iterator
unsigned getMainFileID() const
const MemoryBuffer * getMemoryBuffer(unsigned i) const
StringRef - Represent a constant reference to a string, i.e.
bool consumeInteger(unsigned Radix, T &Result)
Parse the current string as an integer of the specified radix.
constexpr bool empty() const
empty - Check if the string is empty.
const TargetMachine & getTargetMachine() const
iterator_range< SmallVectorImpl< MCPhysReg >::const_iterator > getRegisters() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A raw_ostream that writes to an std::string.
bool isEntryFunctionCC(CallingConv::ID CC)
bool isChainCC(CallingConv::ID CC)
unsigned getInitialPSInputAddr(const Function &F)
bool isGraphics(CallingConv::ID cc)
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ SPIR_KERNEL
Used for SPIR kernel functions.
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
This is an optimization pass for GlobalISel generic memory operations.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
Helper struct shared between Function Specialization and SCCP Solver.
MachineFunctionInfo - This class can be derived from and used by targets to hold private target-speci...
A serializaable representation of a reference to a stack object or fixed stack object.
std::optional< SIArgument > PrivateSegmentWaveByteOffset
std::optional< SIArgument > WorkGroupIDY
std::optional< SIArgument > FlatScratchInit
std::optional< SIArgument > DispatchPtr
std::optional< SIArgument > DispatchID
std::optional< SIArgument > WorkItemIDY
std::optional< SIArgument > WorkGroupIDX
std::optional< SIArgument > ImplicitArgPtr
std::optional< SIArgument > QueuePtr
std::optional< SIArgument > WorkGroupInfo
std::optional< SIArgument > LDSKernelId
std::optional< SIArgument > ImplicitBufferPtr
std::optional< SIArgument > WorkItemIDX
std::optional< SIArgument > KernargSegmentPtr
std::optional< SIArgument > WorkItemIDZ
std::optional< SIArgument > PrivateSegmentSize
std::optional< SIArgument > PrivateSegmentBuffer
std::optional< SIArgument > WorkGroupIDZ
std::optional< unsigned > Mask
static SIArgument createArgument(bool IsReg)
StringValue SGPRForEXECCopy
SmallVector< StringValue > WWMReservedRegs
uint32_t HighBitsOf32BitAddress
SIMachineFunctionInfo()=default
StringValue LongBranchReservedReg
uint64_t ExplicitKernArgSize
void mappingImpl(yaml::IO &YamlIO) override
StringValue VGPRForAGPRCopy
std::optional< FrameIndex > ScavengeFI
unsigned BytesInStackArgArea
A wrapper around std::string which contains a source range that's being set during parsing.