41 UserSGPRInfo(
F, *STI), WorkGroupIDX(
false), WorkGroupIDY(
false),
43 PrivateSegmentWaveByteOffset(
false), WorkItemIDX(
false),
45 GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0) {
47 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(
F);
48 WavesPerEU = ST.getWavesPerEU(
F);
49 MaxNumWorkGroups = ST.getMaxNumWorkGroups(
F);
55 VRegFlags.reserve(1024);
67 MayNeedAGPRs = ST.hasMAIInsts();
74 StackPtrOffsetReg = AMDGPU::SGPR32;
76 ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51;
81 ImplicitArgPtr =
false;
87 FrameOffsetReg = AMDGPU::SGPR33;
88 StackPtrOffsetReg = AMDGPU::SGPR32;
90 if (!ST.enableFlatScratch()) {
93 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
99 if (!
F.hasFnAttribute(
"amdgpu-no-implicitarg-ptr"))
100 ImplicitArgPtr =
true;
102 ImplicitArgPtr =
false;
106 if (ST.hasGFX90AInsts() &&
107 ST.getMaxNumVGPRs(
F) <= AMDGPU::VGPR_32RegClass.getNumRegs() &&
109 MayNeedAGPRs =
false;
114 if (IsKernel || !
F.hasFnAttribute(
"amdgpu-no-workgroup-id-x"))
117 if (!
F.hasFnAttribute(
"amdgpu-no-workgroup-id-y"))
120 if (!
F.hasFnAttribute(
"amdgpu-no-workgroup-id-z"))
125 if (IsKernel || !
F.hasFnAttribute(
"amdgpu-no-workitem-id-x"))
128 if (!
F.hasFnAttribute(
"amdgpu-no-workitem-id-y") &&
129 ST.getMaxWorkitemID(
F, 1) != 0)
132 if (!
F.hasFnAttribute(
"amdgpu-no-workitem-id-z") &&
133 ST.getMaxWorkitemID(
F, 2) != 0)
136 if (!IsKernel && !
F.hasFnAttribute(
"amdgpu-no-lds-kernel-id"))
146 if (!ST.flatScratchIsArchitected()) {
147 PrivateSegmentWaveByteOffset =
true;
152 ArgInfo.PrivateSegmentWaveByteOffset =
157 Attribute A =
F.getFnAttribute(
"amdgpu-git-ptr-high");
162 A =
F.getFnAttribute(
"amdgpu-32bit-address-high-bits");
163 S =
A.getValueAsString();
170 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
172 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(
F) - 1);
194 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));
196 return ArgInfo.PrivateSegmentBuffer.getRegister();
201 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
203 return ArgInfo.DispatchPtr.getRegister();
208 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
210 return ArgInfo.QueuePtr.getRegister();
216 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
218 return ArgInfo.KernargSegmentPtr.getRegister();
223 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
225 return ArgInfo.DispatchID.getRegister();
230 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
232 return ArgInfo.FlatScratchInit.getRegister();
237 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));
239 return ArgInfo.ImplicitBufferPtr.getRegister();
245 return ArgInfo.LDSKernelId.getRegister();
250 unsigned AllocSizeDWord,
int KernArgIdx,
int PaddingSGPRs) {
252 "Preload kernel argument allocated twice.");
253 NumUserSGPRs += PaddingSGPRs;
258 TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC);
260 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {
261 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(PreloadReg);
262 NumUserSGPRs += AllocSizeDWord;
264 for (
unsigned I = 0;
I < AllocSizeDWord; ++
I) {
265 ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(getNextUserSGPR());
272 return &
ArgInfo.PreloadKernArgs[KernArgIdx].Regs;
289 WWMSpills.
insert(std::make_pair(
299 for (
auto &Reg : WWMSpills) {
301 CalleeSavedRegs.push_back(Reg);
303 ScratchRegs.push_back(Reg);
309 for (
unsigned I = 0; CSRegs[
I]; ++
I) {
310 if (CSRegs[
I] == Reg)
321 for (
unsigned I = 0, E = SpillPhysVGPRs.size();
I < E; ++
I) {
324 TRI->findUnusedRegister(
MRI, &AMDGPU::VGPR_32RegClass, MF);
325 if (!NewReg || NewReg >= Reg)
328 MRI.replaceRegWith(Reg, NewReg);
331 SpillPhysVGPRs[
I] = NewReg;
332 WWMReservedRegs.
remove(Reg);
333 WWMReservedRegs.
insert(NewReg);
334 WWMSpills.
insert(std::make_pair(NewReg, WWMSpills[Reg]));
335 WWMSpills.
erase(Reg);
344bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
349 LaneVGPR =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
350 SpillVGPRs.push_back(LaneVGPR);
352 LaneVGPR = SpillVGPRs.back();
355 SGPRSpillsToVirtualVGPRLanes[FI].push_back(
360bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
361 MachineFunction &MF,
int FI,
unsigned LaneIndex,
bool IsPrologEpilog) {
370 LaneVGPR =
TRI->findUnusedRegister(
MRI, &AMDGPU::VGPR_32RegClass, MF,
372 if (LaneVGPR == AMDGPU::NoRegister) {
375 SGPRSpillsToPhysicalVGPRLanes.erase(FI);
385 SpillPhysVGPRs.push_back(LaneVGPR);
387 LaneVGPR = SpillPhysVGPRs.back();
390 SGPRSpillsToPhysicalVGPRLanes[FI].push_back(
397 bool IsPrologEpilog) {
398 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
399 SpillToPhysVGPRLane ? SGPRSpillsToPhysicalVGPRLanes[FI]
400 : SGPRSpillsToVirtualVGPRLanes[FI];
403 if (!SpillLanes.empty())
408 unsigned WaveSize = ST.getWavefrontSize();
410 unsigned Size = FrameInfo.getObjectSize(FI);
411 unsigned NumLanes =
Size / 4;
413 if (NumLanes > WaveSize)
416 assert(
Size >= 4 &&
"invalid sgpr spill size");
417 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
418 "not spilling SGPRs to VGPRs");
420 unsigned &NumSpillLanes = SpillToPhysVGPRLane ? NumPhysicalVGPRSpillLanes
421 : NumVirtualVGPRSpillLanes;
423 for (
unsigned I = 0;
I < NumLanes; ++
I, ++NumSpillLanes) {
424 unsigned LaneIndex = (NumSpillLanes % WaveSize);
426 bool Allocated = SpillToPhysVGPRLane
427 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex,
429 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
451 auto &Spill = VGPRToAGPRSpills[FI];
454 if (!Spill.Lanes.empty())
455 return Spill.FullyAllocated;
458 unsigned NumLanes =
Size / 4;
459 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);
462 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
465 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
467 Spill.FullyAllocated =
true;
482 OtherUsedRegs.
set(Reg);
484 OtherUsedRegs.
set(Reg);
487 for (
int I = NumLanes - 1;
I >= 0; --
I) {
488 NextSpillReg = std::find_if(
490 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&
494 if (NextSpillReg == Regs.
end()) {
495 Spill.FullyAllocated =
false;
499 OtherUsedRegs.
set(*NextSpillReg);
501 MRI.reserveReg(*NextSpillReg,
TRI);
502 Spill.Lanes[
I] = *NextSpillReg++;
505 return Spill.FullyAllocated;
518 SGPRSpillsToVirtualVGPRLanes.erase(R.first);
523 if (!ResetSGPRSpillStackIDs) {
526 SGPRSpillsToPhysicalVGPRLanes.erase(R.first);
529 bool HaveSGPRToMemory =
false;
531 if (ResetSGPRSpillStackIDs) {
539 HaveSGPRToMemory =
true;
545 for (
auto &R : VGPRToAGPRSpills) {
550 return HaveSGPRToMemory;
560 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass),
false);
564MCPhysReg SIMachineFunctionInfo::getNextUserSGPR()
const {
565 assert(NumSystemSGPRs == 0 &&
"System SGPRs must be added after user SGPRs");
566 return AMDGPU::SGPR0 + NumUserSGPRs;
569MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR()
const {
570 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
573void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(
Register Reg) {
577void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(
Register NewReg,
579 VRegFlags.grow(NewReg);
580 VRegFlags[NewReg] = VRegFlags[SrcReg];
586 if (!ST.isAmdPalOS())
589 if (ST.hasMergedShaders()) {
595 GitPtrLo = AMDGPU::SGPR8;
614static std::optional<yaml::SIArgumentInfo>
619 auto convertArg = [&](std::optional<yaml::SIArgument> &
A,
626 if (Arg.isRegister()) {
633 SA.
Mask = Arg.getMask();
654 ArgInfo.PrivateSegmentWaveByteOffset);
670 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
671 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
672 GDSSize(MFI.getGDSSize()),
673 DynLDSAlign(MFI.getDynLDSAlign()), IsEntryFunction(MFI.isEntryFunction()),
674 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),
675 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
676 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
677 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
678 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
679 Occupancy(MFI.getOccupancy()),
683 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
684 ReturnsVoid(MFI.returnsVoid()),
686 PSInputAddr(MFI.getPSInputAddr()),
687 PSInputEnable(MFI.getPSInputEnable()),
688 Mode(MFI.getMode()) {
739 "", std::nullopt, std::nullopt);
740 SourceRange = YamlMFI.
ScavengeFI->SourceRange;
751 return !
F.hasFnAttribute(
"amdgpu-no-agpr");
758 if (!mayNeedAGPRs()) {
771 for (
unsigned I = 0, E =
MRI.getNumVirtRegs();
I != E; ++
I) {
777 }
else if (!RC && !
MRI.use_empty(Reg) &&
MRI.getType(Reg).isValid()) {
783 for (
MCRegister Reg : AMDGPU::AGPR_32RegClass) {
784 if (
MRI.isPhysRegUsed(Reg)) {
unsigned const MachineRegisterInfo * MRI
Provides AMDGPU specific target descriptions.
Base class for AMDGPU specific classes of TargetSubtarget.
The AMDGPU TargetMachine interface definition for hw codegen targets.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
AMD GCN specific subclass of TargetSubtarget.
unsigned const TargetRegisterInfo * TRI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const GCNTargetMachine & getTM(const GCNSubtarget *STI)
static std::optional< yaml::SIArgumentInfo > convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo, const TargetRegisterInfo &TRI)
static yaml::StringValue regToString(Register Reg, const TargetRegisterInfo &TRI)
Interface definition for SIRegisterInfo.
static const AMDGPUFunctionArgInfo FixedABIFunctionInfo
uint32_t getLDSSize() const
bool isChainFunction() const
bool isEntryFunction() const
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
void setBitsInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
setBitsInMask - Add '1' bits from Mask to this vector.
Allocate memory in an ever growing pool, as if by bump-pointer.
Lightweight error class with error context and mandatory checking.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
const SITargetLowering * getTargetLowering() const override
void allocKernargPreloadSGPRs(unsigned NumSGPRs)
Wrapper class representing physical registers. Should be passed by value.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
void removeLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Remove the specified register from the live in set.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasCalls() const
Return true if the current function has any function calls.
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
void setStackID(int ObjectIdx, uint8_t ID)
bool isSpillSlotObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a spill slot.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
uint8_t getStackID(int ObjectIdx) const
int getObjectIndexBegin() const
Return the minimum frame object index.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * cloneInfo(const Ty &Old)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
size_type count(const KeyT &Key) const
VectorType::iterator erase(typename VectorType::iterator Iterator)
Remove the element given by Iterator.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This interface provides simple read-only access to a block of memory, and provides simple methods for...
virtual StringRef getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool usesAGPRs(const MachineFunction &MF) const
bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange)
void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size=4, Align Alignment=Align(4))
Register addDispatchPtr(const SIRegisterInfo &TRI)
Register getLongBranchReservedReg() const
Register addFlatScratchInit(const SIRegisterInfo &TRI)
unsigned getMaxWavesPerEU() const
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI)
Register addQueuePtr(const SIRegisterInfo &TRI)
SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI)=default
Register getGITPtrLoReg(const MachineFunction &MF) const
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR)
Reserve AGPRs or VGPRs to support spilling for FrameIndex FI.
void splitWWMSpillRegisters(MachineFunction &MF, SmallVectorImpl< std::pair< Register, int > > &CalleeSavedRegs, SmallVectorImpl< std::pair< Register, int > > &ScratchRegs) const
Register getSGPRForEXECCopy() const
bool mayUseAGPRs(const Function &F) const
bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const
void shiftSpillPhysVGPRsToLowestRange(MachineFunction &MF)
Register addLDSKernelId()
Register getVGPRForAGPRCopy() const
bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI, bool SpillToPhysVGPRLane=false, bool IsPrologEpilog=false)
Register addKernargSegmentPtr(const SIRegisterInfo &TRI)
Register addDispatchID(const SIRegisterInfo &TRI)
bool removeDeadFrameIndices(MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs)
If ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill to the default stack.
MachineFunctionInfo * clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF, const DenseMap< MachineBasicBlock *, MachineBasicBlock * > &Src2DstMBB) const override
Make a functionally equivalent copy of this MachineFunctionInfo in MF.
bool checkIndexInPrologEpilogSGPRSpills(int FI) const
Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI)
const ReservedRegSet & getWWMReservedRegs() const
std::optional< int > getOptionalScavengeFI() const
Register addImplicitBufferPtr(const SIRegisterInfo &TRI)
void limitOccupancy(const MachineFunction &MF)
SmallVectorImpl< MCRegister > * addPreloadedKernArg(const SIRegisterInfo &TRI, const TargetRegisterClass *RC, unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs)
void reserveWWMRegister(Register Reg)
static bool isChainScratchRegister(Register VGPR)
static bool isAGPRClass(const TargetRegisterClass *RC)
Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...
Represents a location in source code.
Represents a range in source code.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
typename SuperClass::const_iterator const_iterator
unsigned getMainFileID() const
const MemoryBuffer * getMemoryBuffer(unsigned i) const
StringRef - Represent a constant reference to a string, i.e.
bool consumeInteger(unsigned Radix, T &Result)
Parse the current string as an integer of the specified radix.
constexpr bool empty() const
empty - Check if the string is empty.
const TargetMachine & getTargetMachine() const
ArrayRef< MCPhysReg > getRegisters() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A raw_ostream that writes to an std::string.
bool isEntryFunctionCC(CallingConv::ID CC)
bool isChainCC(CallingConv::ID CC)
unsigned getInitialPSInputAddr(const Function &F)
bool isGraphics(CallingConv::ID cc)
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ SPIR_KERNEL
Used for SPIR kernel functions.
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
This is an optimization pass for GlobalISel generic memory operations.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
Helper struct shared between Function Specialization and SCCP Solver.
MachineFunctionInfo - This class can be derived from and used by targets to hold private target-speci...
A serializaable representation of a reference to a stack object or fixed stack object.
std::optional< SIArgument > PrivateSegmentWaveByteOffset
std::optional< SIArgument > WorkGroupIDY
std::optional< SIArgument > FlatScratchInit
std::optional< SIArgument > DispatchPtr
std::optional< SIArgument > DispatchID
std::optional< SIArgument > WorkItemIDY
std::optional< SIArgument > WorkGroupIDX
std::optional< SIArgument > ImplicitArgPtr
std::optional< SIArgument > QueuePtr
std::optional< SIArgument > WorkGroupInfo
std::optional< SIArgument > LDSKernelId
std::optional< SIArgument > ImplicitBufferPtr
std::optional< SIArgument > WorkItemIDX
std::optional< SIArgument > KernargSegmentPtr
std::optional< SIArgument > WorkItemIDZ
std::optional< SIArgument > PrivateSegmentSize
std::optional< SIArgument > PrivateSegmentBuffer
std::optional< SIArgument > WorkGroupIDZ
std::optional< unsigned > Mask
static SIArgument createArgument(bool IsReg)
StringValue SGPRForEXECCopy
SmallVector< StringValue > WWMReservedRegs
uint32_t HighBitsOf32BitAddress
SIMachineFunctionInfo()=default
StringValue LongBranchReservedReg
uint64_t ExplicitKernArgSize
void mappingImpl(yaml::IO &YamlIO) override
StringValue VGPRForAGPRCopy
std::optional< FrameIndex > ScavengeFI
unsigned BytesInStackArgArea
A wrapper around std::string which contains a source range that's being set during parsing.