26#include "llvm/IR/IntrinsicsAMDGPU.h"
27#include "llvm/IR/IntrinsicsR600.h"
33#define DEBUG_TYPE "amdgpu-subtarget"
49 const unsigned WavesPerWorkgroup =
50 std::max(1u, (WorkGroupSize + WaveSize - 1) / WaveSize);
52 const unsigned WorkGroupsPerCU =
53 std::max(1u, (NWaves *
getEUsPerCU()) / WavesPerWorkgroup);
58std::pair<unsigned, unsigned>
73 auto PropsFromWGSize = [=](
unsigned WGSize)
74 -> std::tuple<const unsigned, const unsigned, unsigned> {
75 unsigned WavesPerWG =
divideCeil(WGSize, WaveSize);
77 return {WavesPerWG, WGsPerCU, WavesPerWG * WGsPerCU};
85 auto [MinWavesPerWG, MaxWGsPerCU, MaxWavesPerCU] = PropsFromWGSize(MinWGSize);
86 auto [MaxWavesPerWG, MinWGsPerCU, MinWavesPerCU] = PropsFromWGSize(MaxWGSize);
91 if (MinWavesPerCU >= MaxWavesPerCU) {
94 const unsigned WaveSlotsPerCU = WavesPerEU *
getEUsPerCU();
99 unsigned MinWavesPerCUForWGSize =
100 divideCeil(WaveSlotsPerCU, MinWGsPerCU + 1) * MinWGsPerCU;
101 if (MinWavesPerCU > MinWavesPerCUForWGSize) {
102 unsigned ExcessSlots = MinWavesPerCU - MinWavesPerCUForWGSize;
103 if (
unsigned ExcessSlotsPerWG = ExcessSlots / MinWGsPerCU) {
110 MinWavesPerCU -= MinWGsPerCU * std::min(ExcessSlotsPerWG,
111 MaxWavesPerWG - MinWavesPerWG);
118 unsigned LeftoverSlots = WaveSlotsPerCU - MaxWGsPerCU * MinWavesPerWG;
119 if (
unsigned LeftoverSlotsPerWG = LeftoverSlots / MaxWGsPerCU) {
126 MaxWavesPerCU += MaxWGsPerCU * std::min(LeftoverSlotsPerWG,
127 ((MaxWGSize - 1) / WaveSize) + 1 -
134 return {std::clamp(MinWavesPerCU /
getEUsPerCU(), 1U, WavesPerEU),
144std::pair<unsigned, unsigned>
162 std::pair<unsigned, unsigned>
Default =
167 F,
"amdgpu-flat-work-group-size",
Default);
170 if (Requested.first > Requested.second)
183 std::pair<unsigned, unsigned> Requested,
184 std::pair<unsigned, unsigned> FlatWorkGroupSizes)
const {
192 unsigned MinImpliedByFlatWorkGroupSize =
194 Default.first = MinImpliedByFlatWorkGroupSize;
197 if (Requested.second && Requested.first > Requested.second)
207 if (Requested.first < MinImpliedByFlatWorkGroupSize)
214 const Function &
F, std::pair<unsigned, unsigned> FlatWorkGroupSizes)
const {
219 std::pair<unsigned, unsigned> Requested =
226 if (
Node &&
Node->getNumOperands() == 3)
227 return mdconst::extract<ConstantInt>(
Node->getOperand(Dim))->getZExtValue();
228 return std::numeric_limits<unsigned>::max();
236 unsigned Dimension)
const {
238 if (ReqdSize != std::numeric_limits<unsigned>::max())
244 for (
int I = 0;
I < 3; ++
I) {
253 Function *Kernel =
I->getParent()->getParent();
254 unsigned MinSize = 0;
256 bool IdQuery =
false;
259 if (
auto *CI = dyn_cast<CallInst>(
I)) {
260 const Function *
F = CI->getCalledFunction();
262 unsigned Dim = UINT_MAX;
263 switch (
F->getIntrinsicID()) {
264 case Intrinsic::amdgcn_workitem_id_x:
265 case Intrinsic::r600_read_tidig_x:
268 case Intrinsic::r600_read_local_size_x:
271 case Intrinsic::amdgcn_workitem_id_y:
272 case Intrinsic::r600_read_tidig_y:
275 case Intrinsic::r600_read_local_size_y:
278 case Intrinsic::amdgcn_workitem_id_z:
279 case Intrinsic::r600_read_tidig_z:
282 case Intrinsic::r600_read_local_size_z:
291 if (ReqdSize != std::numeric_limits<unsigned>::max())
292 MinSize = MaxSize = ReqdSize;
309 if (
auto *CI = dyn_cast<CallBase>(
I)) {
311 CI->addRangeRetAttr(
Range);
315 I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
325 if (
F.hasFnAttribute(
"amdgpu-no-implicitarg-ptr"))
332 const Module *M =
F.getParent();
335 return F.getFnAttributeAsParsedInteger(
"amdgpu-implicitarg-num-bytes",
340 Align &MaxAlign)
const {
349 if (Arg.hasAttribute(
"amdgpu-hidden-argument"))
352 const bool IsByRef = Arg.hasByRefAttr();
353 Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
354 Align Alignment =
DL.getValueOrABITypeAlignment(
355 IsByRef ? Arg.getParamAlign() : std::nullopt, ArgTy);
356 uint64_t AllocSize =
DL.getTypeAllocSize(ArgTy);
357 ExplicitArgBytes =
alignTo(ExplicitArgBytes, Alignment) + AllocSize;
358 MaxAlign = std::max(MaxAlign, Alignment);
361 return ExplicitArgBytes;
365 Align &MaxAlign)
const {
374 uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes;
376 if (ImplicitBytes != 0) {
378 TotalSize =
alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes;
379 MaxAlign = std::max(MaxAlign, Alignment);
408 std::numeric_limits<uint32_t>::max());
This file describes how to lower LLVM calls to machine code calls.
This file declares the targeting of the InstructionSelector class for AMDGPU.
This file declares the targeting of the Machinelegalizer class for AMDGPU.
This file declares the targeting of the RegisterBankInfo class for AMDGPU.
static unsigned getReqdWorkGroupSize(const Function &Kernel, unsigned Dim)
Base class for AMDGPU specific classes of TargetSubtarget.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file describes how to lower LLVM inline asm to machine code INLINEASM.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
AMDGPU R600 specific subclass of TargetSubtarget.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::pair< unsigned, unsigned > getOccupancyWithWorkGroupSizes(uint32_t LDSBytes, const Function &F) const
Subtarget's minimum/maximum occupancy, in number of waves per EU, that can be achieved when the only ...
std::pair< unsigned, unsigned > getDefaultFlatWorkGroupSize(CallingConv::ID CC) const
bool EnableRealTrue16Insts
Align getAlignmentForImplicitArgPtr() const
unsigned getEUsPerCU() const
Number of SIMDs/EUs (execution units) per "CU" ("compute unit"), where the "CU" is the unit onto whic...
bool isMesaKernel(const Function &F) const
std::pair< unsigned, unsigned > getWavesPerEU(const Function &F) const
bool useRealTrue16Insts() const
Return true if real (non-fake) variants of True16 instructions using 16-bit registers should be code-...
virtual unsigned getMinWavesPerEU() const =0
std::pair< unsigned, unsigned > getFlatWorkGroupSizes(const Function &F) const
bool makeLIDRangeMetadata(Instruction *I) const
Creates value range metadata on an workitemid.* intrinsic call or load.
unsigned getMaxWorkitemID(const Function &Kernel, unsigned Dimension) const
Return the maximum workitem ID value in the function, for the given (0, 1, 2) dimension.
unsigned getImplicitArgNumBytes(const Function &F) const
unsigned getLocalMemorySize() const
Return the maximum number of bytes of LDS available for all workgroups running on the same WGP or CU.
SmallVector< unsigned > getMaxNumWorkGroups(const Function &F) const
Return the number of work groups for the function.
virtual unsigned getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const =0
virtual unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const =0
unsigned getKernArgSegmentSize(const Function &F, Align &MaxAlign) const
bool hasTrue16BitInsts() const
Return true if the subtarget supports True16 instructions.
AMDGPUDwarfFlavour getAMDGPUDwarfFlavour() const
unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount, const Function &) const
Return the amount of LDS that can be used that will not restrict the occupancy lower than WaveCount.
virtual unsigned getMaxFlatWorkGroupSize() const =0
AMDGPUSubtarget(Triple TT)
unsigned getExplicitKernelArgOffset() const
Returns the offset in bytes from the start of the input buffer of the first explicit kernel argument.
unsigned getMaxWavesPerEU() const
uint64_t getExplicitKernArgSize(const Function &F, Align &MaxAlign) const
std::pair< unsigned, unsigned > getEffectiveWavesPerEU(std::pair< unsigned, unsigned > WavesPerEU, std::pair< unsigned, unsigned > FlatWorkGroupSizes) const
bool isSingleLaneExecution(const Function &Kernel) const
Return true if only a single workitem can be active in a wave.
static const AMDGPUSubtarget & get(const MachineFunction &MF)
unsigned getWavefrontSize() const
virtual unsigned getMinFlatWorkGroupSize() const =0
Class for arbitrary precision integers.
This class represents an incoming formal argument to a Function.
This class represents a range of values.
A parsed version of the target data layout string in and methods for querying it.
MDNode * getMetadata(unsigned KindID) const
Get the current metadata attachments for the given kind, if any.
MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
A Module instance is used to store all the information related to an LLVM module.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Primary interface to the complete machine description for the target machine.
const Triple & getTargetTriple() const
Triple - Helper class for working with autoconf configuration names.
ArchType getArch() const
Get the parsed architecture type of this triple.
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_READNONE bool isKernel(CallingConv::ID CC)
unsigned getAMDHSACodeObjectVersion(const Module &M)
bool isShader(CallingConv::ID cc)
SmallVector< unsigned > getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size, unsigned DefaultVal)
std::pair< unsigned, unsigned > getIntegerPairAttribute(const Function &F, StringRef Name, std::pair< unsigned, unsigned > Default, bool OnlyFirstRequired)
@ AMDGPU_VS
Used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (vertex shader if tess...
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ AMDGPU_ES
Used for AMDPAL shader stage before geometry shader if geometry is in use.
@ AMDGPU_LS
Used for AMDPAL vertex shader if tessellation is in use.
This is an optimization pass for GlobalISel generic memory operations.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
@ Default
The result values are uniform if and only if all operands are uniform.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.