LLVM 20.0.0git
AMDGPULowerKernelAttributes.cpp
Go to the documentation of this file.
1//===-- AMDGPULowerKernelAttributes.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass does attempts to make use of reqd_work_group_size metadata
10/// to eliminate loads from the dispatch packet and to constant fold OpenCL
11/// get_local_size-like functions.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPU.h"
19#include "llvm/CodeGen/Passes.h"
21#include "llvm/IR/Constants.h"
22#include "llvm/IR/Function.h"
25#include "llvm/IR/IntrinsicsAMDGPU.h"
27#include "llvm/Pass.h"
28
29#define DEBUG_TYPE "amdgpu-lower-kernel-attributes"
30
31using namespace llvm;
32
33namespace {
34
35// Field offsets in hsa_kernel_dispatch_packet_t.
36enum DispatchPackedOffsets {
37 WORKGROUP_SIZE_X = 4,
38 WORKGROUP_SIZE_Y = 6,
39 WORKGROUP_SIZE_Z = 8,
40
41 GRID_SIZE_X = 12,
42 GRID_SIZE_Y = 16,
43 GRID_SIZE_Z = 20
44};
45
46// Field offsets to implicit kernel argument pointer.
47enum ImplicitArgOffsets {
48 HIDDEN_BLOCK_COUNT_X = 0,
49 HIDDEN_BLOCK_COUNT_Y = 4,
50 HIDDEN_BLOCK_COUNT_Z = 8,
51
52 HIDDEN_GROUP_SIZE_X = 12,
53 HIDDEN_GROUP_SIZE_Y = 14,
54 HIDDEN_GROUP_SIZE_Z = 16,
55
56 HIDDEN_REMAINDER_X = 18,
57 HIDDEN_REMAINDER_Y = 20,
58 HIDDEN_REMAINDER_Z = 22,
59};
60
61class AMDGPULowerKernelAttributes : public ModulePass {
62public:
63 static char ID;
64
65 AMDGPULowerKernelAttributes() : ModulePass(ID) {}
66
67 bool runOnModule(Module &M) override;
68
69 StringRef getPassName() const override {
70 return "AMDGPU Kernel Attributes";
71 }
72
73 void getAnalysisUsage(AnalysisUsage &AU) const override {
74 AU.setPreservesAll();
75 }
76};
77
78Function *getBasePtrIntrinsic(Module &M, bool IsV5OrAbove) {
79 auto IntrinsicId = IsV5OrAbove ? Intrinsic::amdgcn_implicitarg_ptr
80 : Intrinsic::amdgcn_dispatch_ptr;
81 StringRef Name = Intrinsic::getName(IntrinsicId);
82 return M.getFunction(Name);
83}
84
85} // end anonymous namespace
86
87static bool processUse(CallInst *CI, bool IsV5OrAbove) {
88 Function *F = CI->getParent()->getParent();
89
90 auto MD = F->getMetadata("reqd_work_group_size");
91 const bool HasReqdWorkGroupSize = MD && MD->getNumOperands() == 3;
92
93 const bool HasUniformWorkGroupSize =
94 F->getFnAttribute("uniform-work-group-size").getValueAsBool();
95
96 if (!HasReqdWorkGroupSize && !HasUniformWorkGroupSize)
97 return false;
98
99 Value *BlockCounts[3] = {nullptr, nullptr, nullptr};
100 Value *GroupSizes[3] = {nullptr, nullptr, nullptr};
101 Value *Remainders[3] = {nullptr, nullptr, nullptr};
102 Value *GridSizes[3] = {nullptr, nullptr, nullptr};
103
104 const DataLayout &DL = F->getDataLayout();
105
106 // We expect to see several GEP users, casted to the appropriate type and
107 // loaded.
108 for (User *U : CI->users()) {
109 if (!U->hasOneUse())
110 continue;
111
112 int64_t Offset = 0;
113 auto *Load = dyn_cast<LoadInst>(U); // Load from ImplicitArgPtr/DispatchPtr?
114 auto *BCI = dyn_cast<BitCastInst>(U);
115 if (!Load && !BCI) {
117 continue;
118 Load = dyn_cast<LoadInst>(*U->user_begin()); // Load from GEP?
119 BCI = dyn_cast<BitCastInst>(*U->user_begin());
120 }
121
122 if (BCI) {
123 if (!BCI->hasOneUse())
124 continue;
125 Load = dyn_cast<LoadInst>(*BCI->user_begin()); // Load from BCI?
126 }
127
128 if (!Load || !Load->isSimple())
129 continue;
130
131 unsigned LoadSize = DL.getTypeStoreSize(Load->getType());
132
133 // TODO: Handle merged loads.
134 if (IsV5OrAbove) { // Base is ImplicitArgPtr.
135 switch (Offset) {
136 case HIDDEN_BLOCK_COUNT_X:
137 if (LoadSize == 4)
138 BlockCounts[0] = Load;
139 break;
140 case HIDDEN_BLOCK_COUNT_Y:
141 if (LoadSize == 4)
142 BlockCounts[1] = Load;
143 break;
144 case HIDDEN_BLOCK_COUNT_Z:
145 if (LoadSize == 4)
146 BlockCounts[2] = Load;
147 break;
148 case HIDDEN_GROUP_SIZE_X:
149 if (LoadSize == 2)
150 GroupSizes[0] = Load;
151 break;
152 case HIDDEN_GROUP_SIZE_Y:
153 if (LoadSize == 2)
154 GroupSizes[1] = Load;
155 break;
156 case HIDDEN_GROUP_SIZE_Z:
157 if (LoadSize == 2)
158 GroupSizes[2] = Load;
159 break;
160 case HIDDEN_REMAINDER_X:
161 if (LoadSize == 2)
162 Remainders[0] = Load;
163 break;
164 case HIDDEN_REMAINDER_Y:
165 if (LoadSize == 2)
166 Remainders[1] = Load;
167 break;
168 case HIDDEN_REMAINDER_Z:
169 if (LoadSize == 2)
170 Remainders[2] = Load;
171 break;
172 default:
173 break;
174 }
175 } else { // Base is DispatchPtr.
176 switch (Offset) {
177 case WORKGROUP_SIZE_X:
178 if (LoadSize == 2)
179 GroupSizes[0] = Load;
180 break;
181 case WORKGROUP_SIZE_Y:
182 if (LoadSize == 2)
183 GroupSizes[1] = Load;
184 break;
185 case WORKGROUP_SIZE_Z:
186 if (LoadSize == 2)
187 GroupSizes[2] = Load;
188 break;
189 case GRID_SIZE_X:
190 if (LoadSize == 4)
191 GridSizes[0] = Load;
192 break;
193 case GRID_SIZE_Y:
194 if (LoadSize == 4)
195 GridSizes[1] = Load;
196 break;
197 case GRID_SIZE_Z:
198 if (LoadSize == 4)
199 GridSizes[2] = Load;
200 break;
201 default:
202 break;
203 }
204 }
205 }
206
207 bool MadeChange = false;
208 if (IsV5OrAbove && HasUniformWorkGroupSize) {
209 // Under v5 __ockl_get_local_size returns the value computed by the expression:
210 //
211 // workgroup_id < hidden_block_count ? hidden_group_size : hidden_remainder
212 //
213 // For functions with the attribute uniform-work-group-size=true. we can evaluate
214 // workgroup_id < hidden_block_count as true, and thus hidden_group_size is returned
215 // for __ockl_get_local_size.
216 for (int I = 0; I < 3; ++I) {
217 Value *BlockCount = BlockCounts[I];
218 if (!BlockCount)
219 continue;
220
221 using namespace llvm::PatternMatch;
222 auto GroupIDIntrin =
223 I == 0 ? m_Intrinsic<Intrinsic::amdgcn_workgroup_id_x>()
224 : (I == 1 ? m_Intrinsic<Intrinsic::amdgcn_workgroup_id_y>()
225 : m_Intrinsic<Intrinsic::amdgcn_workgroup_id_z>());
226
227 for (User *ICmp : BlockCount->users()) {
228 if (match(ICmp, m_SpecificICmp(ICmpInst::ICMP_ULT, GroupIDIntrin,
229 m_Specific(BlockCount)))) {
230 ICmp->replaceAllUsesWith(llvm::ConstantInt::getTrue(ICmp->getType()));
231 MadeChange = true;
232 }
233 }
234 }
235
236 // All remainders should be 0 with uniform work group size.
237 for (Value *Remainder : Remainders) {
238 if (!Remainder)
239 continue;
240 Remainder->replaceAllUsesWith(Constant::getNullValue(Remainder->getType()));
241 MadeChange = true;
242 }
243 } else if (HasUniformWorkGroupSize) { // Pre-V5.
244 // Pattern match the code used to handle partial workgroup dispatches in the
245 // library implementation of get_local_size, so the entire function can be
246 // constant folded with a known group size.
247 //
248 // uint r = grid_size - group_id * group_size;
249 // get_local_size = (r < group_size) ? r : group_size;
250 //
251 // If we have uniform-work-group-size (which is the default in OpenCL 1.2),
252 // the grid_size is required to be a multiple of group_size). In this case:
253 //
254 // grid_size - (group_id * group_size) < group_size
255 // ->
256 // grid_size < group_size + (group_id * group_size)
257 //
258 // (grid_size / group_size) < 1 + group_id
259 //
260 // grid_size / group_size is at least 1, so we can conclude the select
261 // condition is false (except for group_id == 0, where the select result is
262 // the same).
263 for (int I = 0; I < 3; ++I) {
264 Value *GroupSize = GroupSizes[I];
265 Value *GridSize = GridSizes[I];
266 if (!GroupSize || !GridSize)
267 continue;
268
269 using namespace llvm::PatternMatch;
270 auto GroupIDIntrin =
271 I == 0 ? m_Intrinsic<Intrinsic::amdgcn_workgroup_id_x>()
272 : (I == 1 ? m_Intrinsic<Intrinsic::amdgcn_workgroup_id_y>()
273 : m_Intrinsic<Intrinsic::amdgcn_workgroup_id_z>());
274
275 for (User *U : GroupSize->users()) {
276 auto *ZextGroupSize = dyn_cast<ZExtInst>(U);
277 if (!ZextGroupSize)
278 continue;
279
280 for (User *UMin : ZextGroupSize->users()) {
281 if (match(UMin,
282 m_UMin(m_Sub(m_Specific(GridSize),
283 m_Mul(GroupIDIntrin, m_Specific(ZextGroupSize))),
284 m_Specific(ZextGroupSize)))) {
285 if (HasReqdWorkGroupSize) {
286 ConstantInt *KnownSize
287 = mdconst::extract<ConstantInt>(MD->getOperand(I));
288 UMin->replaceAllUsesWith(ConstantFoldIntegerCast(
289 KnownSize, UMin->getType(), false, DL));
290 } else {
291 UMin->replaceAllUsesWith(ZextGroupSize);
292 }
293
294 MadeChange = true;
295 }
296 }
297 }
298 }
299 }
300
301 // If reqd_work_group_size is set, we can replace work group size with it.
302 if (!HasReqdWorkGroupSize)
303 return MadeChange;
304
305 for (int I = 0; I < 3; I++) {
306 Value *GroupSize = GroupSizes[I];
307 if (!GroupSize)
308 continue;
309
310 ConstantInt *KnownSize = mdconst::extract<ConstantInt>(MD->getOperand(I));
311 GroupSize->replaceAllUsesWith(
312 ConstantFoldIntegerCast(KnownSize, GroupSize->getType(), false, DL));
313 MadeChange = true;
314 }
315
316 return MadeChange;
317}
318
319
320// TODO: Move makeLIDRangeMetadata usage into here. Seem to not get
321// TargetPassConfig for subtarget.
322bool AMDGPULowerKernelAttributes::runOnModule(Module &M) {
323 bool MadeChange = false;
324 bool IsV5OrAbove =
326 Function *BasePtr = getBasePtrIntrinsic(M, IsV5OrAbove);
327
328 if (!BasePtr) // ImplicitArgPtr/DispatchPtr not used.
329 return false;
330
332 for (auto *U : BasePtr->users()) {
333 CallInst *CI = cast<CallInst>(U);
334 if (HandledUses.insert(CI).second) {
335 if (processUse(CI, IsV5OrAbove))
336 MadeChange = true;
337 }
338 }
339
340 return MadeChange;
341}
342
343
344INITIALIZE_PASS_BEGIN(AMDGPULowerKernelAttributes, DEBUG_TYPE,
345 "AMDGPU Kernel Attributes", false, false)
346INITIALIZE_PASS_END(AMDGPULowerKernelAttributes, DEBUG_TYPE,
347 "AMDGPU Kernel Attributes", false, false)
348
349char AMDGPULowerKernelAttributes::ID = 0;
350
352 return new AMDGPULowerKernelAttributes();
353}
354
357 bool IsV5OrAbove =
359 Function *BasePtr = getBasePtrIntrinsic(*F.getParent(), IsV5OrAbove);
360
361 if (!BasePtr) // ImplicitArgPtr/DispatchPtr not used.
362 return PreservedAnalyses::all();
363
364 for (Instruction &I : instructions(F)) {
365 if (CallInst *CI = dyn_cast<CallInst>(&I)) {
366 if (CI->getCalledFunction() == BasePtr)
367 processUse(CI, IsV5OrAbove);
368 }
369 }
370
371 return PreservedAnalyses::all();
372}
AMDGPU Kernel Attributes
#define DEBUG_TYPE
static bool processUse(CallInst *CI, bool IsV5OrAbove)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
This file contains the declarations for the subclasses of Constant, which represent the different fla...
std::string Name
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:57
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
Target-Independent Code Generator Pass Configuration Options pass.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
Represent the analysis usage information of a pass.
void setPreservesAll()
Set by analyses that do not transform their input at all.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
This class represents a function call, abstracting a target machine's calling convention.
This is the shared class of boolean and integer constants.
Definition: Constants.h:81
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:850
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: Pass.cpp:98
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:111
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:117
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:503
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
iterator_range< user_iterator > users()
Definition: Value.h:421
const ParentTy * getParent() const
Definition: ilist_node.h:32
unsigned getAMDHSACodeObjectVersion(const Module &M)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:1096
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:875
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
SpecificCmpClass_match< LHS, RHS, ICmpInst, ICmpInst::Predicate > m_SpecificICmp(ICmpInst::Predicate MatchPred, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > m_UMin(const LHS &L, const RHS &R)
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:480
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
ModulePass * createAMDGPULowerKernelAttributesPass()
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
Constant * ConstantFoldIntegerCast(Constant *C, Type *DestTy, bool IsSigned, const DataLayout &DL)
Constant fold a zext, sext or trunc, depending on IsSigned and whether the DestTy is wider or narrowe...
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)