LLVM 19.0.0git
NVPTXUtilities.cpp
Go to the documentation of this file.
1//===- NVPTXUtilities.cpp - Utility Functions -----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains miscellaneous utility functions
10//
11//===----------------------------------------------------------------------===//
12
13#include "NVPTXUtilities.h"
14#include "NVPTX.h"
15#include "NVPTXTargetMachine.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/Function.h"
20#include "llvm/IR/Module.h"
21#include "llvm/IR/Operator.h"
23#include "llvm/Support/Mutex.h"
24#include <algorithm>
25#include <cstring>
26#include <map>
27#include <mutex>
28#include <optional>
29#include <string>
30#include <vector>
31
32namespace llvm {
33
34namespace {
35typedef std::map<std::string, std::vector<unsigned> > key_val_pair_t;
36typedef std::map<const GlobalValue *, key_val_pair_t> global_val_annot_t;
37
38struct AnnotationCache {
39 sys::Mutex Lock;
40 std::map<const Module *, global_val_annot_t> Cache;
41};
42
43AnnotationCache &getAnnotationCache() {
44 static AnnotationCache AC;
45 return AC;
46}
47} // anonymous namespace
48
50 auto &AC = getAnnotationCache();
51 std::lock_guard<sys::Mutex> Guard(AC.Lock);
52 AC.Cache.erase(Mod);
53}
54
55static void cacheAnnotationFromMD(const MDNode *md, key_val_pair_t &retval) {
56 auto &AC = getAnnotationCache();
57 std::lock_guard<sys::Mutex> Guard(AC.Lock);
58 assert(md && "Invalid mdnode for annotation");
59 assert((md->getNumOperands() % 2) == 1 && "Invalid number of operands");
60 // start index = 1, to skip the global variable key
61 // increment = 2, to skip the value for each property-value pairs
62 for (unsigned i = 1, e = md->getNumOperands(); i != e; i += 2) {
63 // property
64 const MDString *prop = dyn_cast<MDString>(md->getOperand(i));
65 assert(prop && "Annotation property not a string");
66
67 // value
68 ConstantInt *Val = mdconst::dyn_extract<ConstantInt>(md->getOperand(i + 1));
69 assert(Val && "Value operand not a constant int");
70
71 std::string keyname = prop->getString().str();
72 if (retval.find(keyname) != retval.end())
73 retval[keyname].push_back(Val->getZExtValue());
74 else {
75 std::vector<unsigned> tmp;
76 tmp.push_back(Val->getZExtValue());
77 retval[keyname] = tmp;
78 }
79 }
80}
81
82static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) {
83 auto &AC = getAnnotationCache();
84 std::lock_guard<sys::Mutex> Guard(AC.Lock);
85 NamedMDNode *NMD = m->getNamedMetadata("nvvm.annotations");
86 if (!NMD)
87 return;
88 key_val_pair_t tmp;
89 for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
90 const MDNode *elem = NMD->getOperand(i);
91
92 GlobalValue *entity =
93 mdconst::dyn_extract_or_null<GlobalValue>(elem->getOperand(0));
94 // entity may be null due to DCE
95 if (!entity)
96 continue;
97 if (entity != gv)
98 continue;
99
100 // accumulate annotations for entity in tmp
101 cacheAnnotationFromMD(elem, tmp);
102 }
103
104 if (tmp.empty()) // no annotations for this gv
105 return;
106
107 if (AC.Cache.find(m) != AC.Cache.end())
108 AC.Cache[m][gv] = std::move(tmp);
109 else {
110 global_val_annot_t tmp1;
111 tmp1[gv] = std::move(tmp);
112 AC.Cache[m] = std::move(tmp1);
113 }
114}
115
116bool findOneNVVMAnnotation(const GlobalValue *gv, const std::string &prop,
117 unsigned &retval) {
118 auto &AC = getAnnotationCache();
119 std::lock_guard<sys::Mutex> Guard(AC.Lock);
120 const Module *m = gv->getParent();
121 if (AC.Cache.find(m) == AC.Cache.end())
123 else if (AC.Cache[m].find(gv) == AC.Cache[m].end())
125 if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end())
126 return false;
127 retval = AC.Cache[m][gv][prop][0];
128 return true;
129}
130
131static std::optional<unsigned>
132findOneNVVMAnnotation(const GlobalValue &GV, const std::string &PropName) {
133 unsigned RetVal;
134 if (findOneNVVMAnnotation(&GV, PropName, RetVal))
135 return RetVal;
136 return std::nullopt;
137}
138
139bool findAllNVVMAnnotation(const GlobalValue *gv, const std::string &prop,
140 std::vector<unsigned> &retval) {
141 auto &AC = getAnnotationCache();
142 std::lock_guard<sys::Mutex> Guard(AC.Lock);
143 const Module *m = gv->getParent();
144 if (AC.Cache.find(m) == AC.Cache.end())
146 else if (AC.Cache[m].find(gv) == AC.Cache[m].end())
148 if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end())
149 return false;
150 retval = AC.Cache[m][gv][prop];
151 return true;
152}
153
154bool isTexture(const Value &val) {
155 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
156 unsigned annot;
157 if (findOneNVVMAnnotation(gv, "texture", annot)) {
158 assert((annot == 1) && "Unexpected annotation on a texture symbol");
159 return true;
160 }
161 }
162 return false;
163}
164
165bool isSurface(const Value &val) {
166 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
167 unsigned annot;
168 if (findOneNVVMAnnotation(gv, "surface", annot)) {
169 assert((annot == 1) && "Unexpected annotation on a surface symbol");
170 return true;
171 }
172 }
173 return false;
174}
175
176bool isSampler(const Value &val) {
177 const char *AnnotationName = "sampler";
178
179 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
180 unsigned annot;
181 if (findOneNVVMAnnotation(gv, AnnotationName, annot)) {
182 assert((annot == 1) && "Unexpected annotation on a sampler symbol");
183 return true;
184 }
185 }
186 if (const Argument *arg = dyn_cast<Argument>(&val)) {
187 const Function *func = arg->getParent();
188 std::vector<unsigned> annot;
189 if (findAllNVVMAnnotation(func, AnnotationName, annot)) {
190 if (is_contained(annot, arg->getArgNo()))
191 return true;
192 }
193 }
194 return false;
195}
196
197bool isImageReadOnly(const Value &val) {
198 if (const Argument *arg = dyn_cast<Argument>(&val)) {
199 const Function *func = arg->getParent();
200 std::vector<unsigned> annot;
201 if (findAllNVVMAnnotation(func, "rdoimage", annot)) {
202 if (is_contained(annot, arg->getArgNo()))
203 return true;
204 }
205 }
206 return false;
207}
208
209bool isImageWriteOnly(const Value &val) {
210 if (const Argument *arg = dyn_cast<Argument>(&val)) {
211 const Function *func = arg->getParent();
212 std::vector<unsigned> annot;
213 if (findAllNVVMAnnotation(func, "wroimage", annot)) {
214 if (is_contained(annot, arg->getArgNo()))
215 return true;
216 }
217 }
218 return false;
219}
220
221bool isImageReadWrite(const Value &val) {
222 if (const Argument *arg = dyn_cast<Argument>(&val)) {
223 const Function *func = arg->getParent();
224 std::vector<unsigned> annot;
225 if (findAllNVVMAnnotation(func, "rdwrimage", annot)) {
226 if (is_contained(annot, arg->getArgNo()))
227 return true;
228 }
229 }
230 return false;
231}
232
233bool isImage(const Value &val) {
234 return isImageReadOnly(val) || isImageWriteOnly(val) || isImageReadWrite(val);
235}
236
237bool isManaged(const Value &val) {
238 if(const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
239 unsigned annot;
240 if (findOneNVVMAnnotation(gv, "managed", annot)) {
241 assert((annot == 1) && "Unexpected annotation on a managed symbol");
242 return true;
243 }
244 }
245 return false;
246}
247
248std::string getTextureName(const Value &val) {
249 assert(val.hasName() && "Found texture variable with no name");
250 return std::string(val.getName());
251}
252
253std::string getSurfaceName(const Value &val) {
254 assert(val.hasName() && "Found surface variable with no name");
255 return std::string(val.getName());
256}
257
258std::string getSamplerName(const Value &val) {
259 assert(val.hasName() && "Found sampler variable with no name");
260 return std::string(val.getName());
261}
262
263std::optional<unsigned> getMaxNTIDx(const Function &F) {
264 return findOneNVVMAnnotation(F, "maxntidx");
265}
266
267std::optional<unsigned> getMaxNTIDy(const Function &F) {
268 return findOneNVVMAnnotation(F, "maxntidy");
269}
270
271std::optional<unsigned> getMaxNTIDz(const Function &F) {
272 return findOneNVVMAnnotation(F, "maxntidz");
273}
274
275std::optional<unsigned> getMaxNTID(const Function &F) {
276 // Note: The semantics here are a bit strange. The PTX ISA states the
277 // following (11.4.2. Performance-Tuning Directives: .maxntid):
278 //
279 // Note that this directive guarantees that the total number of threads does
280 // not exceed the maximum, but does not guarantee that the limit in any
281 // particular dimension is not exceeded.
282 std::optional<unsigned> MaxNTIDx = getMaxNTIDx(F);
283 std::optional<unsigned> MaxNTIDy = getMaxNTIDy(F);
284 std::optional<unsigned> MaxNTIDz = getMaxNTIDz(F);
285 if (MaxNTIDx || MaxNTIDy || MaxNTIDz)
286 return MaxNTIDx.value_or(1) * MaxNTIDy.value_or(1) * MaxNTIDz.value_or(1);
287 return std::nullopt;
288}
289
290bool getMaxClusterRank(const Function &F, unsigned &x) {
291 return findOneNVVMAnnotation(&F, "maxclusterrank", x);
292}
293
294std::optional<unsigned> getReqNTIDx(const Function &F) {
295 return findOneNVVMAnnotation(F, "reqntidx");
296}
297
298std::optional<unsigned> getReqNTIDy(const Function &F) {
299 return findOneNVVMAnnotation(F, "reqntidy");
300}
301
302std::optional<unsigned> getReqNTIDz(const Function &F) {
303 return findOneNVVMAnnotation(F, "reqntidz");
304}
305
306std::optional<unsigned> getReqNTID(const Function &F) {
307 // Note: The semantics here are a bit strange. See getMaxNTID.
308 std::optional<unsigned> ReqNTIDx = getReqNTIDx(F);
309 std::optional<unsigned> ReqNTIDy = getReqNTIDy(F);
310 std::optional<unsigned> ReqNTIDz = getReqNTIDz(F);
311 if (ReqNTIDx || ReqNTIDy || ReqNTIDz)
312 return ReqNTIDx.value_or(1) * ReqNTIDy.value_or(1) * ReqNTIDz.value_or(1);
313 return std::nullopt;
314}
315
316bool getMinCTASm(const Function &F, unsigned &x) {
317 return findOneNVVMAnnotation(&F, "minctasm", x);
318}
319
320bool getMaxNReg(const Function &F, unsigned &x) {
321 return findOneNVVMAnnotation(&F, "maxnreg", x);
322}
323
325 unsigned x = 0;
326 bool retval = findOneNVVMAnnotation(&F, "kernel", x);
327 if (!retval) {
328 // There is no NVVM metadata, check the calling convention
329 return F.getCallingConv() == CallingConv::PTX_Kernel;
330 }
331 return (x == 1);
332}
333
334MaybeAlign getAlign(const Function &F, unsigned Index) {
335 // First check the alignstack metadata
336 if (MaybeAlign StackAlign =
337 F.getAttributes().getAttributes(Index).getStackAlignment())
338 return StackAlign;
339
340 // If that is missing, check the legacy nvvm metadata
341 std::vector<unsigned> Vs;
342 bool retval = findAllNVVMAnnotation(&F, "align", Vs);
343 if (!retval)
344 return std::nullopt;
345 for (unsigned V : Vs)
346 if ((V >> 16) == Index)
347 return Align(V & 0xFFFF);
348
349 return std::nullopt;
350}
351
352MaybeAlign getAlign(const CallInst &I, unsigned Index) {
353 // First check the alignstack metadata
354 if (MaybeAlign StackAlign =
355 I.getAttributes().getAttributes(Index).getStackAlignment())
356 return StackAlign;
357
358 // If that is missing, check the legacy nvvm metadata
359 if (MDNode *alignNode = I.getMetadata("callalign")) {
360 for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) {
361 if (const ConstantInt *CI =
362 mdconst::dyn_extract<ConstantInt>(alignNode->getOperand(i))) {
363 unsigned V = CI->getZExtValue();
364 if ((V >> 16) == Index)
365 return Align(V & 0xFFFF);
366 if ((V >> 16) > Index)
367 return std::nullopt;
368 }
369 }
370 }
371 return std::nullopt;
372}
373
375 return dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());
376}
377
379 const auto &ST =
380 *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl();
381 if (!ST.hasNoReturn())
382 return false;
383
384 assert((isa<Function>(V) || isa<CallInst>(V)) &&
385 "Expect either a call instruction or a function");
386
387 if (const CallInst *CallI = dyn_cast<CallInst>(V))
388 return CallI->doesNotReturn() &&
389 CallI->getFunctionType()->getReturnType()->isVoidTy();
390
391 const Function *F = cast<Function>(V);
392 return F->doesNotReturn() &&
393 F->getFunctionType()->getReturnType()->isVoidTy() &&
395}
396
397bool Isv2x16VT(EVT VT) {
398 return (VT == MVT::v2f16 || VT == MVT::v2bf16 || VT == MVT::v2i16);
399}
400
401} // namespace llvm
This file contains the declarations for the subclasses of Constant, which represent the different fla...
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Module.h This file contains the declarations for the Module class.
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1494
Value * getCalledOperand() const
Definition: InstrTypes.h:1743
This class represents a function call, abstracting a target machine's calling convention.
This is the shared class of boolean and integer constants.
Definition: Constants.h:81
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:155
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
Metadata node.
Definition: Metadata.h:1067
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1428
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1434
A single uniqued string.
Definition: Metadata.h:720
StringRef getString() const
Definition: Metadata.cpp:610
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
NamedMDNode * getNamedMetadata(const Twine &Name) const
Return the first NamedMDNode in the module with the specified name.
Definition: Module.cpp:262
A tuple of MDNodes.
Definition: Metadata.h:1729
MDNode * getOperand(unsigned i) const
Definition: Metadata.cpp:1381
unsigned getNumOperands() const
Definition: Metadata.cpp:1377
std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:223
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
LLVM Value Representation.
Definition: Value.h:74
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:693
bool hasName() const
Definition: Value.h:261
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition: CallingConv.h:125
SmartMutex< false > Mutex
Mutex - A standard, always enforced mutex.
Definition: Mutex.h:66
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM)
bool Isv2x16VT(EVT VT)
std::optional< unsigned > getMaxNTIDy(const Function &F)
std::string getSamplerName(const Value &val)
bool getMinCTASm(const Function &F, unsigned &x)
bool isImage(const Value &val)
std::optional< unsigned > getMaxNTIDz(const Function &F)
MaybeAlign getAlign(const Function &F, unsigned Index)
std::optional< unsigned > getMaxNTIDx(const Function &F)
bool isImageReadOnly(const Value &val)
bool isManaged(const Value &val)
bool isSurface(const Value &val)
bool findAllNVVMAnnotation(const GlobalValue *gv, const std::string &prop, std::vector< unsigned > &retval)
void clearAnnotationCache(const Module *Mod)
std::string getSurfaceName(const Value &val)
std::optional< unsigned > getReqNTIDy(const Function &F)
@ Mod
The access may modify the value stored in memory.
bool getMaxNReg(const Function &F, unsigned &x)
bool isTexture(const Value &val)
static void cacheAnnotationFromMD(const MDNode *md, key_val_pair_t &retval)
bool isImageWriteOnly(const Value &val)
bool isImageReadWrite(const Value &val)
std::string getTextureName(const Value &val)
bool isKernelFunction(const Function &F)
Function * getMaybeBitcastedCallee(const CallBase *CB)
bool findOneNVVMAnnotation(const GlobalValue *gv, const std::string &prop, unsigned &retval)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1879
std::optional< unsigned > getReqNTIDz(const Function &F)
std::optional< unsigned > getReqNTIDx(const Function &F)
bool isSampler(const Value &val)
bool getMaxClusterRank(const Function &F, unsigned &x)
std::optional< unsigned > getReqNTID(const Function &F)
std::optional< unsigned > getMaxNTID(const Function &F)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117