LLVM 19.0.0git
NVPTXCtorDtorLowering.cpp
Go to the documentation of this file.
1//===-- NVPTXCtorDtorLowering.cpp - Handle global ctors and dtors --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This pass creates a unified init and fini kernel with the required metadata
11//===----------------------------------------------------------------------===//
12
15#include "NVPTX.h"
17#include "llvm/IR/Constants.h"
18#include "llvm/IR/Function.h"
20#include "llvm/IR/IRBuilder.h"
21#include "llvm/IR/Module.h"
22#include "llvm/IR/Value.h"
23#include "llvm/Pass.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "nvptx-lower-ctor-dtor"
30
32 GlobalStr("nvptx-lower-global-ctor-dtor-id",
33 cl::desc("Override unique ID of ctor/dtor globals."),
34 cl::init(""), cl::Hidden);
35
36static cl::opt<bool>
37 CreateKernels("nvptx-emit-init-fini-kernel",
38 cl::desc("Emit kernels to call ctor/dtor globals."),
39 cl::init(true), cl::Hidden);
40
41namespace {
42
43static std::string getHash(StringRef Str) {
44 llvm::MD5 Hasher;
46 Hasher.update(Str);
47 Hasher.final(Hash);
48 return llvm::utohexstr(Hash.low(), /*LowerCase=*/true);
49}
50
51static void addKernelMetadata(Module &M, GlobalValue *GV) {
52 llvm::LLVMContext &Ctx = M.getContext();
53
54 // Get "nvvm.annotations" metadata node.
55 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations");
56
57 llvm::Metadata *KernelMDVals[] = {
60 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
61
62 // This kernel is only to be called single-threaded.
63 llvm::Metadata *ThreadXMDVals[] = {
66 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
67 llvm::Metadata *ThreadYMDVals[] = {
70 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
71 llvm::Metadata *ThreadZMDVals[] = {
74 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
75
76 llvm::Metadata *BlockMDVals[] = {
78 llvm::MDString::get(Ctx, "maxclusterrank"),
80 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
81
82 // Append metadata to nvvm.annotations.
83 MD->addOperand(llvm::MDNode::get(Ctx, KernelMDVals));
84 MD->addOperand(llvm::MDNode::get(Ctx, ThreadXMDVals));
85 MD->addOperand(llvm::MDNode::get(Ctx, ThreadYMDVals));
86 MD->addOperand(llvm::MDNode::get(Ctx, ThreadZMDVals));
87 MD->addOperand(llvm::MDNode::get(Ctx, BlockMDVals));
88}
89
90static Function *createInitOrFiniKernelFunction(Module &M, bool IsCtor) {
91 StringRef InitOrFiniKernelName =
92 IsCtor ? "nvptx$device$init" : "nvptx$device$fini";
93 if (M.getFunction(InitOrFiniKernelName))
94 return nullptr;
95
96 Function *InitOrFiniKernel = Function::createWithDefaultAttr(
97 FunctionType::get(Type::getVoidTy(M.getContext()), false),
98 GlobalValue::WeakODRLinkage, 0, InitOrFiniKernelName, &M);
99 addKernelMetadata(M, InitOrFiniKernel);
100
101 return InitOrFiniKernel;
102}
103
104// We create the IR required to call each callback in this section. This is
105// equivalent to the following code. Normally, the linker would provide us with
106// the definitions of the init and fini array sections. The 'nvlink' linker does
107// not do this so initializing these values is done by the runtime.
108//
109// extern "C" void **__init_array_start = nullptr;
110// extern "C" void **__init_array_end = nullptr;
111// extern "C" void **__fini_array_start = nullptr;
112// extern "C" void **__fini_array_end = nullptr;
113//
114// using InitCallback = void();
115// using FiniCallback = void();
116//
117// void call_init_array_callbacks() {
118// for (auto start = __init_array_start; start != __init_array_end; ++start)
119// reinterpret_cast<InitCallback *>(*start)();
120// }
121//
122// void call_init_array_callbacks() {
123// size_t fini_array_size = __fini_array_end - __fini_array_start;
124// for (size_t i = fini_array_size; i > 0; --i)
125// reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
126// }
127static void createInitOrFiniCalls(Function &F, bool IsCtor) {
128 Module &M = *F.getParent();
129 LLVMContext &C = M.getContext();
130
131 IRBuilder<> IRB(BasicBlock::Create(C, "entry", &F));
132 auto *LoopBB = BasicBlock::Create(C, "while.entry", &F);
133 auto *ExitBB = BasicBlock::Create(C, "while.end", &F);
134 Type *PtrTy = IRB.getPtrTy(llvm::ADDRESS_SPACE_GLOBAL);
135
136 auto *Begin = M.getOrInsertGlobal(
137 IsCtor ? "__init_array_start" : "__fini_array_start",
138 PointerType::get(C, 0), [&]() {
139 auto *GV = new GlobalVariable(
140 M, PointerType::get(C, 0),
141 /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
142 Constant::getNullValue(PointerType::get(C, 0)),
143 IsCtor ? "__init_array_start" : "__fini_array_start",
144 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
145 /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL);
146 GV->setVisibility(GlobalVariable::ProtectedVisibility);
147 return GV;
148 });
149 auto *End = M.getOrInsertGlobal(
150 IsCtor ? "__init_array_end" : "__fini_array_end", PointerType::get(C, 0),
151 [&]() {
152 auto *GV = new GlobalVariable(
153 M, PointerType::get(C, 0),
154 /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
155 Constant::getNullValue(PointerType::get(C, 0)),
156 IsCtor ? "__init_array_end" : "__fini_array_end",
157 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
158 /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL);
159 GV->setVisibility(GlobalVariable::ProtectedVisibility);
160 return GV;
161 });
162
163 // The constructor type is suppoed to allow using the argument vectors, but
164 // for now we just call them with no arguments.
165 auto *CallBackTy = FunctionType::get(IRB.getVoidTy(), {});
166
167 // The destructor array must be called in reverse order. Get an expression to
168 // the end of the array and iterate backwards in that case.
169 Value *BeginVal = IRB.CreateLoad(Begin->getType(), Begin, "begin");
170 Value *EndVal = IRB.CreateLoad(Begin->getType(), End, "stop");
171 if (!IsCtor) {
172 auto *BeginInt = IRB.CreatePtrToInt(BeginVal, IntegerType::getInt64Ty(C));
173 auto *EndInt = IRB.CreatePtrToInt(EndVal, IntegerType::getInt64Ty(C));
174 auto *SubInst = IRB.CreateSub(EndInt, BeginInt);
175 auto *Offset = IRB.CreateAShr(
176 SubInst, ConstantInt::get(IntegerType::getInt64Ty(C), 3), "offset",
177 /*IsExact=*/true);
178 auto *ValuePtr = IRB.CreateGEP(PointerType::get(C, 0), BeginVal,
180 EndVal = BeginVal;
181 BeginVal = IRB.CreateInBoundsGEP(
182 PointerType::get(C, 0), ValuePtr,
183 ArrayRef<Value *>(ConstantInt::get(IntegerType::getInt64Ty(C), -1)),
184 "start");
185 }
186 IRB.CreateCondBr(
187 IRB.CreateCmp(IsCtor ? ICmpInst::ICMP_NE : ICmpInst::ICMP_UGT, BeginVal,
188 EndVal),
189 LoopBB, ExitBB);
190 IRB.SetInsertPoint(LoopBB);
191 auto *CallBackPHI = IRB.CreatePHI(PtrTy, 2, "ptr");
192 auto *CallBack = IRB.CreateLoad(IRB.getPtrTy(F.getAddressSpace()),
193 CallBackPHI, "callback");
194 IRB.CreateCall(CallBackTy, CallBack);
195 auto *NewCallBack =
196 IRB.CreateConstGEP1_64(PtrTy, CallBackPHI, IsCtor ? 1 : -1, "next");
197 auto *EndCmp = IRB.CreateCmp(IsCtor ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_ULT,
198 NewCallBack, EndVal, "end");
199 CallBackPHI->addIncoming(BeginVal, &F.getEntryBlock());
200 CallBackPHI->addIncoming(NewCallBack, LoopBB);
201 IRB.CreateCondBr(EndCmp, ExitBB, LoopBB);
202 IRB.SetInsertPoint(ExitBB);
203 IRB.CreateRetVoid();
204}
205
206static bool createInitOrFiniGlobals(Module &M, GlobalVariable *GV,
207 bool IsCtor) {
208 ConstantArray *GA = dyn_cast<ConstantArray>(GV->getInitializer());
209 if (!GA || GA->getNumOperands() == 0)
210 return false;
211
212 // NVPTX has no way to emit variables at specific sections or support for
213 // the traditional constructor sections. Instead, we emit mangled global
214 // names so the runtime can build the list manually.
215 for (Value *V : GA->operands()) {
216 auto *CS = cast<ConstantStruct>(V);
217 auto *F = cast<Constant>(CS->getOperand(1));
218 uint64_t Priority = cast<ConstantInt>(CS->getOperand(0))->getSExtValue();
219 std::string PriorityStr = "." + std::to_string(Priority);
220 // We append a semi-unique hash and the priority to the global name.
221 std::string GlobalID =
222 !GlobalStr.empty() ? GlobalStr : getHash(M.getSourceFileName());
223 std::string NameStr =
224 ((IsCtor ? "__init_array_object_" : "__fini_array_object_") +
225 F->getName() + "_" + GlobalID + "_" + std::to_string(Priority))
226 .str();
227 // PTX does not support exported names with '.' in them.
228 llvm::transform(NameStr, NameStr.begin(),
229 [](char c) { return c == '.' ? '_' : c; });
230
231 auto *GV = new GlobalVariable(M, F->getType(), /*IsConstant=*/true,
234 /*AddressSpace=*/4);
235 // This isn't respected by Nvidia, simply put here for clarity.
236 GV->setSection(IsCtor ? ".init_array" + PriorityStr
237 : ".fini_array" + PriorityStr);
238 GV->setVisibility(GlobalVariable::ProtectedVisibility);
239 appendToUsed(M, {GV});
240 }
241
242 return true;
243}
244
245static bool createInitOrFiniKernel(Module &M, StringRef GlobalName,
246 bool IsCtor) {
247 GlobalVariable *GV = M.getGlobalVariable(GlobalName);
248 if (!GV || !GV->hasInitializer())
249 return false;
250
251 if (!createInitOrFiniGlobals(M, GV, IsCtor))
252 return false;
253
254 if (!CreateKernels)
255 return true;
256
257 Function *InitOrFiniKernel = createInitOrFiniKernelFunction(M, IsCtor);
258 if (!InitOrFiniKernel)
259 return false;
260
261 createInitOrFiniCalls(*InitOrFiniKernel, IsCtor);
262
263 GV->eraseFromParent();
264 return true;
265}
266
267static bool lowerCtorsAndDtors(Module &M) {
268 bool Modified = false;
269 Modified |= createInitOrFiniKernel(M, "llvm.global_ctors", /*IsCtor =*/true);
270 Modified |= createInitOrFiniKernel(M, "llvm.global_dtors", /*IsCtor =*/false);
271 return Modified;
272}
273
274class NVPTXCtorDtorLoweringLegacy final : public ModulePass {
275public:
276 static char ID;
277 NVPTXCtorDtorLoweringLegacy() : ModulePass(ID) {}
278 bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); }
279};
280
281} // End anonymous namespace
282
285 return lowerCtorsAndDtors(M) ? PreservedAnalyses::none()
287}
288
289char NVPTXCtorDtorLoweringLegacy::ID = 0;
290char &llvm::NVPTXCtorDtorLoweringLegacyPassID = NVPTXCtorDtorLoweringLegacy::ID;
291INITIALIZE_PASS(NVPTXCtorDtorLoweringLegacy, DEBUG_TYPE,
292 "Lower ctors and dtors for NVPTX", false, false)
293
295 return new NVPTXCtorDtorLoweringLegacy();
296}
This file contains the declarations for the subclasses of Constant, which represent the different fla...
bool End
Definition: ELF_riscv.cpp:480
#define DEBUG_TYPE
#define F(x, y, z)
Definition: MD5.cpp:55
Module.h This file contains the declarations for the Module class.
static cl::opt< bool > CreateKernels("nvptx-emit-init-fini-kernel", cl::desc("Emit kernels to call ctor/dtor globals."), cl::init(true), cl::Hidden)
static cl::opt< std::string > GlobalStr("nvptx-lower-global-ctor-dtor-id", cl::desc("Override unique ID of ctor/dtor globals."), cl::init(""), cl::Hidden)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
This file contains some functions that are useful when dealing with strings.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:348
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:198
ConstantArray - Constant Array Declarations.
Definition: Constants.h:423
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:528
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
static Function * createWithDefaultAttr(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Creates a function with some attributes recorded in llvm.module.flags applied.
Definition: Function.cpp:367
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:251
void setVisibility(VisibilityTypes V)
Definition: GlobalValue.h:254
@ WeakODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:57
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:52
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:56
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:455
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2644
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
Definition: MD5.h:41
void update(ArrayRef< uint8_t > Data)
Updates the hash for the byte stream provided.
Definition: MD5.cpp:189
void final(MD5Result &Result)
Finishes off the hash and puts the result in result.
Definition: MD5.cpp:234
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1541
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:600
Root of the metadata hierarchy.
Definition: Metadata.h:62
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A tuple of MDNodes.
Definition: Metadata.h:1729
void addOperand(MDNode *M)
Definition: Metadata.cpp:1388
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
op_range operands()
Definition: User.h:242
unsigned getNumOperands() const
Definition: User.h:191
LLVM Value Representation.
Definition: Value.h:74
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
@ ADDRESS_SPACE_GLOBAL
Definition: NVPTXBaseInfo.h:23
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
Definition: STLExtras.h:1937
char & NVPTXCtorDtorLoweringLegacyPassID
ModulePass * createNVPTXCtorDtorLoweringLegacyPass()
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
uint64_t low() const
Definition: MD5.h:46