95 : TheModule(M),
Context(M.getContext()),
114 "makeSubFnCall: Index value out of range");
125 static const char *
const CoroIntrinsics[] = {
134 "llvm.coro.id.retcon",
135 "llvm.coro.id.retcon.once",
138 "llvm.coro.prepare.retcon",
143 "llvm.coro.subfn.addr",
145 "llvm.coro.suspend.retcon",
154 std::initializer_list<StringRef>
List) {
169 if (
auto CF = dyn_cast<CoroFreeInst>(U))
172 if (CoroFrees.
empty())
177 : CoroFrees.
front()->getFrame();
180 CF->replaceAllUsesWith(Replacement);
181 CF->eraseFromParent();
193 if (
auto *Call = dyn_cast<CallBase>(&
I)) {
210 auto *ParentNode = CG[&ParentFunc];
211 ParentNode->removeAllCalledFunctions();
218 Nodes.push_back(Callee);
249 bool HasFinalSuspend =
false;
250 size_t FinalSuspendIndex = 0;
256 if (
auto II = dyn_cast<IntrinsicInst>(&
I)) {
257 switch (II->getIntrinsicID()) {
260 case Intrinsic::coro_size:
261 CoroSizes.
push_back(cast<CoroSizeInst>(II));
263 case Intrinsic::coro_frame:
264 CoroFrames.
push_back(cast<CoroFrameInst>(II));
266 case Intrinsic::coro_save:
270 UnusedCoroSaves.
push_back(cast<CoroSaveInst>(II));
272 case Intrinsic::coro_suspend_retcon: {
273 auto Suspend = cast<CoroSuspendRetconInst>(II);
274 CoroSuspends.push_back(Suspend);
277 case Intrinsic::coro_suspend: {
278 auto Suspend = cast<CoroSuspendInst>(II);
279 CoroSuspends.push_back(Suspend);
280 if (Suspend->isFinal()) {
283 "Only one suspend point can be marked as final");
284 HasFinalSuspend =
true;
285 FinalSuspendIndex = CoroSuspends.size() - 1;
289 case Intrinsic::coro_begin: {
290 auto CB = cast<CoroBeginInst>(II);
294 if (Id && !Id->getInfo().isPreSplit())
299 "coroutine should have exactly one defining @llvm.coro.begin");
303 Attribute::NoDuplicate);
307 case Intrinsic::coro_end:
308 CoroEnds.push_back(cast<CoroEndInst>(II));
309 if (CoroEnds.back()->isFallthrough()) {
312 if (CoroEnds.size() > 1) {
313 if (CoroEnds.front()->isFallthrough())
315 "Only one coro.end can be marked as fallthrough");
316 std::swap(CoroEnds.front(), CoroEnds.back());
330 CF->replaceAllUsesWith(
Undef);
331 CF->eraseFromParent();
338 CS->eraseFromParent();
339 if (
auto *CoroSave = CS->getCoroSave())
340 CoroSave->eraseFromParent();
350 auto Id = CoroBegin->getId();
351 switch (
auto IdIntrinsic =
Id->getIntrinsicID()) {
352 case Intrinsic::coro_id: {
353 auto SwitchId = cast<CoroIdInst>(
Id);
355 this->SwitchLowering.HasFinalSuspend = HasFinalSuspend;
356 this->SwitchLowering.ResumeSwitch =
nullptr;
357 this->SwitchLowering.PromiseAlloca = SwitchId->getPromise();
358 this->SwitchLowering.ResumeEntryBlock =
nullptr;
360 for (
auto AnySuspend : CoroSuspends) {
369 if (!Suspend->getCoroSave())
375 case Intrinsic::coro_id_retcon:
376 case Intrinsic::coro_id_retcon_once: {
377 auto ContinuationId = cast<AnyCoroIdRetconInst>(
Id);
378 ContinuationId->checkWellFormed();
379 this->
ABI = (IdIntrinsic == Intrinsic::coro_id_retcon
382 auto Prototype = ContinuationId->getPrototype();
383 this->RetconLowering.ResumePrototype =
Prototype;
384 this->RetconLowering.Alloc = ContinuationId->getAllocFunction();
385 this->RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
386 this->RetconLowering.ReturnBlock =
nullptr;
387 this->RetconLowering.IsFrameInlineInStorage =
false;
391 auto ResultTys = getRetconResultTypes();
392 auto ResumeTys = getRetconResumeTypes();
394 for (
auto AnySuspend : CoroSuspends) {
401 "coro.suspend.retcon");
405 auto SI = Suspend->value_begin(), SE = Suspend->value_end();
406 auto RI = ResultTys.begin(), RE = ResultTys.end();
407 for (;
SI != SE && RI != RE; ++
SI, ++RI) {
408 auto SrcTy = (*SI)->getType();
424 "match corresponding prototype function result");
427 if (
SI != SE || RI != RE) {
436 Type *SResultTy = Suspend->getType();
440 }
else if (
auto SResultStructTy = dyn_cast<StructType>(SResultTy)) {
441 SuspendResultTys = SResultStructTy->elements();
444 SuspendResultTys = SResultTy;
446 if (SuspendResultTys.
size() != ResumeTys.size()) {
453 for (
size_t I = 0,
E = ResumeTys.size();
I !=
E; ++
I) {
454 if (SuspendResultTys[
I] != ResumeTys[
I]) {
460 "match corresponding prototype function param");
473 CF->replaceAllUsesWith(CoroBegin);
474 CF->eraseFromParent();
479 SwitchLowering.HasFinalSuspend &&
480 FinalSuspendIndex != CoroSuspends.size() - 1)
481 std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());
485 CoroSave->eraseFromParent();
495 (*CG)[Call->
getFunction()]->addCalledFunction(Call, (*CG)[Callee]);
506 auto Alloc = RetconLowering.Alloc;
508 Alloc->getFunctionType()->getParamType(0),
527 auto Dealloc = RetconLowering.Dealloc;
529 Dealloc->getFunctionType()->getParamType(0));
530 auto *Call = Builder.
CreateCall(Dealloc, Ptr);
544 errs() <<
" Value: ";
557 fail(I,
"llvm.coro.id.retcon.* prototype not a Function", V);
559 auto FT =
F->getFunctionType();
561 if (isa<CoroIdRetconInst>(I)) {
563 if (FT->getReturnType()->isPointerTy()) {
565 }
else if (
auto SRetTy = dyn_cast<StructType>(FT->getReturnType())) {
566 ResultOkay = (!SRetTy->isOpaque() &&
567 SRetTy->getNumElements() > 0 &&
568 SRetTy->getElementType(0)->isPointerTy());
573 fail(I,
"llvm.coro.id.retcon prototype must return pointer as first " 576 if (FT->getReturnType() !=
578 fail(I,
"llvm.coro.id.retcon prototype return type must be same as" 579 "current function return type",
F);
584 if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
585 fail(I,
"llvm.coro.id.retcon.* prototype must take pointer as " 586 "its first parameter",
F);
593 fail(I,
"llvm.coro.* allocator not a Function", V);
595 auto FT =
F->getFunctionType();
596 if (!FT->getReturnType()->isPointerTy())
597 fail(I,
"llvm.coro.* allocator must return a pointer",
F);
599 if (FT->getNumParams() != 1 ||
600 !FT->getParamType(0)->isIntegerTy())
601 fail(I,
"llvm.coro.* allocator must take integer as only param",
F);
608 fail(I,
"llvm.coro.* deallocator not a Function", V);
610 auto FT =
F->getFunctionType();
611 if (!FT->getReturnType()->isVoidTy())
612 fail(I,
"llvm.coro.* deallocator must return void",
F);
614 if (FT->getNumParams() != 1 ||
615 !FT->getParamType(0)->isPointerTy())
616 fail(I,
"llvm.coro.* deallocator must take pointer as only param",
F);
620 const char *Reason) {
621 if (!isa<ConstantInt>(V)) {
628 "size argument to coro.id.retcon.* must be constant");
630 "alignment argument to coro.id.retcon.* must be constant");
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
This represents the llvm.coro.suspend.retcon instruction.
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
PassManagerBuilder - This class is used to set up a standard optimization sequence for languages like...
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
CoroBeginInst * CoroBegin
This represents the llvm.coro.id instruction.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
This class represents lattice values for constants.
void initializeCoroEarlyPass(PassRegistry &)
A Module instance is used to store all the information related to an LLVM module. ...
void initializeCoroElidePass(PassRegistry &)
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
void push_back(const T &Elt)
A global registry used in conjunction with static constructors to make pluggable components (like tar...
static void buildCGN(CallGraph &CG, CallGraphNode *Node)
This class represents a function call, abstracting a target machine's calling convention.
EP_ScalarOptimizerLate - This extension point allows adding optimization passes after most of the mai...
The two locations do not alias at all.
static void addCoroutineSCCPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
LLVMContext & getContext() const
All values hold a context through their type.
virtual void add(Pass *P)=0
Add a pass to the queue of passes to run.
void setArgOperand(unsigned i, Value *v)
Pass * createCoroEarlyPass()
Lower coroutine intrinsics that are not needed by later passes.
A node in the call graph for a module.
void dump() const
Support for debugging, callable in GDB: V->dump()
static bool isCoroutineIntrinsicName(StringRef Name)
Attribute unwrap(LLVMAttributeRef Attr)
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
This represents the llvm.coro.suspend instruction.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This file contains the simple types necessary to represent the attributes associated with functions a...
void buildFrom(Function &F)
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
Class to represent function types.
void initializeCoroSplitPass(PassRegistry &)
EP_EnabledOnOptLevel0 - This extension point allows adding passes that should not be disabled by O0 o...
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
FunctionType *const ResumeFnType
SmallVector< CoroSizeInst *, 2 > CoroSizes
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
static void checkConstantInt(const Instruction *I, Value *V, const char *Reason)
static void addCoroutineOptimizerLastPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
This class represents a no-op cast from one type to another.
GlobalValue * getNamedValue(StringRef Name) const
Return the global value in the module with the specified name, of arbitrary type. ...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type *> Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
This represents either the llvm.coro.id.retcon or llvm.coro.id.retcon.once instruction.
static void addCoroutineScalarOptimizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
static void checkWFDealloc(const Instruction *I, Value *V)
Check that the given value is a well-formed deallocator.
bool isVoidTy() const
Return true if this is 'void'.
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
bool isLeaf(ID id)
Returns true if the intrinsic is a leaf, i.e.
CallGraphNode * getCallsExternalNode() const
The instances of the Type class are immutable: once they are created, they are never changed...
size_t size() const
size - Get the array size.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Pass * createCoroCleanupPass()
Lower all remaining coroutine intrinsics.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Pass * createCoroElidePass()
Analyze coroutines use sites, devirtualize resume/destroy calls and elide heap allocation for corouti...
static bool isBitCastable(Type *SrcTy, Type *DestTy)
Check whether a bitcast between these types is valid.
This represents the llvm.coro.end instruction.
static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee)
void setCallingConv(CallingConv::ID CC)
ModulePass * createBarrierNoopPass()
createBarrierNoopPass - This pass is purely a module pass barrier in a pass manager.
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This represents the llvm.coro.save instruction.
void initialize(ArrayRef< CallGraphNode *> NewNodes)
const Function * getFunction() const
Return the function this instruction belongs to.
static void checkWFAlloc(const Instruction *I, Value *V)
Check that the given value is a well-formed allocator.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
EP_OptimizerLast – This extension point allows adding passes that run after everything else...
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
EP_EarlyAsPossible - This extension point allows adding passes before any other transformations, allowing them to see the code as it is coming out of the frontend.
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
This represents the llvm.coro.free instruction.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Function * getFunction() const
Returns the function that this call graph node represents.
EP_CGSCCOptimizerLate - This extension point allows adding CallGraphSCC passes at the end of the main...
static void addCoroutineOpt0Passes(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
PassManagerBase - An abstract interface to allow code to add passes to a pass manager without having ...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
void initializeCoroutines(PassRegistry &)
Initialize all passes linked into the Coroutines library.
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
static CoroSaveInst * createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
A constant pointer value that points to null.
Type * getReturnType() const
void checkWellFormed() const
#define LLVM_ATTRIBUTE_NORETURN
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static void addCoroutineEarlyPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
struct LLVMOpaquePassManager * LLVMPassManagerRef
This class represents the llvm.coro.begin instruction.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Value * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
amdgpu Simplify well known AMD library false FunctionCallee Callee
iterator_range< user_iterator > users()
static void clear(coro::Shape &Shape)
void initializeCoroCleanupPass(PassRegistry &)
The basic data container for the call graph of a Module of IR.
LLVM_NODISCARD bool empty() const
Pass * createCoroSplitPass()
Split up coroutines into multiple functions driving their state machines.
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
static LLVM_ATTRIBUTE_NORETURN void fail(const Instruction *I, const char *Reason, Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SmallVector< CoroEndInst *, 4 > CoroEnds
This represents the llvm.coro.frame instruction.
LLVM Value Representation.
CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on.
CallGraphNode * getOrInsertFunction(const Function *F)
Similar to operator[], but this will insert a new CallGraphNode for F if one does not already exist...
void addCoroutinePassesToExtensionPoints(PassManagerBuilder &Builder)
Add all coroutine passes to appropriate extension points.
bool declaresIntrinsics(Module &M, std::initializer_list< StringRef >)
StringRef - Represent a constant reference to a string, i.e.
inst_range instructions(Function *F)
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
int lookupLLVMIntrinsicByName(ArrayRef< const char *> NameTable, StringRef Name)
Looks up Name in NameTable via binary search.
BasicBlock * AllocaSpillBlock
void addExtension(ExtensionPointTy Ty, ExtensionFn Fn)
static void checkWFRetconPrototype(const AnyCoroIdRetconInst *I, Value *V)
Check that the given value is a well-formed prototype for the llvm.coro.id.retcon.
unsigned changeToUnreachable(Instruction *I, bool UseLLVMTrap, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
static IntegerType * getInt8Ty(LLVMContext &C)
CoroSaveInst * getCoroSave() const
void updateCallGraph(Function &Caller, ArrayRef< Function *> Funcs, CallGraph &CG, CallGraphSCC &SCC)
static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee)
void addCalledFunction(CallBase *Call, CallGraphNode *M)
Adds a function to the list of functions called by this one.