39 : TheModule(M),
Context(M.getContext()),
58 "makeSubFnCall: Index value out of range");
62 new BitCastInst(Call, ResumeFnType->getPointerTo(),
"", InsertPt);
70 "llvm.coro.async.context.alloc",
71 "llvm.coro.async.context.dealloc",
72 "llvm.coro.async.resume",
73 "llvm.coro.async.size.replace",
74 "llvm.coro.async.store_resume",
79 "llvm.coro.end.async",
84 "llvm.coro.id.retcon",
85 "llvm.coro.id.retcon.once",
87 "llvm.coro.prepare.async",
88 "llvm.coro.prepare.retcon",
93 "llvm.coro.subfn.addr",
95 "llvm.coro.suspend.async",
96 "llvm.coro.suspend.retcon",
108 if (M.getNamedValue(
Name))
118 const std::initializer_list<StringRef>
List) {
121 if (M.getNamedValue(
Name))
133 if (
auto CF = dyn_cast<CoroFreeInst>(U))
136 if (CoroFrees.
empty())
141 : CoroFrees.
front()->getFrame();
144 CF->replaceAllUsesWith(Replacement);
145 CF->eraseFromParent();
173 bool HasFinalSuspend =
false;
174 bool HasUnwindCoroEnd =
false;
175 size_t FinalSuspendIndex = 0;
181 if (
auto II = dyn_cast<IntrinsicInst>(&
I)) {
182 switch (II->getIntrinsicID()) {
185 case Intrinsic::coro_size:
186 CoroSizes.
push_back(cast<CoroSizeInst>(II));
188 case Intrinsic::coro_align:
189 CoroAligns.push_back(cast<CoroAlignInst>(II));
191 case Intrinsic::coro_frame:
192 CoroFrames.
push_back(cast<CoroFrameInst>(II));
194 case Intrinsic::coro_save:
198 UnusedCoroSaves.
push_back(cast<CoroSaveInst>(II));
200 case Intrinsic::coro_suspend_async: {
201 auto *Suspend = cast<CoroSuspendAsyncInst>(II);
202 Suspend->checkWellFormed();
203 CoroSuspends.push_back(Suspend);
206 case Intrinsic::coro_suspend_retcon: {
207 auto Suspend = cast<CoroSuspendRetconInst>(II);
208 CoroSuspends.push_back(Suspend);
211 case Intrinsic::coro_suspend: {
212 auto Suspend = cast<CoroSuspendInst>(II);
213 CoroSuspends.push_back(Suspend);
214 if (Suspend->isFinal()) {
217 "Only one suspend point can be marked as final");
218 HasFinalSuspend =
true;
219 FinalSuspendIndex = CoroSuspends.size() - 1;
223 case Intrinsic::coro_begin: {
224 auto CB = cast<CoroBeginInst>(II);
227 auto Id = dyn_cast<CoroIdInst>(CB->getId());
228 if (Id && !Id->getInfo().isPreSplit())
233 "coroutine should have exactly one defining @llvm.coro.begin");
234 CB->addRetAttr(Attribute::NonNull);
235 CB->addRetAttr(Attribute::NoAlias);
236 CB->removeFnAttr(Attribute::NoDuplicate);
240 case Intrinsic::coro_end_async:
241 case Intrinsic::coro_end:
242 CoroEnds.push_back(cast<AnyCoroEndInst>(II));
243 if (
auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(II)) {
244 AsyncEnd->checkWellFormed();
247 if (CoroEnds.back()->isUnwind())
248 HasUnwindCoroEnd =
true;
250 if (CoroEnds.back()->isFallthrough() && isa<CoroEndInst>(II)) {
254 if (CoroEnds.size() > 1) {
255 if (CoroEnds.front()->isFallthrough())
257 "Only one coro.end can be marked as fallthrough");
258 std::swap(CoroEnds.front(), CoroEnds.back());
272 CF->replaceAllUsesWith(Undef);
273 CF->eraseFromParent();
280 CS->eraseFromParent();
281 if (
auto *CoroSave = CS->getCoroSave())
282 CoroSave->eraseFromParent();
292 auto Id = CoroBegin->getId();
293 switch (
auto IdIntrinsic = Id->getIntrinsicID()) {
294 case Intrinsic::coro_id: {
295 auto SwitchId = cast<CoroIdInst>(Id);
297 this->SwitchLowering.HasFinalSuspend = HasFinalSuspend;
298 this->SwitchLowering.HasUnwindCoroEnd = HasUnwindCoroEnd;
299 this->SwitchLowering.ResumeSwitch =
nullptr;
300 this->SwitchLowering.PromiseAlloca = SwitchId->getPromise();
301 this->SwitchLowering.ResumeEntryBlock =
nullptr;
303 for (
auto *AnySuspend : CoroSuspends) {
304 auto Suspend = dyn_cast<CoroSuspendInst>(AnySuspend);
312 if (!Suspend->getCoroSave())
317 case Intrinsic::coro_id_async: {
318 auto *AsyncId = cast<CoroIdAsyncInst>(Id);
319 AsyncId->checkWellFormed();
321 this->AsyncLowering.Context = AsyncId->getStorage();
322 this->AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex();
323 this->AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize();
324 this->AsyncLowering.ContextAlignment =
325 AsyncId->getStorageAlignment().value();
326 this->AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer();
327 this->AsyncLowering.AsyncCC =
F.getCallingConv();
330 case Intrinsic::coro_id_retcon:
331 case Intrinsic::coro_id_retcon_once: {
332 auto ContinuationId = cast<AnyCoroIdRetconInst>(Id);
333 ContinuationId->checkWellFormed();
334 this->
ABI = (IdIntrinsic == Intrinsic::coro_id_retcon
337 auto Prototype = ContinuationId->getPrototype();
338 this->RetconLowering.ResumePrototype = Prototype;
339 this->RetconLowering.Alloc = ContinuationId->getAllocFunction();
340 this->RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
341 this->RetconLowering.ReturnBlock =
nullptr;
342 this->RetconLowering.IsFrameInlineInStorage =
false;
346 auto ResultTys = getRetconResultTypes();
347 auto ResumeTys = getRetconResumeTypes();
349 for (
auto *AnySuspend : CoroSuspends) {
350 auto Suspend = dyn_cast<CoroSuspendRetconInst>(AnySuspend);
356 "coro.suspend.retcon");
360 auto SI = Suspend->value_begin(), SE = Suspend->value_end();
361 auto RI = ResultTys.begin(), RE = ResultTys.end();
362 for (;
SI != SE && RI != RE; ++
SI, ++RI) {
363 auto SrcTy = (*SI)->getType();
376 Prototype->getFunctionType()->dump();
379 "match corresponding prototype function result");
382 if (
SI != SE || RI != RE) {
385 Prototype->getFunctionType()->dump();
391 Type *SResultTy = Suspend->getType();
395 }
else if (
auto SResultStructTy = dyn_cast<StructType>(SResultTy)) {
396 SuspendResultTys = SResultStructTy->elements();
399 SuspendResultTys = SResultTy;
401 if (SuspendResultTys.
size() != ResumeTys.size()) {
404 Prototype->getFunctionType()->dump();
408 for (
size_t I = 0,
E = ResumeTys.size();
I !=
E; ++
I) {
409 if (SuspendResultTys[
I] != ResumeTys[
I]) {
412 Prototype->getFunctionType()->dump();
415 "match corresponding prototype function param");
428 CF->replaceAllUsesWith(CoroBegin);
429 CF->eraseFromParent();
434 SwitchLowering.HasFinalSuspend &&
435 FinalSuspendIndex != CoroSuspends.size() - 1)
436 std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());
440 CoroSave->eraseFromParent();
444 Call->setCallingConv(
Callee->getCallingConv());
450 (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[
Callee]);
461 auto Alloc = RetconLowering.Alloc;
463 Alloc->getFunctionType()->getParamType(0),
484 auto Dealloc = RetconLowering.Dealloc;
486 Dealloc->getFunctionType()->getParamType(0));
487 auto *Call =
Builder.CreateCall(Dealloc,
Ptr);
503 errs() <<
" Value: ";
514 auto F = dyn_cast<Function>(V->stripPointerCasts());
516 fail(
I,
"llvm.coro.id.retcon.* prototype not a Function", V);
518 auto FT =
F->getFunctionType();
520 if (isa<CoroIdRetconInst>(
I)) {
522 if (FT->getReturnType()->isPointerTy()) {
524 }
else if (
auto SRetTy = dyn_cast<StructType>(FT->getReturnType())) {
525 ResultOkay = (!SRetTy->isOpaque() &&
526 SRetTy->getNumElements() > 0 &&
527 SRetTy->getElementType(0)->isPointerTy());
532 fail(
I,
"llvm.coro.id.retcon prototype must return pointer as first "
535 if (FT->getReturnType() !=
536 I->getFunction()->getFunctionType()->getReturnType())
537 fail(
I,
"llvm.coro.id.retcon prototype return type must be same as"
538 "current function return type",
F);
543 if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
544 fail(
I,
"llvm.coro.id.retcon.* prototype must take pointer as "
545 "its first parameter",
F);
550 auto F = dyn_cast<Function>(V->stripPointerCasts());
552 fail(
I,
"llvm.coro.* allocator not a Function", V);
554 auto FT =
F->getFunctionType();
555 if (!FT->getReturnType()->isPointerTy())
556 fail(
I,
"llvm.coro.* allocator must return a pointer",
F);
558 if (FT->getNumParams() != 1 ||
559 !FT->getParamType(0)->isIntegerTy())
560 fail(
I,
"llvm.coro.* allocator must take integer as only param",
F);
565 auto F = dyn_cast<Function>(V->stripPointerCasts());
567 fail(
I,
"llvm.coro.* deallocator not a Function", V);
569 auto FT =
F->getFunctionType();
570 if (!FT->getReturnType()->isVoidTy())
571 fail(
I,
"llvm.coro.* deallocator must return void",
F);
573 if (FT->getNumParams() != 1 ||
574 !FT->getParamType(0)->isPointerTy())
575 fail(
I,
"llvm.coro.* deallocator must take pointer as only param",
F);
579 const char *Reason) {
580 if (!isa<ConstantInt>(V)) {
587 "size argument to coro.id.retcon.* must be constant");
589 "alignment argument to coro.id.retcon.* must be constant");
596 auto *AsyncFuncPtrAddr = dyn_cast<GlobalVariable>(V->stripPointerCasts());
597 if (!AsyncFuncPtrAddr)
598 fail(
I,
"llvm.coro.id.async async function pointer not a global", V);
600 if (AsyncFuncPtrAddr->getType()->isOpaquePointerTy())
603 auto *StructTy = cast<StructType>(
604 AsyncFuncPtrAddr->getType()->getNonOpaquePointerElementType());
605 if (StructTy->isOpaque() || !StructTy->isPacked() ||
606 StructTy->getNumElements() != 2 ||
607 !StructTy->getElementType(0)->isIntegerTy(32) ||
608 !StructTy->getElementType(1)->isIntegerTy(32))
610 "llvm.coro.id.async async function pointer argument's type is not "
617 "size argument to coro.id.async must be constant");
619 "alignment argument to coro.id.async must be constant");
621 "storage argument offset to coro.id.async must be constant");
627 auto *FunTy = cast<FunctionType>(
F->getValueType());
629 auto *RetPtrTy = dyn_cast<PointerType>(FunTy->getReturnType());
630 if (!RetPtrTy || !RetPtrTy->isOpaqueOrPointeeTypeMatches(Int8Ty))
632 "llvm.coro.suspend.async resume function projection function must "
633 "return an i8* type",
635 if (FunTy->getNumParams() != 1 || !FunTy->getParamType(0)->isPointerTy() ||
636 !cast<PointerType>(FunTy->getParamType(0))
637 ->isOpaqueOrPointeeTypeMatches(Int8Ty))
639 "llvm.coro.suspend.async resume function projection function must "
640 "take one i8* type as parameter",
649 auto *MustTailCallFunc = getMustTailCallFunction();
650 if (!MustTailCallFunc)
652 auto *FnTy = MustTailCallFunc->getFunctionType();
653 if (FnTy->getNumParams() != (arg_size() - 3))
655 "llvm.coro.end.async must tail call function argument type must "
656 "match the tail arguments",
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file contains the simple types necessary to represent the attributes associated with functions a...
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static void checkWFDealloc(const Instruction *I, Value *V)
Check that the given value is a well-formed deallocator.
static bool isCoroutineIntrinsicName(StringRef Name)
static void checkConstantInt(const Instruction *I, Value *V, const char *Reason)
static void checkWFRetconPrototype(const AnyCoroIdRetconInst *I, Value *V)
Check that the given value is a well-formed prototype for the llvm.coro.id.retcon.
static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee)
static void checkAsyncContextProjectFunction(const Instruction *I, Function *F)
static void clear(coro::Shape &Shape)
static const char *const CoroIntrinsics[]
static CoroSaveInst * createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst)
static void checkWFAlloc(const Instruction *I, Value *V)
Check that the given value is a well-formed allocator.
static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee)
static void checkAsyncFuncPointer(const Instruction *I, Value *V)
Module.h This file contains the declarations for the Module class.
print must be executed print the must be executed context for all instructions
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This represents either the llvm.coro.id.retcon or llvm.coro.id.retcon.once instruction.
void checkWellFormed() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
This class represents a no-op cast from one type to another.
void setArgOperand(unsigned i, Value *v)
The basic data container for the call graph of a Module of IR.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
static bool isBitCastable(Type *SrcTy, Type *DestTy)
Check whether a bitcast between these types is valid.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
A constant pointer value that points to null.
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
void checkWellFormed() const
This class represents the llvm.coro.begin instruction.
This represents the llvm.coro.frame instruction.
This represents the llvm.coro.free instruction.
void checkWellFormed() const
This represents the llvm.coro.id instruction.
This represents the llvm.coro.save instruction.
void checkWellFormed() const
This represents the llvm.coro.suspend instruction.
CoroSaveInst * getCoroSave() const
Class to represent function types.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
A Module instance is used to store all the information related to an LLVM module.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt8Ty(LLVMContext &C)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
bool isVoidTy() const
Return true if this is 'void'.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int lookupLLVMIntrinsicByName(ArrayRef< const char * > NameTable, StringRef Name)
Looks up Name in NameTable via binary search.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
bool declaresAnyIntrinsic(const Module &M)
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Value * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
SmallVector< CoroSizeInst *, 2 > CoroSizes
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
void buildFrom(Function &F)
CoroBeginInst * CoroBegin
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
BasicBlock * AllocaSpillBlock