39 : TheModule(M),
Context(M.getContext()),
57 "makeSubFnCall: Index value out of range");
65 "llvm.coro.async.context.alloc",
66 "llvm.coro.async.context.dealloc",
67 "llvm.coro.async.resume",
68 "llvm.coro.async.size.replace",
69 "llvm.coro.async.store_resume",
74 "llvm.coro.end.async",
79 "llvm.coro.id.retcon",
80 "llvm.coro.id.retcon.once",
82 "llvm.coro.prepare.async",
83 "llvm.coro.prepare.retcon",
88 "llvm.coro.subfn.addr",
90 "llvm.coro.suspend.async",
91 "llvm.coro.suspend.retcon",
103 if (M.getNamedValue(
Name))
113 const std::initializer_list<StringRef>
List) {
116 if (M.getNamedValue(
Name))
128 if (
auto CF = dyn_cast<CoroFreeInst>(U))
131 if (CoroFrees.
empty())
137 : CoroFrees.
front()->getFrame();
140 CF->replaceAllUsesWith(Replacement);
141 CF->eraseFromParent();
169 bool HasFinalSuspend =
false;
170 bool HasUnwindCoroEnd =
false;
171 size_t FinalSuspendIndex = 0;
177 if (
auto II = dyn_cast<IntrinsicInst>(&
I)) {
178 switch (II->getIntrinsicID()) {
181 case Intrinsic::coro_size:
182 CoroSizes.
push_back(cast<CoroSizeInst>(II));
184 case Intrinsic::coro_align:
185 CoroAligns.push_back(cast<CoroAlignInst>(II));
187 case Intrinsic::coro_frame:
188 CoroFrames.
push_back(cast<CoroFrameInst>(II));
190 case Intrinsic::coro_save:
194 UnusedCoroSaves.
push_back(cast<CoroSaveInst>(II));
196 case Intrinsic::coro_suspend_async: {
197 auto *Suspend = cast<CoroSuspendAsyncInst>(II);
198 Suspend->checkWellFormed();
199 CoroSuspends.push_back(Suspend);
202 case Intrinsic::coro_suspend_retcon: {
203 auto Suspend = cast<CoroSuspendRetconInst>(II);
204 CoroSuspends.push_back(Suspend);
207 case Intrinsic::coro_suspend: {
208 auto Suspend = cast<CoroSuspendInst>(II);
209 CoroSuspends.push_back(Suspend);
210 if (Suspend->isFinal()) {
213 "Only one suspend point can be marked as final");
214 HasFinalSuspend =
true;
215 FinalSuspendIndex = CoroSuspends.size() - 1;
219 case Intrinsic::coro_begin: {
220 auto CB = cast<CoroBeginInst>(II);
223 auto Id = dyn_cast<CoroIdInst>(CB->getId());
224 if (Id && !Id->getInfo().isPreSplit())
229 "coroutine should have exactly one defining @llvm.coro.begin");
230 CB->addRetAttr(Attribute::NonNull);
231 CB->addRetAttr(Attribute::NoAlias);
232 CB->removeFnAttr(Attribute::NoDuplicate);
236 case Intrinsic::coro_end_async:
237 case Intrinsic::coro_end:
238 CoroEnds.push_back(cast<AnyCoroEndInst>(II));
239 if (
auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(II)) {
240 AsyncEnd->checkWellFormed();
243 if (CoroEnds.back()->isUnwind())
244 HasUnwindCoroEnd =
true;
246 if (CoroEnds.back()->isFallthrough() && isa<CoroEndInst>(II)) {
250 if (CoroEnds.size() > 1) {
251 if (CoroEnds.front()->isFallthrough())
253 "Only one coro.end can be marked as fallthrough");
254 std::swap(CoroEnds.front(), CoroEnds.back());
268 CF->replaceAllUsesWith(Undef);
269 CF->eraseFromParent();
276 CS->eraseFromParent();
277 if (
auto *CoroSave = CS->getCoroSave())
278 CoroSave->eraseFromParent();
288 auto Id = CoroBegin->getId();
289 switch (
auto IdIntrinsic = Id->getIntrinsicID()) {
290 case Intrinsic::coro_id: {
291 auto SwitchId = cast<CoroIdInst>(Id);
293 this->SwitchLowering.HasFinalSuspend = HasFinalSuspend;
294 this->SwitchLowering.HasUnwindCoroEnd = HasUnwindCoroEnd;
295 this->SwitchLowering.ResumeSwitch =
nullptr;
296 this->SwitchLowering.PromiseAlloca = SwitchId->getPromise();
297 this->SwitchLowering.ResumeEntryBlock =
nullptr;
299 for (
auto *AnySuspend : CoroSuspends) {
300 auto Suspend = dyn_cast<CoroSuspendInst>(AnySuspend);
308 if (!Suspend->getCoroSave())
313 case Intrinsic::coro_id_async: {
314 auto *AsyncId = cast<CoroIdAsyncInst>(Id);
315 AsyncId->checkWellFormed();
317 this->AsyncLowering.Context = AsyncId->getStorage();
318 this->AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex();
319 this->AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize();
320 this->AsyncLowering.ContextAlignment =
321 AsyncId->getStorageAlignment().value();
322 this->AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer();
323 this->AsyncLowering.AsyncCC =
F.getCallingConv();
326 case Intrinsic::coro_id_retcon:
327 case Intrinsic::coro_id_retcon_once: {
328 auto ContinuationId = cast<AnyCoroIdRetconInst>(Id);
329 ContinuationId->checkWellFormed();
330 this->
ABI = (IdIntrinsic == Intrinsic::coro_id_retcon
333 auto Prototype = ContinuationId->getPrototype();
334 this->RetconLowering.ResumePrototype = Prototype;
335 this->RetconLowering.Alloc = ContinuationId->getAllocFunction();
336 this->RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
337 this->RetconLowering.ReturnBlock =
nullptr;
338 this->RetconLowering.IsFrameInlineInStorage =
false;
342 auto ResultTys = getRetconResultTypes();
343 auto ResumeTys = getRetconResumeTypes();
345 for (
auto *AnySuspend : CoroSuspends) {
346 auto Suspend = dyn_cast<CoroSuspendRetconInst>(AnySuspend);
352 "coro.suspend.retcon");
356 auto SI = Suspend->value_begin(), SE = Suspend->value_end();
357 auto RI = ResultTys.begin(), RE = ResultTys.end();
358 for (; SI != SE && RI != RE; ++SI, ++RI) {
359 auto SrcTy = (*SI)->getType();
372 Prototype->getFunctionType()->dump();
375 "match corresponding prototype function result");
378 if (SI != SE || RI != RE) {
381 Prototype->getFunctionType()->dump();
387 Type *SResultTy = Suspend->getType();
391 }
else if (
auto SResultStructTy = dyn_cast<StructType>(SResultTy)) {
392 SuspendResultTys = SResultStructTy->elements();
395 SuspendResultTys = SResultTy;
397 if (SuspendResultTys.
size() != ResumeTys.size()) {
400 Prototype->getFunctionType()->dump();
404 for (
size_t I = 0,
E = ResumeTys.size();
I !=
E; ++
I) {
405 if (SuspendResultTys[
I] != ResumeTys[
I]) {
408 Prototype->getFunctionType()->dump();
411 "match corresponding prototype function param");
424 CF->replaceAllUsesWith(CoroBegin);
425 CF->eraseFromParent();
430 SwitchLowering.HasFinalSuspend &&
431 FinalSuspendIndex != CoroSuspends.size() - 1)
432 std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());
436 CoroSave->eraseFromParent();
440 Call->setCallingConv(Callee->getCallingConv());
446 (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[Callee]);
457 auto Alloc = RetconLowering.Alloc;
459 Alloc->getFunctionType()->getParamType(0),
480 auto Dealloc = RetconLowering.Dealloc;
482 Dealloc->getFunctionType()->getParamType(0));
499 errs() <<
" Value: ";
510 auto F = dyn_cast<Function>(V->stripPointerCasts());
512 fail(
I,
"llvm.coro.id.retcon.* prototype not a Function", V);
514 auto FT =
F->getFunctionType();
516 if (isa<CoroIdRetconInst>(
I)) {
518 if (FT->getReturnType()->isPointerTy()) {
520 }
else if (
auto SRetTy = dyn_cast<StructType>(FT->getReturnType())) {
521 ResultOkay = (!SRetTy->isOpaque() &&
522 SRetTy->getNumElements() > 0 &&
523 SRetTy->getElementType(0)->isPointerTy());
528 fail(
I,
"llvm.coro.id.retcon prototype must return pointer as first "
531 if (FT->getReturnType() !=
532 I->getFunction()->getFunctionType()->getReturnType())
533 fail(
I,
"llvm.coro.id.retcon prototype return type must be same as"
534 "current function return type",
F);
539 if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
540 fail(
I,
"llvm.coro.id.retcon.* prototype must take pointer as "
541 "its first parameter",
F);
546 auto F = dyn_cast<Function>(V->stripPointerCasts());
548 fail(
I,
"llvm.coro.* allocator not a Function", V);
550 auto FT =
F->getFunctionType();
551 if (!FT->getReturnType()->isPointerTy())
552 fail(
I,
"llvm.coro.* allocator must return a pointer",
F);
554 if (FT->getNumParams() != 1 ||
555 !FT->getParamType(0)->isIntegerTy())
556 fail(
I,
"llvm.coro.* allocator must take integer as only param",
F);
561 auto F = dyn_cast<Function>(V->stripPointerCasts());
563 fail(
I,
"llvm.coro.* deallocator not a Function", V);
565 auto FT =
F->getFunctionType();
566 if (!FT->getReturnType()->isVoidTy())
567 fail(
I,
"llvm.coro.* deallocator must return void",
F);
569 if (FT->getNumParams() != 1 ||
570 !FT->getParamType(0)->isPointerTy())
571 fail(
I,
"llvm.coro.* deallocator must take pointer as only param",
F);
575 const char *Reason) {
576 if (!isa<ConstantInt>(V)) {
583 "size argument to coro.id.retcon.* must be constant");
585 "alignment argument to coro.id.retcon.* must be constant");
592 auto *AsyncFuncPtrAddr = dyn_cast<GlobalVariable>(V->stripPointerCasts());
593 if (!AsyncFuncPtrAddr)
594 fail(
I,
"llvm.coro.id.async async function pointer not a global", V);
599 "size argument to coro.id.async must be constant");
601 "alignment argument to coro.id.async must be constant");
603 "storage argument offset to coro.id.async must be constant");
609 auto *FunTy = cast<FunctionType>(
F->getValueType());
610 if (!FunTy->getReturnType()->isPointerTy())
612 "llvm.coro.suspend.async resume function projection function must "
615 if (FunTy->getNumParams() != 1 || !FunTy->getParamType(0)->isPointerTy())
617 "llvm.coro.suspend.async resume function projection function must "
618 "take one ptr type as parameter",
627 auto *MustTailCallFunc = getMustTailCallFunction();
628 if (!MustTailCallFunc)
630 auto *FnTy = MustTailCallFunc->getFunctionType();
631 if (FnTy->getNumParams() != (arg_size() - 3))
633 "llvm.coro.end.async must tail call function argument type must "
634 "match the tail arguments",
This file contains the simple types necessary to represent the attributes associated with functions a...
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg, SDValue Val={})
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static void checkWFDealloc(const Instruction *I, Value *V)
Check that the given value is a well-formed deallocator.
static bool isCoroutineIntrinsicName(StringRef Name)
static void checkConstantInt(const Instruction *I, Value *V, const char *Reason)
static void checkWFRetconPrototype(const AnyCoroIdRetconInst *I, Value *V)
Check that the given value is a well-formed prototype for the llvm.coro.id.retcon.
static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee)
static void checkAsyncContextProjectFunction(const Instruction *I, Function *F)
static void clear(coro::Shape &Shape)
static const char *const CoroIntrinsics[]
static CoroSaveInst * createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst)
static void checkWFAlloc(const Instruction *I, Value *V)
Check that the given value is a well-formed allocator.
static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee)
static void checkAsyncFuncPointer(const Instruction *I, Value *V)
Select target instructions out of generic instructions
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This represents either the llvm.coro.id.retcon or llvm.coro.id.retcon.once instruction.
void checkWellFormed() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
This class represents a no-op cast from one type to another.
void setArgOperand(unsigned i, Value *v)
The basic data container for the call graph of a Module of IR.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
static bool isBitCastable(Type *SrcTy, Type *DestTy)
Check whether a bitcast between these types is valid.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
A constant pointer value that points to null.
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
void checkWellFormed() const
This class represents the llvm.coro.begin instruction.
This represents the llvm.coro.frame instruction.
This represents the llvm.coro.free instruction.
void checkWellFormed() const
This represents the llvm.coro.id instruction.
This represents the llvm.coro.save instruction.
void checkWellFormed() const
This represents the llvm.coro.suspend instruction.
CoroSaveInst * getCoroSave() const
Class to represent function types.
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
A Module instance is used to store all the information related to an LLVM module.
Class to represent pointers.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int lookupLLVMIntrinsicByName(ArrayRef< const char * > NameTable, StringRef Name)
Looks up Name in NameTable via binary search.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
bool declaresAnyIntrinsic(const Module &M)
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Value * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
SmallVector< CoroSizeInst *, 2 > CoroSizes
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
void buildFrom(Function &F)
CoroBeginInst * CoroBegin
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
BasicBlock * AllocaSpillBlock