58 "makeSubFnCall: Index value out of range");
62 new BitCastInst(Call, ResumeFnType->getPointerTo(),
"", InsertPt);
70 "llvm.coro.async.context.alloc",
71 "llvm.coro.async.context.dealloc",
72 "llvm.coro.async.resume",
73 "llvm.coro.async.size.replace",
74 "llvm.coro.async.store_resume",
79 "llvm.coro.end.async",
84 "llvm.coro.id.retcon",
85 "llvm.coro.id.retcon.once",
87 "llvm.coro.prepare.async",
88 "llvm.coro.prepare.retcon",
93 "llvm.coro.subfn.addr",
95 "llvm.coro.suspend.async",
96 "llvm.coro.suspend.retcon",
108 if (
M.getNamedValue(Name))
118 const std::initializer_list<StringRef> List) {
121 if (
M.getNamedValue(Name))
133 if (
auto CF = dyn_cast<CoroFreeInst>(U))
134 CoroFrees.push_back(CF);
136 if (CoroFrees.empty())
141 : CoroFrees.front()->getFrame();
144 CF->replaceAllUsesWith(Replacement);
145 CF->eraseFromParent();
173 bool HasFinalSuspend =
false;
174 bool HasUnwindCoroEnd =
false;
175 size_t FinalSuspendIndex = 0;
181 if (
auto II = dyn_cast<IntrinsicInst>(&
I)) {
182 switch (II->getIntrinsicID()) {
185 case Intrinsic::coro_size:
186 CoroSizes.push_back(cast<CoroSizeInst>(II));
188 case Intrinsic::coro_align:
189 CoroAligns.push_back(cast<CoroAlignInst>(II));
191 case Intrinsic::coro_frame:
192 CoroFrames.push_back(cast<CoroFrameInst>(II));
194 case Intrinsic::coro_save:
198 UnusedCoroSaves.push_back(cast<CoroSaveInst>(II));
200 case Intrinsic::coro_suspend_async: {
201 auto *Suspend = cast<CoroSuspendAsyncInst>(II);
202 Suspend->checkWellFormed();
203 CoroSuspends.push_back(Suspend);
206 case Intrinsic::coro_suspend_retcon: {
207 auto Suspend = cast<CoroSuspendRetconInst>(II);
208 CoroSuspends.push_back(Suspend);
211 case Intrinsic::coro_suspend: {
212 auto Suspend = cast<CoroSuspendInst>(II);
213 CoroSuspends.push_back(Suspend);
214 if (Suspend->isFinal()) {
217 "Only one suspend point can be marked as final");
218 HasFinalSuspend =
true;
219 FinalSuspendIndex = CoroSuspends.size() - 1;
223 case Intrinsic::coro_begin: {
224 auto CB = cast<CoroBeginInst>(II);
227 auto Id = dyn_cast<CoroIdInst>(CB->getId());
228 if (
Id && !
Id->getInfo().isPreSplit())
233 "coroutine should have exactly one defining @llvm.coro.begin");
234 CB->addRetAttr(Attribute::NonNull);
235 CB->addRetAttr(Attribute::NoAlias);
236 CB->removeFnAttr(Attribute::NoDuplicate);
240 case Intrinsic::coro_end_async:
241 case Intrinsic::coro_end:
242 CoroEnds.push_back(cast<AnyCoroEndInst>(II));
243 if (
auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(II)) {
244 AsyncEnd->checkWellFormed();
247 if (CoroEnds.back()->isUnwind())
248 HasUnwindCoroEnd =
true;
250 if (CoroEnds.back()->isFallthrough() && isa<CoroEndInst>(II)) {
254 if (CoroEnds.size() > 1) {
255 if (CoroEnds.front()->isFallthrough())
257 "Only one coro.end can be marked as fallthrough");
258 std::swap(CoroEnds.front(), CoroEnds.back());
272 CF->replaceAllUsesWith(
Undef);
273 CF->eraseFromParent();
280 CS->eraseFromParent();
281 if (
auto *CoroSave = CS->getCoroSave())
282 CoroSave->eraseFromParent();
292 auto Id = CoroBegin->getId();
293 switch (
auto IdIntrinsic =
Id->getIntrinsicID()) {
294 case Intrinsic::coro_id: {
295 auto SwitchId = cast<CoroIdInst>(
Id);
297 this->SwitchLowering.HasFinalSuspend = HasFinalSuspend;
298 this->SwitchLowering.HasUnwindCoroEnd = HasUnwindCoroEnd;
299 this->SwitchLowering.ResumeSwitch =
nullptr;
300 this->SwitchLowering.PromiseAlloca = SwitchId->getPromise();
301 this->SwitchLowering.ResumeEntryBlock =
nullptr;
303 for (
auto *AnySuspend : CoroSuspends) {
304 auto Suspend = dyn_cast<CoroSuspendInst>(AnySuspend);
312 if (!Suspend->getCoroSave())
317 case Intrinsic::coro_id_async: {
318 auto *AsyncId = cast<CoroIdAsyncInst>(
Id);
319 AsyncId->checkWellFormed();
321 this->AsyncLowering.Context = AsyncId->getStorage();
322 this->AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex();
323 this->AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize();
324 this->AsyncLowering.ContextAlignment =
325 AsyncId->getStorageAlignment().value();
326 this->AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer();
327 this->AsyncLowering.AsyncCC =
F.getCallingConv();
330 case Intrinsic::coro_id_retcon:
331 case Intrinsic::coro_id_retcon_once: {
332 auto ContinuationId = cast<AnyCoroIdRetconInst>(
Id);
333 ContinuationId->checkWellFormed();
334 this->
ABI = (IdIntrinsic == Intrinsic::coro_id_retcon
337 auto Prototype = ContinuationId->getPrototype();
338 this->RetconLowering.ResumePrototype =
Prototype;
339 this->RetconLowering.Alloc = ContinuationId->getAllocFunction();
340 this->RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
341 this->RetconLowering.ReturnBlock =
nullptr;
342 this->RetconLowering.IsFrameInlineInStorage =
false;
346 auto ResultTys = getRetconResultTypes();
347 auto ResumeTys = getRetconResumeTypes();
349 for (
auto *AnySuspend : CoroSuspends) {
350 auto Suspend = dyn_cast<CoroSuspendRetconInst>(AnySuspend);
356 "coro.suspend.retcon");
360 auto SI = Suspend->value_begin(), SE = Suspend->value_end();
361 auto RI = ResultTys.begin(), RE = ResultTys.end();
362 for (;
SI != SE && RI != RE; ++
SI, ++RI) {
363 auto SrcTy = (*SI)->getType();
379 "match corresponding prototype function result");
382 if (
SI != SE || RI != RE) {
391 Type *SResultTy = Suspend->getType();
395 }
else if (
auto SResultStructTy = dyn_cast<StructType>(SResultTy)) {
396 SuspendResultTys = SResultStructTy->elements();
399 SuspendResultTys = SResultTy;
401 if (SuspendResultTys.
size() != ResumeTys.size()) {
408 for (
size_t I = 0,
E = ResumeTys.size();
I !=
E; ++
I) {
409 if (SuspendResultTys[
I] != ResumeTys[
I]) {
415 "match corresponding prototype function param");
428 CF->replaceAllUsesWith(CoroBegin);
429 CF->eraseFromParent();
434 SwitchLowering.HasFinalSuspend &&
435 FinalSuspendIndex != CoroSuspends.size() - 1)
436 std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());
440 CoroSave->eraseFromParent();
444 Call->setCallingConv(
Callee->getCallingConv());
450 (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[
Callee]);
461 auto Alloc = RetconLowering.Alloc;
462 Size =
Builder.CreateIntCast(Size,
463 Alloc->getFunctionType()->getParamType(0),
465 auto *Call =
Builder.CreateCall(Alloc, Size);
484 auto Dealloc = RetconLowering.Dealloc;
486 Dealloc->getFunctionType()->getParamType(0));
487 auto *Call =
Builder.CreateCall(Dealloc,
Ptr);
503 errs() <<
" Value: ";
516 fail(
I,
"llvm.coro.id.retcon.* prototype not a Function", V);
518 auto FT =
F->getFunctionType();
520 if (isa<CoroIdRetconInst>(
I)) {
522 if (FT->getReturnType()->isPointerTy()) {
524 }
else if (
auto SRetTy = dyn_cast<StructType>(FT->getReturnType())) {
525 ResultOkay = (!SRetTy->isOpaque() &&
526 SRetTy->getNumElements() > 0 &&
527 SRetTy->getElementType(0)->isPointerTy());
532 fail(
I,
"llvm.coro.id.retcon prototype must return pointer as first "
535 if (FT->getReturnType() !=
536 I->getFunction()->getFunctionType()->getReturnType())
537 fail(
I,
"llvm.coro.id.retcon prototype return type must be same as"
538 "current function return type",
F);
543 if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
544 fail(
I,
"llvm.coro.id.retcon.* prototype must take pointer as "
545 "its first parameter",
F);
552 fail(
I,
"llvm.coro.* allocator not a Function", V);
554 auto FT =
F->getFunctionType();
555 if (!FT->getReturnType()->isPointerTy())
556 fail(
I,
"llvm.coro.* allocator must return a pointer",
F);
558 if (FT->getNumParams() != 1 ||
559 !FT->getParamType(0)->isIntegerTy())
560 fail(
I,
"llvm.coro.* allocator must take integer as only param",
F);
567 fail(
I,
"llvm.coro.* deallocator not a Function", V);
569 auto FT =
F->getFunctionType();
570 if (!FT->getReturnType()->isVoidTy())
571 fail(
I,
"llvm.coro.* deallocator must return void",
F);
573 if (FT->getNumParams() != 1 ||
574 !FT->getParamType(0)->isPointerTy())
575 fail(
I,
"llvm.coro.* deallocator must take pointer as only param",
F);
579 const char *Reason) {
580 if (!isa<ConstantInt>(V)) {
587 "size argument to coro.id.retcon.* must be constant");
589 "alignment argument to coro.id.retcon.* must be constant");
597 if (!AsyncFuncPtrAddr)
598 fail(
I,
"llvm.coro.id.async async function pointer not a global", V);
600 if (AsyncFuncPtrAddr->getType()->isOpaquePointerTy())
603 auto *StructTy = cast<StructType>(
604 AsyncFuncPtrAddr->getType()->getNonOpaquePointerElementType());
605 if (StructTy->isOpaque() || !StructTy->isPacked() ||
606 StructTy->getNumElements() != 2 ||
607 !StructTy->getElementType(0)->isIntegerTy(32) ||
608 !StructTy->getElementType(1)->isIntegerTy(32))
610 "llvm.coro.id.async async function pointer argument's type is not "
617 "size argument to coro.id.async must be constant");
619 "alignment argument to coro.id.async must be constant");
621 "storage argument offset to coro.id.async must be constant");
627 auto *FunTy = cast<FunctionType>(
F->getValueType());
629 auto *RetPtrTy = dyn_cast<PointerType>(FunTy->getReturnType());
630 if (!RetPtrTy || !RetPtrTy->isOpaqueOrPointeeTypeMatches(Int8Ty))
632 "llvm.coro.suspend.async resume function projection function must "
633 "return an i8* type",
635 if (FunTy->getNumParams() != 1 || !FunTy->getParamType(0)->isPointerTy() ||
636 !cast<PointerType>(FunTy->getParamType(0))
637 ->isOpaqueOrPointeeTypeMatches(Int8Ty))
639 "llvm.coro.suspend.async resume function projection function must "
640 "take one i8* type as parameter",
649 auto *MustTailCallFunc = getMustTailCallFunction();
650 if (!MustTailCallFunc)
652 auto *FnTy = MustTailCallFunc->getFunctionType();
653 if (FnTy->getNumParams() != (arg_size() - 3))
655 "llvm.coro.end.async must tail call function argument type must "
656 "match the tail arguments",