58 "makeSubFnCall: Index value out of range");
62 new BitCastInst(Call, ResumeFnType->getPointerTo(),
"", InsertPt);
70 "llvm.coro.async.context.alloc",
71 "llvm.coro.async.context.dealloc",
72 "llvm.coro.async.resume",
73 "llvm.coro.async.size.replace",
74 "llvm.coro.async.store_resume",
79 "llvm.coro.end.async",
84 "llvm.coro.id.retcon",
85 "llvm.coro.id.retcon.once",
87 "llvm.coro.prepare.async",
88 "llvm.coro.prepare.retcon",
93 "llvm.coro.subfn.addr",
95 "llvm.coro.suspend.async",
96 "llvm.coro.suspend.retcon",
108 if (
M.getNamedValue(
Name))
118 const std::initializer_list<StringRef> List) {
121 if (
M.getNamedValue(
Name))
133 if (
auto CF = dyn_cast<CoroFreeInst>(U))
134 CoroFrees.push_back(CF);
136 if (CoroFrees.empty())
141 : CoroFrees.front()->getFrame();
144 CF->replaceAllUsesWith(Replacement);
145 CF->eraseFromParent();
173 bool HasFinalSuspend =
false;
174 size_t FinalSuspendIndex = 0;
180 if (
auto II = dyn_cast<IntrinsicInst>(&
I)) {
181 switch (II->getIntrinsicID()) {
184 case Intrinsic::coro_size:
185 CoroSizes.push_back(cast<CoroSizeInst>(II));
187 case Intrinsic::coro_align:
188 CoroAligns.push_back(cast<CoroAlignInst>(II));
190 case Intrinsic::coro_frame:
191 CoroFrames.push_back(cast<CoroFrameInst>(II));
193 case Intrinsic::coro_save:
197 UnusedCoroSaves.push_back(cast<CoroSaveInst>(II));
199 case Intrinsic::coro_suspend_async: {
200 auto *Suspend = cast<CoroSuspendAsyncInst>(II);
201 Suspend->checkWellFormed();
202 CoroSuspends.push_back(Suspend);
205 case Intrinsic::coro_suspend_retcon: {
206 auto Suspend = cast<CoroSuspendRetconInst>(II);
207 CoroSuspends.push_back(Suspend);
210 case Intrinsic::coro_suspend: {
211 auto Suspend = cast<CoroSuspendInst>(II);
212 CoroSuspends.push_back(Suspend);
213 if (Suspend->isFinal()) {
216 "Only one suspend point can be marked as final");
217 HasFinalSuspend =
true;
218 FinalSuspendIndex = CoroSuspends.size() - 1;
222 case Intrinsic::coro_begin: {
223 auto CB = cast<CoroBeginInst>(II);
226 auto Id = dyn_cast<CoroIdInst>(CB->getId());
227 if (
Id && !
Id->getInfo().isPreSplit())
232 "coroutine should have exactly one defining @llvm.coro.begin");
233 CB->addRetAttr(Attribute::NonNull);
234 CB->addRetAttr(Attribute::NoAlias);
235 CB->removeFnAttr(Attribute::NoDuplicate);
239 case Intrinsic::coro_end_async:
240 case Intrinsic::coro_end:
241 CoroEnds.push_back(cast<AnyCoroEndInst>(II));
242 if (
auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(II)) {
243 AsyncEnd->checkWellFormed();
245 if (CoroEnds.back()->isFallthrough() && isa<CoroEndInst>(II)) {
249 if (CoroEnds.size() > 1) {
250 if (CoroEnds.front()->isFallthrough())
252 "Only one coro.end can be marked as fallthrough");
253 std::swap(CoroEnds.front(), CoroEnds.back());
267 CF->replaceAllUsesWith(
Undef);
268 CF->eraseFromParent();
275 CS->eraseFromParent();
276 if (
auto *CoroSave = CS->getCoroSave())
277 CoroSave->eraseFromParent();
287 auto Id = CoroBegin->getId();
288 switch (
auto IdIntrinsic =
Id->getIntrinsicID()) {
289 case Intrinsic::coro_id: {
290 auto SwitchId = cast<CoroIdInst>(
Id);
292 this->SwitchLowering.HasFinalSuspend = HasFinalSuspend;
293 this->SwitchLowering.ResumeSwitch =
nullptr;
294 this->SwitchLowering.PromiseAlloca = SwitchId->getPromise();
295 this->SwitchLowering.ResumeEntryBlock =
nullptr;
297 for (
auto AnySuspend : CoroSuspends) {
298 auto Suspend = dyn_cast<CoroSuspendInst>(AnySuspend);
306 if (!Suspend->getCoroSave())
311 case Intrinsic::coro_id_async: {
312 auto *AsyncId = cast<CoroIdAsyncInst>(
Id);
313 AsyncId->checkWellFormed();
315 this->AsyncLowering.Context = AsyncId->getStorage();
316 this->AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex();
317 this->AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize();
318 this->AsyncLowering.ContextAlignment =
319 AsyncId->getStorageAlignment().value();
320 this->AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer();
321 this->AsyncLowering.AsyncCC =
F.getCallingConv();
324 case Intrinsic::coro_id_retcon:
325 case Intrinsic::coro_id_retcon_once: {
326 auto ContinuationId = cast<AnyCoroIdRetconInst>(
Id);
327 ContinuationId->checkWellFormed();
328 this->
ABI = (IdIntrinsic == Intrinsic::coro_id_retcon
331 auto Prototype = ContinuationId->getPrototype();
332 this->RetconLowering.ResumePrototype =
Prototype;
333 this->RetconLowering.Alloc = ContinuationId->getAllocFunction();
334 this->RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
335 this->RetconLowering.ReturnBlock =
nullptr;
336 this->RetconLowering.IsFrameInlineInStorage =
false;
340 auto ResultTys = getRetconResultTypes();
341 auto ResumeTys = getRetconResumeTypes();
343 for (
auto AnySuspend : CoroSuspends) {
344 auto Suspend = dyn_cast<CoroSuspendRetconInst>(AnySuspend);
350 "coro.suspend.retcon");
354 auto SI = Suspend->value_begin(), SE = Suspend->value_end();
355 auto RI = ResultTys.begin(), RE = ResultTys.end();
356 for (;
SI != SE && RI != RE; ++
SI, ++RI) {
357 auto SrcTy = (*SI)->getType();
373 "match corresponding prototype function result");
376 if (
SI != SE || RI != RE) {
385 Type *SResultTy = Suspend->getType();
389 }
else if (
auto SResultStructTy = dyn_cast<StructType>(SResultTy)) {
390 SuspendResultTys = SResultStructTy->elements();
393 SuspendResultTys = SResultTy;
395 if (SuspendResultTys.
size() != ResumeTys.size()) {
402 for (
size_t I = 0,
E = ResumeTys.size();
I !=
E; ++
I) {
403 if (SuspendResultTys[
I] != ResumeTys[
I]) {
409 "match corresponding prototype function param");
422 CF->replaceAllUsesWith(CoroBegin);
423 CF->eraseFromParent();
428 SwitchLowering.HasFinalSuspend &&
429 FinalSuspendIndex != CoroSuspends.size() - 1)
430 std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());
434 CoroSave->eraseFromParent();
438 Call->setCallingConv(
Callee->getCallingConv());
444 (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[
Callee]);
455 auto Alloc = RetconLowering.Alloc;
456 Size =
Builder.CreateIntCast(Size,
457 Alloc->getFunctionType()->getParamType(0),
459 auto *Call =
Builder.CreateCall(Alloc, Size);
478 auto Dealloc = RetconLowering.Dealloc;
479 Ptr =
Builder.CreateBitCast(Ptr,
480 Dealloc->getFunctionType()->getParamType(0));
481 auto *Call =
Builder.CreateCall(Dealloc, Ptr);
497 errs() <<
" Value: ";
510 fail(
I,
"llvm.coro.id.retcon.* prototype not a Function", V);
512 auto FT =
F->getFunctionType();
514 if (isa<CoroIdRetconInst>(
I)) {
516 if (FT->getReturnType()->isPointerTy()) {
518 }
else if (
auto SRetTy = dyn_cast<StructType>(FT->getReturnType())) {
519 ResultOkay = (!SRetTy->isOpaque() &&
520 SRetTy->getNumElements() > 0 &&
521 SRetTy->getElementType(0)->isPointerTy());
526 fail(
I,
"llvm.coro.id.retcon prototype must return pointer as first "
529 if (FT->getReturnType() !=
530 I->getFunction()->getFunctionType()->getReturnType())
531 fail(
I,
"llvm.coro.id.retcon prototype return type must be same as"
532 "current function return type",
F);
537 if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
538 fail(
I,
"llvm.coro.id.retcon.* prototype must take pointer as "
539 "its first parameter",
F);
546 fail(
I,
"llvm.coro.* allocator not a Function", V);
548 auto FT =
F->getFunctionType();
549 if (!FT->getReturnType()->isPointerTy())
550 fail(
I,
"llvm.coro.* allocator must return a pointer",
F);
552 if (FT->getNumParams() != 1 ||
553 !FT->getParamType(0)->isIntegerTy())
554 fail(
I,
"llvm.coro.* allocator must take integer as only param",
F);
561 fail(
I,
"llvm.coro.* deallocator not a Function", V);
563 auto FT =
F->getFunctionType();
564 if (!FT->getReturnType()->isVoidTy())
565 fail(
I,
"llvm.coro.* deallocator must return void",
F);
567 if (FT->getNumParams() != 1 ||
568 !FT->getParamType(0)->isPointerTy())
569 fail(
I,
"llvm.coro.* deallocator must take pointer as only param",
F);
573 const char *Reason) {
574 if (!isa<ConstantInt>(V)) {
581 "size argument to coro.id.retcon.* must be constant");
583 "alignment argument to coro.id.retcon.* must be constant");
591 if (!AsyncFuncPtrAddr)
592 fail(
I,
"llvm.coro.id.async async function pointer not a global", V);
594 if (AsyncFuncPtrAddr->getType()->isOpaquePointerTy())
597 auto *StructTy = cast<StructType>(
598 AsyncFuncPtrAddr->getType()->getNonOpaquePointerElementType());
599 if (StructTy->isOpaque() || !StructTy->isPacked() ||
600 StructTy->getNumElements() != 2 ||
601 !StructTy->getElementType(0)->isIntegerTy(32) ||
602 !StructTy->getElementType(1)->isIntegerTy(32))
604 "llvm.coro.id.async async function pointer argument's type is not "
611 "size argument to coro.id.async must be constant");
613 "alignment argument to coro.id.async must be constant");
615 "storage argument offset to coro.id.async must be constant");
621 auto *FunTy = cast<FunctionType>(
F->getValueType());
623 auto *RetPtrTy = dyn_cast<PointerType>(FunTy->getReturnType());
624 if (!RetPtrTy || !RetPtrTy->isOpaqueOrPointeeTypeMatches(Int8Ty))
626 "llvm.coro.suspend.async resume function projection function must "
627 "return an i8* type",
629 if (FunTy->getNumParams() != 1 || !FunTy->getParamType(0)->isPointerTy() ||
630 !cast<PointerType>(FunTy->getParamType(0))
631 ->isOpaqueOrPointeeTypeMatches(Int8Ty))
633 "llvm.coro.suspend.async resume function projection function must "
634 "take one i8* type as parameter",
643 auto *MustTailCallFunc = getMustTailCallFunction();
644 if (!MustTailCallFunc)
646 auto *FnTy = MustTailCallFunc->getFunctionType();
647 if (FnTy->getNumParams() != (arg_size() - 3))
649 "llvm.coro.end.async must tail call function argument type must "
650 "match the tail arguments",