LLVM  16.0.0git
Coroutines.cpp
Go to the documentation of this file.
1 //===- Coroutines.cpp -----------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the common infrastructure for Coroutine Passes.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CoroInstr.h"
14 #include "CoroInternal.h"
15 #include "llvm/ADT/SmallVector.h"
16 #include "llvm/ADT/StringRef.h"
18 #include "llvm/IR/Attributes.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/DerivedTypes.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/InstIterator.h"
23 #include "llvm/IR/Instructions.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/Intrinsics.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/IR/Type.h"
28 #include "llvm/Support/Casting.h"
31 #include <cassert>
32 #include <cstddef>
33 #include <utility>
34 
35 using namespace llvm;
36 
37 // Construct the lowerer base class and initialize its members.
39  : TheModule(M), Context(M.getContext()),
40  Int8Ptr(Type::getInt8PtrTy(Context)),
41  ResumeFnType(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
42  /*isVarArg=*/false)),
43  NullPtr(ConstantPointerNull::get(Int8Ptr)) {}
44 
45 // Creates a sequence of instructions to obtain a resume function address using
46 // llvm.coro.subfn.addr. It generates the following sequence:
47 //
48 // call i8* @llvm.coro.subfn.addr(i8* %Arg, i8 %index)
49 // bitcast i8* %2 to void(i8*)*
50 
52  Instruction *InsertPt) {
53  auto *IndexVal = ConstantInt::get(Type::getInt8Ty(Context), Index);
54  auto *Fn = Intrinsic::getDeclaration(&TheModule, Intrinsic::coro_subfn_addr);
55 
58  "makeSubFnCall: Index value out of range");
59  auto *Call = CallInst::Create(Fn, {Arg, IndexVal}, "", InsertPt);
60 
61  auto *Bitcast =
62  new BitCastInst(Call, ResumeFnType->getPointerTo(), "", InsertPt);
63  return Bitcast;
64 }
65 
66 // NOTE: Must be sorted!
67 static const char *const CoroIntrinsics[] = {
68  "llvm.coro.align",
69  "llvm.coro.alloc",
70  "llvm.coro.async.context.alloc",
71  "llvm.coro.async.context.dealloc",
72  "llvm.coro.async.resume",
73  "llvm.coro.async.size.replace",
74  "llvm.coro.async.store_resume",
75  "llvm.coro.begin",
76  "llvm.coro.destroy",
77  "llvm.coro.done",
78  "llvm.coro.end",
79  "llvm.coro.end.async",
80  "llvm.coro.frame",
81  "llvm.coro.free",
82  "llvm.coro.id",
83  "llvm.coro.id.async",
84  "llvm.coro.id.retcon",
85  "llvm.coro.id.retcon.once",
86  "llvm.coro.noop",
87  "llvm.coro.prepare.async",
88  "llvm.coro.prepare.retcon",
89  "llvm.coro.promise",
90  "llvm.coro.resume",
91  "llvm.coro.save",
92  "llvm.coro.size",
93  "llvm.coro.subfn.addr",
94  "llvm.coro.suspend",
95  "llvm.coro.suspend.async",
96  "llvm.coro.suspend.retcon",
97 };
98 
99 #ifndef NDEBUG
102 }
103 #endif
104 
106  for (StringRef Name : CoroIntrinsics) {
107  assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
108  if (M.getNamedValue(Name))
109  return true;
110  }
111 
112  return false;
113 }
114 
115 // Verifies if a module has named values listed. Also, in debug mode verifies
116 // that names are intrinsic names.
118  const std::initializer_list<StringRef> List) {
119  for (StringRef Name : List) {
120  assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
121  if (M.getNamedValue(Name))
122  return true;
123  }
124 
125  return false;
126 }
127 
128 // Replace all coro.frees associated with the provided CoroId either with 'null'
129 // if Elide is true and with its frame parameter otherwise.
130 void coro::replaceCoroFree(CoroIdInst *CoroId, bool Elide) {
132  for (User *U : CoroId->users())
133  if (auto CF = dyn_cast<CoroFreeInst>(U))
134  CoroFrees.push_back(CF);
135 
136  if (CoroFrees.empty())
137  return;
138 
139  Value *Replacement =
141  : CoroFrees.front()->getFrame();
142 
143  for (CoroFreeInst *CF : CoroFrees) {
144  CF->replaceAllUsesWith(Replacement);
145  CF->eraseFromParent();
146  }
147 }
148 
149 static void clear(coro::Shape &Shape) {
150  Shape.CoroBegin = nullptr;
151  Shape.CoroEnds.clear();
152  Shape.CoroSizes.clear();
153  Shape.CoroSuspends.clear();
154 
155  Shape.FrameTy = nullptr;
156  Shape.FramePtr = nullptr;
157  Shape.AllocaSpillBlock = nullptr;
158 }
159 
161  CoroSuspendInst *SuspendInst) {
162  Module *M = SuspendInst->getModule();
163  auto *Fn = Intrinsic::getDeclaration(M, Intrinsic::coro_save);
164  auto *SaveInst =
165  cast<CoroSaveInst>(CallInst::Create(Fn, CoroBegin, "", SuspendInst));
166  assert(!SuspendInst->getCoroSave());
167  SuspendInst->setArgOperand(0, SaveInst);
168  return SaveInst;
169 }
170 
171 // Collect "interesting" coroutine intrinsics.
173  bool HasFinalSuspend = false;
174  bool HasUnwindCoroEnd = false;
175  size_t FinalSuspendIndex = 0;
176  clear(*this);
178  SmallVector<CoroSaveInst *, 2> UnusedCoroSaves;
179 
180  for (Instruction &I : instructions(F)) {
181  if (auto II = dyn_cast<IntrinsicInst>(&I)) {
182  switch (II->getIntrinsicID()) {
183  default:
184  continue;
185  case Intrinsic::coro_size:
186  CoroSizes.push_back(cast<CoroSizeInst>(II));
187  break;
188  case Intrinsic::coro_align:
189  CoroAligns.push_back(cast<CoroAlignInst>(II));
190  break;
191  case Intrinsic::coro_frame:
192  CoroFrames.push_back(cast<CoroFrameInst>(II));
193  break;
194  case Intrinsic::coro_save:
195  // After optimizations, coro_suspends using this coro_save might have
196  // been removed, remember orphaned coro_saves to remove them later.
197  if (II->use_empty())
198  UnusedCoroSaves.push_back(cast<CoroSaveInst>(II));
199  break;
200  case Intrinsic::coro_suspend_async: {
201  auto *Suspend = cast<CoroSuspendAsyncInst>(II);
202  Suspend->checkWellFormed();
203  CoroSuspends.push_back(Suspend);
204  break;
205  }
206  case Intrinsic::coro_suspend_retcon: {
207  auto Suspend = cast<CoroSuspendRetconInst>(II);
208  CoroSuspends.push_back(Suspend);
209  break;
210  }
211  case Intrinsic::coro_suspend: {
212  auto Suspend = cast<CoroSuspendInst>(II);
213  CoroSuspends.push_back(Suspend);
214  if (Suspend->isFinal()) {
215  if (HasFinalSuspend)
217  "Only one suspend point can be marked as final");
218  HasFinalSuspend = true;
219  FinalSuspendIndex = CoroSuspends.size() - 1;
220  }
221  break;
222  }
223  case Intrinsic::coro_begin: {
224  auto CB = cast<CoroBeginInst>(II);
225 
226  // Ignore coro id's that aren't pre-split.
227  auto Id = dyn_cast<CoroIdInst>(CB->getId());
228  if (Id && !Id->getInfo().isPreSplit())
229  break;
230 
231  if (CoroBegin)
233  "coroutine should have exactly one defining @llvm.coro.begin");
234  CB->addRetAttr(Attribute::NonNull);
235  CB->addRetAttr(Attribute::NoAlias);
236  CB->removeFnAttr(Attribute::NoDuplicate);
237  CoroBegin = CB;
238  break;
239  }
240  case Intrinsic::coro_end_async:
241  case Intrinsic::coro_end:
242  CoroEnds.push_back(cast<AnyCoroEndInst>(II));
243  if (auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(II)) {
244  AsyncEnd->checkWellFormed();
245  }
246 
247  if (CoroEnds.back()->isUnwind())
248  HasUnwindCoroEnd = true;
249 
250  if (CoroEnds.back()->isFallthrough() && isa<CoroEndInst>(II)) {
251  // Make sure that the fallthrough coro.end is the first element in the
252  // CoroEnds vector.
253  // Note: I don't think this is neccessary anymore.
254  if (CoroEnds.size() > 1) {
255  if (CoroEnds.front()->isFallthrough())
257  "Only one coro.end can be marked as fallthrough");
258  std::swap(CoroEnds.front(), CoroEnds.back());
259  }
260  }
261  break;
262  }
263  }
264  }
265 
266  // If for some reason, we were not able to find coro.begin, bailout.
267  if (!CoroBegin) {
268  // Replace coro.frame which are supposed to be lowered to the result of
269  // coro.begin with undef.
270  auto *Undef = UndefValue::get(Type::getInt8PtrTy(F.getContext()));
271  for (CoroFrameInst *CF : CoroFrames) {
272  CF->replaceAllUsesWith(Undef);
273  CF->eraseFromParent();
274  }
275 
276  // Replace all coro.suspend with undef and remove related coro.saves if
277  // present.
278  for (AnyCoroSuspendInst *CS : CoroSuspends) {
279  CS->replaceAllUsesWith(UndefValue::get(CS->getType()));
280  CS->eraseFromParent();
281  if (auto *CoroSave = CS->getCoroSave())
282  CoroSave->eraseFromParent();
283  }
284 
285  // Replace all coro.ends with unreachable instruction.
286  for (AnyCoroEndInst *CE : CoroEnds)
288 
289  return;
290  }
291 
292  auto Id = CoroBegin->getId();
293  switch (auto IdIntrinsic = Id->getIntrinsicID()) {
294  case Intrinsic::coro_id: {
295  auto SwitchId = cast<CoroIdInst>(Id);
296  this->ABI = coro::ABI::Switch;
297  this->SwitchLowering.HasFinalSuspend = HasFinalSuspend;
298  this->SwitchLowering.HasUnwindCoroEnd = HasUnwindCoroEnd;
299  this->SwitchLowering.ResumeSwitch = nullptr;
300  this->SwitchLowering.PromiseAlloca = SwitchId->getPromise();
301  this->SwitchLowering.ResumeEntryBlock = nullptr;
302 
303  for (auto *AnySuspend : CoroSuspends) {
304  auto Suspend = dyn_cast<CoroSuspendInst>(AnySuspend);
305  if (!Suspend) {
306 #ifndef NDEBUG
307  AnySuspend->dump();
308 #endif
309  report_fatal_error("coro.id must be paired with coro.suspend");
310  }
311 
312  if (!Suspend->getCoroSave())
313  createCoroSave(CoroBegin, Suspend);
314  }
315  break;
316  }
317  case Intrinsic::coro_id_async: {
318  auto *AsyncId = cast<CoroIdAsyncInst>(Id);
319  AsyncId->checkWellFormed();
320  this->ABI = coro::ABI::Async;
321  this->AsyncLowering.Context = AsyncId->getStorage();
322  this->AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex();
323  this->AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize();
324  this->AsyncLowering.ContextAlignment =
325  AsyncId->getStorageAlignment().value();
326  this->AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer();
327  this->AsyncLowering.AsyncCC = F.getCallingConv();
328  break;
329  };
330  case Intrinsic::coro_id_retcon:
331  case Intrinsic::coro_id_retcon_once: {
332  auto ContinuationId = cast<AnyCoroIdRetconInst>(Id);
333  ContinuationId->checkWellFormed();
334  this->ABI = (IdIntrinsic == Intrinsic::coro_id_retcon
337  auto Prototype = ContinuationId->getPrototype();
338  this->RetconLowering.ResumePrototype = Prototype;
339  this->RetconLowering.Alloc = ContinuationId->getAllocFunction();
340  this->RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
341  this->RetconLowering.ReturnBlock = nullptr;
342  this->RetconLowering.IsFrameInlineInStorage = false;
343 
344  // Determine the result value types, and make sure they match up with
345  // the values passed to the suspends.
346  auto ResultTys = getRetconResultTypes();
347  auto ResumeTys = getRetconResumeTypes();
348 
349  for (auto *AnySuspend : CoroSuspends) {
350  auto Suspend = dyn_cast<CoroSuspendRetconInst>(AnySuspend);
351  if (!Suspend) {
352 #ifndef NDEBUG
353  AnySuspend->dump();
354 #endif
355  report_fatal_error("coro.id.retcon.* must be paired with "
356  "coro.suspend.retcon");
357  }
358 
359  // Check that the argument types of the suspend match the results.
360  auto SI = Suspend->value_begin(), SE = Suspend->value_end();
361  auto RI = ResultTys.begin(), RE = ResultTys.end();
362  for (; SI != SE && RI != RE; ++SI, ++RI) {
363  auto SrcTy = (*SI)->getType();
364  if (SrcTy != *RI) {
365  // The optimizer likes to eliminate bitcasts leading into variadic
366  // calls, but that messes with our invariants. Re-insert the
367  // bitcast and ignore this type mismatch.
368  if (CastInst::isBitCastable(SrcTy, *RI)) {
369  auto BCI = new BitCastInst(*SI, *RI, "", Suspend);
370  SI->set(BCI);
371  continue;
372  }
373 
374 #ifndef NDEBUG
375  Suspend->dump();
376  Prototype->getFunctionType()->dump();
377 #endif
378  report_fatal_error("argument to coro.suspend.retcon does not "
379  "match corresponding prototype function result");
380  }
381  }
382  if (SI != SE || RI != RE) {
383 #ifndef NDEBUG
384  Suspend->dump();
385  Prototype->getFunctionType()->dump();
386 #endif
387  report_fatal_error("wrong number of arguments to coro.suspend.retcon");
388  }
389 
390  // Check that the result type of the suspend matches the resume types.
391  Type *SResultTy = Suspend->getType();
392  ArrayRef<Type*> SuspendResultTys;
393  if (SResultTy->isVoidTy()) {
394  // leave as empty array
395  } else if (auto SResultStructTy = dyn_cast<StructType>(SResultTy)) {
396  SuspendResultTys = SResultStructTy->elements();
397  } else {
398  // forms an ArrayRef using SResultTy, be careful
399  SuspendResultTys = SResultTy;
400  }
401  if (SuspendResultTys.size() != ResumeTys.size()) {
402 #ifndef NDEBUG
403  Suspend->dump();
404  Prototype->getFunctionType()->dump();
405 #endif
406  report_fatal_error("wrong number of results from coro.suspend.retcon");
407  }
408  for (size_t I = 0, E = ResumeTys.size(); I != E; ++I) {
409  if (SuspendResultTys[I] != ResumeTys[I]) {
410 #ifndef NDEBUG
411  Suspend->dump();
412  Prototype->getFunctionType()->dump();
413 #endif
414  report_fatal_error("result from coro.suspend.retcon does not "
415  "match corresponding prototype function param");
416  }
417  }
418  }
419  break;
420  }
421 
422  default:
423  llvm_unreachable("coro.begin is not dependent on a coro.id call");
424  }
425 
426  // The coro.free intrinsic is always lowered to the result of coro.begin.
427  for (CoroFrameInst *CF : CoroFrames) {
428  CF->replaceAllUsesWith(CoroBegin);
429  CF->eraseFromParent();
430  }
431 
432  // Move final suspend to be the last element in the CoroSuspends vector.
433  if (ABI == coro::ABI::Switch &&
434  SwitchLowering.HasFinalSuspend &&
435  FinalSuspendIndex != CoroSuspends.size() - 1)
436  std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());
437 
438  // Remove orphaned coro.saves.
439  for (CoroSaveInst *CoroSave : UnusedCoroSaves)
440  CoroSave->eraseFromParent();
441 }
442 
443 static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee) {
444  Call->setCallingConv(Callee->getCallingConv());
445  // TODO: attributes?
446 }
447 
448 static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee){
449  if (CG)
450  (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[Callee]);
451 }
452 
454  CallGraph *CG) const {
455  switch (ABI) {
456  case coro::ABI::Switch:
457  llvm_unreachable("can't allocate memory in coro switch-lowering");
458 
459  case coro::ABI::Retcon:
460  case coro::ABI::RetconOnce: {
461  auto Alloc = RetconLowering.Alloc;
462  Size = Builder.CreateIntCast(Size,
463  Alloc->getFunctionType()->getParamType(0),
464  /*is signed*/ false);
465  auto *Call = Builder.CreateCall(Alloc, Size);
467  addCallToCallGraph(CG, Call, Alloc);
468  return Call;
469  }
470  case coro::ABI::Async:
471  llvm_unreachable("can't allocate memory in coro async-lowering");
472  }
473  llvm_unreachable("Unknown coro::ABI enum");
474 }
475 
477  CallGraph *CG) const {
478  switch (ABI) {
479  case coro::ABI::Switch:
480  llvm_unreachable("can't allocate memory in coro switch-lowering");
481 
482  case coro::ABI::Retcon:
483  case coro::ABI::RetconOnce: {
484  auto Dealloc = RetconLowering.Dealloc;
485  Ptr = Builder.CreateBitCast(Ptr,
486  Dealloc->getFunctionType()->getParamType(0));
487  auto *Call = Builder.CreateCall(Dealloc, Ptr);
488  propagateCallAttrsFromCallee(Call, Dealloc);
489  addCallToCallGraph(CG, Call, Dealloc);
490  return;
491  }
492  case coro::ABI::Async:
493  llvm_unreachable("can't allocate memory in coro async-lowering");
494  }
495  llvm_unreachable("Unknown coro::ABI enum");
496 }
497 
498 [[noreturn]] static void fail(const Instruction *I, const char *Reason,
499  Value *V) {
500 #ifndef NDEBUG
501  I->dump();
502  if (V) {
503  errs() << " Value: ";
505  errs() << '\n';
506  }
507 #endif
508  report_fatal_error(Reason);
509 }
510 
511 /// Check that the given value is a well-formed prototype for the
512 /// llvm.coro.id.retcon.* intrinsics.
514  auto F = dyn_cast<Function>(V->stripPointerCasts());
515  if (!F)
516  fail(I, "llvm.coro.id.retcon.* prototype not a Function", V);
517 
518  auto FT = F->getFunctionType();
519 
520  if (isa<CoroIdRetconInst>(I)) {
521  bool ResultOkay;
522  if (FT->getReturnType()->isPointerTy()) {
523  ResultOkay = true;
524  } else if (auto SRetTy = dyn_cast<StructType>(FT->getReturnType())) {
525  ResultOkay = (!SRetTy->isOpaque() &&
526  SRetTy->getNumElements() > 0 &&
527  SRetTy->getElementType(0)->isPointerTy());
528  } else {
529  ResultOkay = false;
530  }
531  if (!ResultOkay)
532  fail(I, "llvm.coro.id.retcon prototype must return pointer as first "
533  "result", F);
534 
535  if (FT->getReturnType() !=
536  I->getFunction()->getFunctionType()->getReturnType())
537  fail(I, "llvm.coro.id.retcon prototype return type must be same as"
538  "current function return type", F);
539  } else {
540  // No meaningful validation to do here for llvm.coro.id.unique.once.
541  }
542 
543  if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
544  fail(I, "llvm.coro.id.retcon.* prototype must take pointer as "
545  "its first parameter", F);
546 }
547 
548 /// Check that the given value is a well-formed allocator.
549 static void checkWFAlloc(const Instruction *I, Value *V) {
550  auto F = dyn_cast<Function>(V->stripPointerCasts());
551  if (!F)
552  fail(I, "llvm.coro.* allocator not a Function", V);
553 
554  auto FT = F->getFunctionType();
555  if (!FT->getReturnType()->isPointerTy())
556  fail(I, "llvm.coro.* allocator must return a pointer", F);
557 
558  if (FT->getNumParams() != 1 ||
559  !FT->getParamType(0)->isIntegerTy())
560  fail(I, "llvm.coro.* allocator must take integer as only param", F);
561 }
562 
563 /// Check that the given value is a well-formed deallocator.
564 static void checkWFDealloc(const Instruction *I, Value *V) {
565  auto F = dyn_cast<Function>(V->stripPointerCasts());
566  if (!F)
567  fail(I, "llvm.coro.* deallocator not a Function", V);
568 
569  auto FT = F->getFunctionType();
570  if (!FT->getReturnType()->isVoidTy())
571  fail(I, "llvm.coro.* deallocator must return void", F);
572 
573  if (FT->getNumParams() != 1 ||
574  !FT->getParamType(0)->isPointerTy())
575  fail(I, "llvm.coro.* deallocator must take pointer as only param", F);
576 }
577 
578 static void checkConstantInt(const Instruction *I, Value *V,
579  const char *Reason) {
580  if (!isa<ConstantInt>(V)) {
581  fail(I, Reason, V);
582  }
583 }
584 
586  checkConstantInt(this, getArgOperand(SizeArg),
587  "size argument to coro.id.retcon.* must be constant");
588  checkConstantInt(this, getArgOperand(AlignArg),
589  "alignment argument to coro.id.retcon.* must be constant");
590  checkWFRetconPrototype(this, getArgOperand(PrototypeArg));
591  checkWFAlloc(this, getArgOperand(AllocArg));
592  checkWFDealloc(this, getArgOperand(DeallocArg));
593 }
594 
595 static void checkAsyncFuncPointer(const Instruction *I, Value *V) {
596  auto *AsyncFuncPtrAddr = dyn_cast<GlobalVariable>(V->stripPointerCasts());
597  if (!AsyncFuncPtrAddr)
598  fail(I, "llvm.coro.id.async async function pointer not a global", V);
599 
600  if (AsyncFuncPtrAddr->getType()->isOpaquePointerTy())
601  return;
602 
603  auto *StructTy = cast<StructType>(
604  AsyncFuncPtrAddr->getType()->getNonOpaquePointerElementType());
605  if (StructTy->isOpaque() || !StructTy->isPacked() ||
606  StructTy->getNumElements() != 2 ||
607  !StructTy->getElementType(0)->isIntegerTy(32) ||
608  !StructTy->getElementType(1)->isIntegerTy(32))
609  fail(I,
610  "llvm.coro.id.async async function pointer argument's type is not "
611  "<{i32, i32}>",
612  V);
613 }
614 
616  checkConstantInt(this, getArgOperand(SizeArg),
617  "size argument to coro.id.async must be constant");
618  checkConstantInt(this, getArgOperand(AlignArg),
619  "alignment argument to coro.id.async must be constant");
620  checkConstantInt(this, getArgOperand(StorageArg),
621  "storage argument offset to coro.id.async must be constant");
622  checkAsyncFuncPointer(this, getArgOperand(AsyncFuncPtrArg));
623 }
624 
626  Function *F) {
627  auto *FunTy = cast<FunctionType>(F->getValueType());
628  Type *Int8Ty = Type::getInt8Ty(F->getContext());
629  auto *RetPtrTy = dyn_cast<PointerType>(FunTy->getReturnType());
630  if (!RetPtrTy || !RetPtrTy->isOpaqueOrPointeeTypeMatches(Int8Ty))
631  fail(I,
632  "llvm.coro.suspend.async resume function projection function must "
633  "return an i8* type",
634  F);
635  if (FunTy->getNumParams() != 1 || !FunTy->getParamType(0)->isPointerTy() ||
636  !cast<PointerType>(FunTy->getParamType(0))
637  ->isOpaqueOrPointeeTypeMatches(Int8Ty))
638  fail(I,
639  "llvm.coro.suspend.async resume function projection function must "
640  "take one i8* type as parameter",
641  F);
642 }
643 
645  checkAsyncContextProjectFunction(this, getAsyncContextProjectionFunction());
646 }
647 
649  auto *MustTailCallFunc = getMustTailCallFunction();
650  if (!MustTailCallFunc)
651  return;
652  auto *FnTy = MustTailCallFunc->getFunctionType();
653  if (FnTy->getNumParams() != (arg_size() - 3))
654  fail(this,
655  "llvm.coro.end.async must tail call function argument type must "
656  "match the tail arguments",
657  MustTailCallFunc);
658 }
llvm::coro::Shape::CoroSizes
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:76
llvm::CoroIdAsyncInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:615
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:65
llvm::AnyCoroSuspendInst
Definition: CoroInstr.h:477
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1421
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:291
IntrinsicInst.h
llvm::coro::Shape::FrameTy
StructType * FrameTy
Definition: CoroInternal.h:98
InstIterator.h
llvm::AnyCoroIdRetconInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:585
llvm::Function
Definition: Function.h:60
StringRef.h
createCoroSave
static CoroSaveInst * createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst)
Definition: Coroutines.cpp:160
llvm::BitCastInst
This class represents a no-op cast from one type to another.
Definition: Instructions.h:5256
checkWFAlloc
static void checkWFAlloc(const Instruction *I, Value *V)
Check that the given value is a well-formed allocator.
Definition: Coroutines.cpp:549
llvm::CoroSubFnInst::IndexFirst
@ IndexFirst
Definition: CoroInstr.h:45
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1182
llvm::LegacyLegalizeActions::Bitcast
@ Bitcast
Perform the operation on a different, but equivalently sized type.
Definition: LegacyLegalizerInfo.h:54
ErrorHandling.h
llvm::IRBuilder<>
llvm::coro::ABI::Retcon
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
llvm::CoroSaveInst
This represents the llvm.coro.save instruction.
Definition: CoroInstr.h:440
Local.h
llvm::CallGraph
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:72
llvm::coro::ABI::Switch
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
llvm::coro::LowererBase::makeSubFnCall
Value * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
Definition: Coroutines.cpp:51
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::Intrinsic::lookupLLVMIntrinsicByName
int lookupLLVMIntrinsicByName(ArrayRef< const char * > NameTable, StringRef Name)
Looks up Name in NameTable via binary search.
Definition: IntrinsicInst.cpp:175
Module.h
llvm::coro::declaresIntrinsics
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:117
checkAsyncContextProjectFunction
static void checkAsyncContextProjectFunction(const Instruction *I, Function *F)
Definition: Coroutines.cpp:625
llvm::CoroFreeInst
This represents the llvm.coro.free instruction.
Definition: CoroInstr.h:404
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:891
fail
static void fail(const Instruction *I, const char *Reason, Value *V)
Definition: Coroutines.cpp:498
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:237
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::CoroSubFnInst::IndexLast
@ IndexLast
Definition: CoroInstr.h:44
llvm::coro::Shape::emitDealloc
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:476
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
checkAsyncFuncPointer
static void checkAsyncFuncPointer(const Instruction *I, Value *V)
Definition: Coroutines.cpp:595
clear
static void clear(coro::Shape &Shape)
Definition: Coroutines.cpp:149
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:186
llvm::ConstantPointerNull
A constant pointer value that points to null.
Definition: Constants.h:535
llvm::coro::Shape::AllocaSpillBlock
BasicBlock * AllocaSpillBlock
Definition: CoroInternal.h:102
Constants.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::User
Definition: User.h:44
Intrinsics.h
CoroIntrinsics
static const char *const CoroIntrinsics[]
Definition: Coroutines.cpp:67
llvm::CallInst::Create
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Definition: Instructions.h:1517
llvm::coro::declaresAnyIntrinsic
bool declaresAnyIntrinsic(const Module &M)
Definition: Coroutines.cpp:105
llvm::CoroSuspendInst
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:493
false
Definition: StackSlotColoring.cpp:141
llvm::coro::Shape::CoroSuspends
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:78
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::coro::Shape::CoroEnds
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:75
llvm::coro::replaceCoroFree
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:130
llvm::Instruction
Definition: Instruction.h:42
llvm::CoroSuspendAsyncInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:644
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1708
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
Type.h
llvm::coro::LowererBase::LowererBase
LowererBase(Module &M)
Definition: Coroutines.cpp:38
llvm::coro::Shape
Definition: CoroInternal.h:73
CoroInternal.h
checkWFRetconPrototype
static void checkWFRetconPrototype(const AnyCoroIdRetconInst *I, Value *V)
Check that the given value is a well-formed prototype for the llvm.coro.id.retcon.
Definition: Coroutines.cpp:513
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
Alloc
llvm::ConstantPointerNull::get
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1694
propagateCallAttrsFromCallee
static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee)
Definition: Coroutines.cpp:443
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:52
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::coro::ABI::Async
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
Ptr
@ Ptr
Definition: TargetLibraryInfo.cpp:60
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::CoroFrameInst
This represents the llvm.coro.frame instruction.
Definition: CoroInstr.h:392
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::Value::printAsOperand
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4677
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::NVPTXISD::Prototype
@ Prototype
Definition: NVPTXISelLowering.h:46
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:994
llvm::coro::ABI
ABI
Definition: CoroInternal.h:46
llvm::coro::Shape::CoroBegin
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:74
llvm::AnyCoroEndInst
Definition: CoroInstr.h:614
llvm::CallBase::setArgOperand
void setArgOperand(unsigned i, Value *v)
Definition: InstrTypes.h:1346
llvm::Value::stripPointerCasts
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:685
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:186
Attributes.h
llvm::coro::ABI::RetconOnce
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::coro::Shape::buildFrom
void buildFrom(Function &F)
Definition: Coroutines.cpp:172
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
Casting.h
Function.h
isCoroutineIntrinsicName
static bool isCoroutineIntrinsicName(StringRef Name)
Definition: Coroutines.cpp:100
llvm::coro::Shape::emitAlloc
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:453
llvm::CoroAsyncEndInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:648
llvm::Type::dump
void dump() const
Definition: AsmWriter.cpp:4841
checkConstantInt
static void checkConstantInt(const Instruction *I, Value *V, const char *Reason)
Definition: Coroutines.cpp:578
CallGraph.h
llvm::AnyCoroIdRetconInst
This represents either the llvm.coro.id.retcon or llvm.coro.id.retcon.once instruction.
Definition: CoroInstr.h:204
llvm::changeToUnreachable
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2111
Instructions.h
llvm::CoroSuspendInst::getCoroSave
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:497
SmallVector.h
List
const NodeList & List
Definition: RDFGraph.cpp:199
checkWFDealloc
static void checkWFDealloc(const Instruction *I, Value *V)
Check that the given value is a well-formed deallocator.
Definition: Coroutines.cpp:564
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:164
llvm::CoroBeginInst
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:420
CoroInstr.h
llvm::coro::Shape::FramePtr
Value * FramePtr
Definition: CoroInternal.h:101
DerivedTypes.h
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1474
llvm::CastInst::isBitCastable
static bool isBitCastable(Type *SrcTy, Type *DestTy)
Check whether a bitcast between these types is valid.
Definition: Instructions.cpp:3463
llvm::AMDGPU::VGPRIndexMode::Id
Id
Definition: SIDefines.h:241
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
llvm::FunctionType
Class to represent function types.
Definition: DerivedTypes.h:103
addCallToCallGraph
static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee)
Definition: Coroutines.cpp:448