LLVM  15.0.0git
Coroutines.cpp
Go to the documentation of this file.
1 //===- Coroutines.cpp -----------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the common infrastructure for Coroutine Passes.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CoroInstr.h"
14 #include "CoroInternal.h"
15 #include "llvm/ADT/SmallVector.h"
16 #include "llvm/ADT/StringRef.h"
18 #include "llvm/IR/Attributes.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/DerivedTypes.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/InstIterator.h"
23 #include "llvm/IR/Instructions.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/Intrinsics.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/IR/Type.h"
28 #include "llvm/Support/Casting.h"
31 #include <cassert>
32 #include <cstddef>
33 #include <utility>
34 
35 using namespace llvm;
36 
37 // Construct the lowerer base class and initialize its members.
39  : TheModule(M), Context(M.getContext()),
40  Int8Ptr(Type::getInt8PtrTy(Context)),
41  ResumeFnType(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
42  /*isVarArg=*/false)),
43  NullPtr(ConstantPointerNull::get(Int8Ptr)) {}
44 
45 // Creates a sequence of instructions to obtain a resume function address using
46 // llvm.coro.subfn.addr. It generates the following sequence:
47 //
48 // call i8* @llvm.coro.subfn.addr(i8* %Arg, i8 %index)
49 // bitcast i8* %2 to void(i8*)*
50 
52  Instruction *InsertPt) {
53  auto *IndexVal = ConstantInt::get(Type::getInt8Ty(Context), Index);
54  auto *Fn = Intrinsic::getDeclaration(&TheModule, Intrinsic::coro_subfn_addr);
55 
57  Index < CoroSubFnInst::IndexLast &&
58  "makeSubFnCall: Index value out of range");
59  auto *Call = CallInst::Create(Fn, {Arg, IndexVal}, "", InsertPt);
60 
61  auto *Bitcast =
62  new BitCastInst(Call, ResumeFnType->getPointerTo(), "", InsertPt);
63  return Bitcast;
64 }
65 
66 // NOTE: Must be sorted!
67 static const char *const CoroIntrinsics[] = {
68  "llvm.coro.align",
69  "llvm.coro.alloc",
70  "llvm.coro.async.context.alloc",
71  "llvm.coro.async.context.dealloc",
72  "llvm.coro.async.resume",
73  "llvm.coro.async.size.replace",
74  "llvm.coro.async.store_resume",
75  "llvm.coro.begin",
76  "llvm.coro.destroy",
77  "llvm.coro.done",
78  "llvm.coro.end",
79  "llvm.coro.end.async",
80  "llvm.coro.frame",
81  "llvm.coro.free",
82  "llvm.coro.id",
83  "llvm.coro.id.async",
84  "llvm.coro.id.retcon",
85  "llvm.coro.id.retcon.once",
86  "llvm.coro.noop",
87  "llvm.coro.prepare.async",
88  "llvm.coro.prepare.retcon",
89  "llvm.coro.promise",
90  "llvm.coro.resume",
91  "llvm.coro.save",
92  "llvm.coro.size",
93  "llvm.coro.subfn.addr",
94  "llvm.coro.suspend",
95  "llvm.coro.suspend.async",
96  "llvm.coro.suspend.retcon",
97 };
98 
99 #ifndef NDEBUG
102 }
103 #endif
104 
106  for (StringRef Name : CoroIntrinsics) {
107  assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
108  if (M.getNamedValue(Name))
109  return true;
110  }
111 
112  return false;
113 }
114 
115 // Verifies if a module has named values listed. Also, in debug mode verifies
116 // that names are intrinsic names.
118  const std::initializer_list<StringRef> List) {
119  for (StringRef Name : List) {
120  assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
121  if (M.getNamedValue(Name))
122  return true;
123  }
124 
125  return false;
126 }
127 
128 // Replace all coro.frees associated with the provided CoroId either with 'null'
129 // if Elide is true and with its frame parameter otherwise.
130 void coro::replaceCoroFree(CoroIdInst *CoroId, bool Elide) {
132  for (User *U : CoroId->users())
133  if (auto CF = dyn_cast<CoroFreeInst>(U))
134  CoroFrees.push_back(CF);
135 
136  if (CoroFrees.empty())
137  return;
138 
139  Value *Replacement =
141  : CoroFrees.front()->getFrame();
142 
143  for (CoroFreeInst *CF : CoroFrees) {
144  CF->replaceAllUsesWith(Replacement);
145  CF->eraseFromParent();
146  }
147 }
148 
149 static void clear(coro::Shape &Shape) {
150  Shape.CoroBegin = nullptr;
151  Shape.CoroEnds.clear();
152  Shape.CoroSizes.clear();
153  Shape.CoroSuspends.clear();
154 
155  Shape.FrameTy = nullptr;
156  Shape.FramePtr = nullptr;
157  Shape.AllocaSpillBlock = nullptr;
158 }
159 
161  CoroSuspendInst *SuspendInst) {
162  Module *M = SuspendInst->getModule();
163  auto *Fn = Intrinsic::getDeclaration(M, Intrinsic::coro_save);
164  auto *SaveInst =
165  cast<CoroSaveInst>(CallInst::Create(Fn, CoroBegin, "", SuspendInst));
166  assert(!SuspendInst->getCoroSave());
167  SuspendInst->setArgOperand(0, SaveInst);
168  return SaveInst;
169 }
170 
171 // Collect "interesting" coroutine intrinsics.
173  bool HasFinalSuspend = false;
174  size_t FinalSuspendIndex = 0;
175  clear(*this);
177  SmallVector<CoroSaveInst *, 2> UnusedCoroSaves;
178 
179  for (Instruction &I : instructions(F)) {
180  if (auto II = dyn_cast<IntrinsicInst>(&I)) {
181  switch (II->getIntrinsicID()) {
182  default:
183  continue;
184  case Intrinsic::coro_size:
185  CoroSizes.push_back(cast<CoroSizeInst>(II));
186  break;
187  case Intrinsic::coro_align:
188  CoroAligns.push_back(cast<CoroAlignInst>(II));
189  break;
190  case Intrinsic::coro_frame:
191  CoroFrames.push_back(cast<CoroFrameInst>(II));
192  break;
193  case Intrinsic::coro_save:
194  // After optimizations, coro_suspends using this coro_save might have
195  // been removed, remember orphaned coro_saves to remove them later.
196  if (II->use_empty())
197  UnusedCoroSaves.push_back(cast<CoroSaveInst>(II));
198  break;
199  case Intrinsic::coro_suspend_async: {
200  auto *Suspend = cast<CoroSuspendAsyncInst>(II);
201  Suspend->checkWellFormed();
202  CoroSuspends.push_back(Suspend);
203  break;
204  }
205  case Intrinsic::coro_suspend_retcon: {
206  auto Suspend = cast<CoroSuspendRetconInst>(II);
207  CoroSuspends.push_back(Suspend);
208  break;
209  }
210  case Intrinsic::coro_suspend: {
211  auto Suspend = cast<CoroSuspendInst>(II);
212  CoroSuspends.push_back(Suspend);
213  if (Suspend->isFinal()) {
214  if (HasFinalSuspend)
216  "Only one suspend point can be marked as final");
217  HasFinalSuspend = true;
218  FinalSuspendIndex = CoroSuspends.size() - 1;
219  }
220  break;
221  }
222  case Intrinsic::coro_begin: {
223  auto CB = cast<CoroBeginInst>(II);
224 
225  // Ignore coro id's that aren't pre-split.
226  auto Id = dyn_cast<CoroIdInst>(CB->getId());
227  if (Id && !Id->getInfo().isPreSplit())
228  break;
229 
230  if (CoroBegin)
232  "coroutine should have exactly one defining @llvm.coro.begin");
233  CB->addRetAttr(Attribute::NonNull);
234  CB->addRetAttr(Attribute::NoAlias);
235  CB->removeFnAttr(Attribute::NoDuplicate);
236  CoroBegin = CB;
237  break;
238  }
239  case Intrinsic::coro_end_async:
240  case Intrinsic::coro_end:
241  CoroEnds.push_back(cast<AnyCoroEndInst>(II));
242  if (auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(II)) {
243  AsyncEnd->checkWellFormed();
244  }
245  if (CoroEnds.back()->isFallthrough() && isa<CoroEndInst>(II)) {
246  // Make sure that the fallthrough coro.end is the first element in the
247  // CoroEnds vector.
248  // Note: I don't think this is neccessary anymore.
249  if (CoroEnds.size() > 1) {
250  if (CoroEnds.front()->isFallthrough())
252  "Only one coro.end can be marked as fallthrough");
253  std::swap(CoroEnds.front(), CoroEnds.back());
254  }
255  }
256  break;
257  }
258  }
259  }
260 
261  // If for some reason, we were not able to find coro.begin, bailout.
262  if (!CoroBegin) {
263  // Replace coro.frame which are supposed to be lowered to the result of
264  // coro.begin with undef.
265  auto *Undef = UndefValue::get(Type::getInt8PtrTy(F.getContext()));
266  for (CoroFrameInst *CF : CoroFrames) {
267  CF->replaceAllUsesWith(Undef);
268  CF->eraseFromParent();
269  }
270 
271  // Replace all coro.suspend with undef and remove related coro.saves if
272  // present.
273  for (AnyCoroSuspendInst *CS : CoroSuspends) {
274  CS->replaceAllUsesWith(UndefValue::get(CS->getType()));
275  CS->eraseFromParent();
276  if (auto *CoroSave = CS->getCoroSave())
277  CoroSave->eraseFromParent();
278  }
279 
280  // Replace all coro.ends with unreachable instruction.
281  for (AnyCoroEndInst *CE : CoroEnds)
283 
284  return;
285  }
286 
287  auto Id = CoroBegin->getId();
288  switch (auto IdIntrinsic = Id->getIntrinsicID()) {
289  case Intrinsic::coro_id: {
290  auto SwitchId = cast<CoroIdInst>(Id);
291  this->ABI = coro::ABI::Switch;
292  this->SwitchLowering.HasFinalSuspend = HasFinalSuspend;
293  this->SwitchLowering.ResumeSwitch = nullptr;
294  this->SwitchLowering.PromiseAlloca = SwitchId->getPromise();
295  this->SwitchLowering.ResumeEntryBlock = nullptr;
296 
297  for (auto AnySuspend : CoroSuspends) {
298  auto Suspend = dyn_cast<CoroSuspendInst>(AnySuspend);
299  if (!Suspend) {
300 #ifndef NDEBUG
301  AnySuspend->dump();
302 #endif
303  report_fatal_error("coro.id must be paired with coro.suspend");
304  }
305 
306  if (!Suspend->getCoroSave())
307  createCoroSave(CoroBegin, Suspend);
308  }
309  break;
310  }
311  case Intrinsic::coro_id_async: {
312  auto *AsyncId = cast<CoroIdAsyncInst>(Id);
313  AsyncId->checkWellFormed();
314  this->ABI = coro::ABI::Async;
315  this->AsyncLowering.Context = AsyncId->getStorage();
316  this->AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex();
317  this->AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize();
318  this->AsyncLowering.ContextAlignment =
319  AsyncId->getStorageAlignment().value();
320  this->AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer();
321  this->AsyncLowering.AsyncCC = F.getCallingConv();
322  break;
323  };
324  case Intrinsic::coro_id_retcon:
325  case Intrinsic::coro_id_retcon_once: {
326  auto ContinuationId = cast<AnyCoroIdRetconInst>(Id);
327  ContinuationId->checkWellFormed();
328  this->ABI = (IdIntrinsic == Intrinsic::coro_id_retcon
331  auto Prototype = ContinuationId->getPrototype();
332  this->RetconLowering.ResumePrototype = Prototype;
333  this->RetconLowering.Alloc = ContinuationId->getAllocFunction();
334  this->RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
335  this->RetconLowering.ReturnBlock = nullptr;
336  this->RetconLowering.IsFrameInlineInStorage = false;
337 
338  // Determine the result value types, and make sure they match up with
339  // the values passed to the suspends.
340  auto ResultTys = getRetconResultTypes();
341  auto ResumeTys = getRetconResumeTypes();
342 
343  for (auto AnySuspend : CoroSuspends) {
344  auto Suspend = dyn_cast<CoroSuspendRetconInst>(AnySuspend);
345  if (!Suspend) {
346 #ifndef NDEBUG
347  AnySuspend->dump();
348 #endif
349  report_fatal_error("coro.id.retcon.* must be paired with "
350  "coro.suspend.retcon");
351  }
352 
353  // Check that the argument types of the suspend match the results.
354  auto SI = Suspend->value_begin(), SE = Suspend->value_end();
355  auto RI = ResultTys.begin(), RE = ResultTys.end();
356  for (; SI != SE && RI != RE; ++SI, ++RI) {
357  auto SrcTy = (*SI)->getType();
358  if (SrcTy != *RI) {
359  // The optimizer likes to eliminate bitcasts leading into variadic
360  // calls, but that messes with our invariants. Re-insert the
361  // bitcast and ignore this type mismatch.
362  if (CastInst::isBitCastable(SrcTy, *RI)) {
363  auto BCI = new BitCastInst(*SI, *RI, "", Suspend);
364  SI->set(BCI);
365  continue;
366  }
367 
368 #ifndef NDEBUG
369  Suspend->dump();
370  Prototype->getFunctionType()->dump();
371 #endif
372  report_fatal_error("argument to coro.suspend.retcon does not "
373  "match corresponding prototype function result");
374  }
375  }
376  if (SI != SE || RI != RE) {
377 #ifndef NDEBUG
378  Suspend->dump();
379  Prototype->getFunctionType()->dump();
380 #endif
381  report_fatal_error("wrong number of arguments to coro.suspend.retcon");
382  }
383 
384  // Check that the result type of the suspend matches the resume types.
385  Type *SResultTy = Suspend->getType();
386  ArrayRef<Type*> SuspendResultTys;
387  if (SResultTy->isVoidTy()) {
388  // leave as empty array
389  } else if (auto SResultStructTy = dyn_cast<StructType>(SResultTy)) {
390  SuspendResultTys = SResultStructTy->elements();
391  } else {
392  // forms an ArrayRef using SResultTy, be careful
393  SuspendResultTys = SResultTy;
394  }
395  if (SuspendResultTys.size() != ResumeTys.size()) {
396 #ifndef NDEBUG
397  Suspend->dump();
398  Prototype->getFunctionType()->dump();
399 #endif
400  report_fatal_error("wrong number of results from coro.suspend.retcon");
401  }
402  for (size_t I = 0, E = ResumeTys.size(); I != E; ++I) {
403  if (SuspendResultTys[I] != ResumeTys[I]) {
404 #ifndef NDEBUG
405  Suspend->dump();
406  Prototype->getFunctionType()->dump();
407 #endif
408  report_fatal_error("result from coro.suspend.retcon does not "
409  "match corresponding prototype function param");
410  }
411  }
412  }
413  break;
414  }
415 
416  default:
417  llvm_unreachable("coro.begin is not dependent on a coro.id call");
418  }
419 
420  // The coro.free intrinsic is always lowered to the result of coro.begin.
421  for (CoroFrameInst *CF : CoroFrames) {
422  CF->replaceAllUsesWith(CoroBegin);
423  CF->eraseFromParent();
424  }
425 
426  // Move final suspend to be the last element in the CoroSuspends vector.
427  if (ABI == coro::ABI::Switch &&
428  SwitchLowering.HasFinalSuspend &&
429  FinalSuspendIndex != CoroSuspends.size() - 1)
430  std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());
431 
432  // Remove orphaned coro.saves.
433  for (CoroSaveInst *CoroSave : UnusedCoroSaves)
434  CoroSave->eraseFromParent();
435 }
436 
437 static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee) {
438  Call->setCallingConv(Callee->getCallingConv());
439  // TODO: attributes?
440 }
441 
442 static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee){
443  if (CG)
444  (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[Callee]);
445 }
446 
448  CallGraph *CG) const {
449  switch (ABI) {
450  case coro::ABI::Switch:
451  llvm_unreachable("can't allocate memory in coro switch-lowering");
452 
453  case coro::ABI::Retcon:
454  case coro::ABI::RetconOnce: {
455  auto Alloc = RetconLowering.Alloc;
456  Size = Builder.CreateIntCast(Size,
457  Alloc->getFunctionType()->getParamType(0),
458  /*is signed*/ false);
459  auto *Call = Builder.CreateCall(Alloc, Size);
460  propagateCallAttrsFromCallee(Call, Alloc);
461  addCallToCallGraph(CG, Call, Alloc);
462  return Call;
463  }
464  case coro::ABI::Async:
465  llvm_unreachable("can't allocate memory in coro async-lowering");
466  }
467  llvm_unreachable("Unknown coro::ABI enum");
468 }
469 
471  CallGraph *CG) const {
472  switch (ABI) {
473  case coro::ABI::Switch:
474  llvm_unreachable("can't allocate memory in coro switch-lowering");
475 
476  case coro::ABI::Retcon:
477  case coro::ABI::RetconOnce: {
478  auto Dealloc = RetconLowering.Dealloc;
479  Ptr = Builder.CreateBitCast(Ptr,
480  Dealloc->getFunctionType()->getParamType(0));
481  auto *Call = Builder.CreateCall(Dealloc, Ptr);
482  propagateCallAttrsFromCallee(Call, Dealloc);
483  addCallToCallGraph(CG, Call, Dealloc);
484  return;
485  }
486  case coro::ABI::Async:
487  llvm_unreachable("can't allocate memory in coro async-lowering");
488  }
489  llvm_unreachable("Unknown coro::ABI enum");
490 }
491 
492 [[noreturn]] static void fail(const Instruction *I, const char *Reason,
493  Value *V) {
494 #ifndef NDEBUG
495  I->dump();
496  if (V) {
497  errs() << " Value: ";
499  errs() << '\n';
500  }
501 #endif
502  report_fatal_error(Reason);
503 }
504 
505 /// Check that the given value is a well-formed prototype for the
506 /// llvm.coro.id.retcon.* intrinsics.
508  auto F = dyn_cast<Function>(V->stripPointerCasts());
509  if (!F)
510  fail(I, "llvm.coro.id.retcon.* prototype not a Function", V);
511 
512  auto FT = F->getFunctionType();
513 
514  if (isa<CoroIdRetconInst>(I)) {
515  bool ResultOkay;
516  if (FT->getReturnType()->isPointerTy()) {
517  ResultOkay = true;
518  } else if (auto SRetTy = dyn_cast<StructType>(FT->getReturnType())) {
519  ResultOkay = (!SRetTy->isOpaque() &&
520  SRetTy->getNumElements() > 0 &&
521  SRetTy->getElementType(0)->isPointerTy());
522  } else {
523  ResultOkay = false;
524  }
525  if (!ResultOkay)
526  fail(I, "llvm.coro.id.retcon prototype must return pointer as first "
527  "result", F);
528 
529  if (FT->getReturnType() !=
530  I->getFunction()->getFunctionType()->getReturnType())
531  fail(I, "llvm.coro.id.retcon prototype return type must be same as"
532  "current function return type", F);
533  } else {
534  // No meaningful validation to do here for llvm.coro.id.unique.once.
535  }
536 
537  if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
538  fail(I, "llvm.coro.id.retcon.* prototype must take pointer as "
539  "its first parameter", F);
540 }
541 
542 /// Check that the given value is a well-formed allocator.
543 static void checkWFAlloc(const Instruction *I, Value *V) {
544  auto F = dyn_cast<Function>(V->stripPointerCasts());
545  if (!F)
546  fail(I, "llvm.coro.* allocator not a Function", V);
547 
548  auto FT = F->getFunctionType();
549  if (!FT->getReturnType()->isPointerTy())
550  fail(I, "llvm.coro.* allocator must return a pointer", F);
551 
552  if (FT->getNumParams() != 1 ||
553  !FT->getParamType(0)->isIntegerTy())
554  fail(I, "llvm.coro.* allocator must take integer as only param", F);
555 }
556 
557 /// Check that the given value is a well-formed deallocator.
558 static void checkWFDealloc(const Instruction *I, Value *V) {
559  auto F = dyn_cast<Function>(V->stripPointerCasts());
560  if (!F)
561  fail(I, "llvm.coro.* deallocator not a Function", V);
562 
563  auto FT = F->getFunctionType();
564  if (!FT->getReturnType()->isVoidTy())
565  fail(I, "llvm.coro.* deallocator must return void", F);
566 
567  if (FT->getNumParams() != 1 ||
568  !FT->getParamType(0)->isPointerTy())
569  fail(I, "llvm.coro.* deallocator must take pointer as only param", F);
570 }
571 
572 static void checkConstantInt(const Instruction *I, Value *V,
573  const char *Reason) {
574  if (!isa<ConstantInt>(V)) {
575  fail(I, Reason, V);
576  }
577 }
578 
580  checkConstantInt(this, getArgOperand(SizeArg),
581  "size argument to coro.id.retcon.* must be constant");
582  checkConstantInt(this, getArgOperand(AlignArg),
583  "alignment argument to coro.id.retcon.* must be constant");
584  checkWFRetconPrototype(this, getArgOperand(PrototypeArg));
585  checkWFAlloc(this, getArgOperand(AllocArg));
586  checkWFDealloc(this, getArgOperand(DeallocArg));
587 }
588 
589 static void checkAsyncFuncPointer(const Instruction *I, Value *V) {
590  auto *AsyncFuncPtrAddr = dyn_cast<GlobalVariable>(V->stripPointerCasts());
591  if (!AsyncFuncPtrAddr)
592  fail(I, "llvm.coro.id.async async function pointer not a global", V);
593 
594  if (AsyncFuncPtrAddr->getType()->isOpaquePointerTy())
595  return;
596 
597  auto *StructTy = cast<StructType>(
598  AsyncFuncPtrAddr->getType()->getNonOpaquePointerElementType());
599  if (StructTy->isOpaque() || !StructTy->isPacked() ||
600  StructTy->getNumElements() != 2 ||
601  !StructTy->getElementType(0)->isIntegerTy(32) ||
602  !StructTy->getElementType(1)->isIntegerTy(32))
603  fail(I,
604  "llvm.coro.id.async async function pointer argument's type is not "
605  "<{i32, i32}>",
606  V);
607 }
608 
610  checkConstantInt(this, getArgOperand(SizeArg),
611  "size argument to coro.id.async must be constant");
612  checkConstantInt(this, getArgOperand(AlignArg),
613  "alignment argument to coro.id.async must be constant");
614  checkConstantInt(this, getArgOperand(StorageArg),
615  "storage argument offset to coro.id.async must be constant");
616  checkAsyncFuncPointer(this, getArgOperand(AsyncFuncPtrArg));
617 }
618 
620  Function *F) {
621  auto *FunTy = cast<FunctionType>(F->getValueType());
622  Type *Int8Ty = Type::getInt8Ty(F->getContext());
623  auto *RetPtrTy = dyn_cast<PointerType>(FunTy->getReturnType());
624  if (!RetPtrTy || !RetPtrTy->isOpaqueOrPointeeTypeMatches(Int8Ty))
625  fail(I,
626  "llvm.coro.suspend.async resume function projection function must "
627  "return an i8* type",
628  F);
629  if (FunTy->getNumParams() != 1 || !FunTy->getParamType(0)->isPointerTy() ||
630  !cast<PointerType>(FunTy->getParamType(0))
631  ->isOpaqueOrPointeeTypeMatches(Int8Ty))
632  fail(I,
633  "llvm.coro.suspend.async resume function projection function must "
634  "take one i8* type as parameter",
635  F);
636 }
637 
639  checkAsyncContextProjectFunction(this, getAsyncContextProjectionFunction());
640 }
641 
643  auto *MustTailCallFunc = getMustTailCallFunction();
644  if (!MustTailCallFunc)
645  return;
646  auto *FnTy = MustTailCallFunc->getFunctionType();
647  if (FnTy->getNumParams() != (arg_size() - 3))
648  fail(this,
649  "llvm.coro.end.async must tail call function argument type must "
650  "match the tail arguments",
651  MustTailCallFunc);
652 }
llvm::coro::Shape::CoroSizes
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:87
llvm::CoroIdAsyncInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:609
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:65
llvm::AnyCoroSuspendInst
Definition: CoroInstr.h:477
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1410
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:291
IntrinsicInst.h
llvm::coro::Shape::FrameTy
StructType * FrameTy
Definition: CoroInternal.h:109
InstIterator.h
llvm::AnyCoroIdRetconInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:579
llvm::Function
Definition: Function.h:60
StringRef.h
createCoroSave
static CoroSaveInst * createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst)
Definition: Coroutines.cpp:160
llvm::BitCastInst
This class represents a no-op cast from one type to another.
Definition: Instructions.h:5225
checkWFAlloc
static void checkWFAlloc(const Instruction *I, Value *V)
Check that the given value is a well-formed allocator.
Definition: Coroutines.cpp:543
llvm::CoroSubFnInst::IndexFirst
@ IndexFirst
Definition: CoroInstr.h:45
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:52
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
llvm::LegacyLegalizeActions::Bitcast
@ Bitcast
Perform the operation on a different, but equivalently sized type.
Definition: LegacyLegalizerInfo.h:54
ErrorHandling.h
llvm::IRBuilder<>
llvm::coro::ABI::Retcon
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
llvm::CoroSaveInst
This represents the llvm.coro.save instruction.
Definition: CoroInstr.h:440
Local.h
llvm::CallGraph
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:72
llvm::coro::ABI::Switch
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
llvm::coro::LowererBase::makeSubFnCall
Value * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
Definition: Coroutines.cpp:51
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::Intrinsic::lookupLLVMIntrinsicByName
int lookupLLVMIntrinsicByName(ArrayRef< const char * > NameTable, StringRef Name)
Looks up Name in NameTable via binary search.
Definition: IntrinsicInst.cpp:142
Module.h
llvm::coro::declaresIntrinsics
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:117
checkAsyncContextProjectFunction
static void checkAsyncContextProjectFunction(const Instruction *I, Function *F)
Definition: Coroutines.cpp:619
llvm::CoroFreeInst
This represents the llvm.coro.free instruction.
Definition: CoroInstr.h:404
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:893
fail
static void fail(const Instruction *I, const char *Reason, Value *V)
Definition: Coroutines.cpp:492
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:237
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::CoroSubFnInst::IndexLast
@ IndexLast
Definition: CoroInstr.h:44
llvm::coro::Shape::emitDealloc
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:470
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
checkAsyncFuncPointer
static void checkAsyncFuncPointer(const Instruction *I, Value *V)
Definition: Coroutines.cpp:589
clear
static void clear(coro::Shape &Shape)
Definition: Coroutines.cpp:149
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:186
llvm::ConstantPointerNull
A constant pointer value that points to null.
Definition: Constants.h:534
llvm::coro::Shape::AllocaSpillBlock
BasicBlock * AllocaSpillBlock
Definition: CoroInternal.h:113
Constants.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::User
Definition: User.h:44
Intrinsics.h
CoroIntrinsics
static const char *const CoroIntrinsics[]
Definition: Coroutines.cpp:67
llvm::CallInst::Create
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Definition: Instructions.h:1517
llvm::coro::declaresAnyIntrinsic
bool declaresAnyIntrinsic(const Module &M)
Definition: Coroutines.cpp:105
llvm::CoroSuspendInst
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:493
false
Definition: StackSlotColoring.cpp:141
llvm::coro::Shape::CoroSuspends
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:89
llvm::coro::Shape::CoroEnds
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:86
llvm::coro::replaceCoroFree
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:130
llvm::Instruction
Definition: Instruction.h:42
llvm::CoroSuspendAsyncInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:638
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:143
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1769
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:919
Type.h
llvm::coro::LowererBase::LowererBase
LowererBase(Module &M)
Definition: Coroutines.cpp:38
llvm::coro::Shape
Definition: CoroInternal.h:84
CoroInternal.h
checkWFRetconPrototype
static void checkWFRetconPrototype(const AnyCoroIdRetconInst *I, Value *V)
Check that the given value is a well-formed prototype for the llvm.coro.id.retcon.
Definition: Coroutines.cpp:507
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::ConstantPointerNull::get
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1755
propagateCallAttrsFromCallee
static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee)
Definition: Coroutines.cpp:437
I
#define I(x, y, z)
Definition: MD5.cpp:58
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::coro::ABI::Async
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::CoroFrameInst
This represents the llvm.coro.frame instruction.
Definition: CoroInstr.h:392
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::Value::printAsOperand
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4662
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::NVPTXISD::Prototype
@ Prototype
Definition: NVPTXISelLowering.h:46
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:991
llvm::coro::ABI
ABI
Definition: CoroInternal.h:57
llvm::coro::Shape::CoroBegin
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:85
llvm::AnyCoroEndInst
Definition: CoroInstr.h:614
llvm::CallBase::setArgOperand
void setArgOperand(unsigned i, Value *v)
Definition: InstrTypes.h:1346
llvm::Value::stripPointerCasts
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:682
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:186
Attributes.h
llvm::coro::ABI::RetconOnce
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::coro::Shape::buildFrom
void buildFrom(Function &F)
Definition: Coroutines.cpp:172
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
Casting.h
Function.h
isCoroutineIntrinsicName
static bool isCoroutineIntrinsicName(StringRef Name)
Definition: Coroutines.cpp:100
llvm::coro::Shape::emitAlloc
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:447
llvm::CoroAsyncEndInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:642
llvm::Type::dump
void dump() const
Definition: AsmWriter.cpp:4827
checkConstantInt
static void checkConstantInt(const Instruction *I, Value *V, const char *Reason)
Definition: Coroutines.cpp:572
CallGraph.h
llvm::AnyCoroIdRetconInst
This represents either the llvm.coro.id.retcon or llvm.coro.id.retcon.once instruction.
Definition: CoroInstr.h:204
llvm::changeToUnreachable
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2122
Instructions.h
llvm::CoroSuspendInst::getCoroSave
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:497
SmallVector.h
List
const NodeList & List
Definition: RDFGraph.cpp:199
checkWFDealloc
static void checkWFDealloc(const Instruction *I, Value *V)
Check that the given value is a well-formed deallocator.
Definition: Coroutines.cpp:558
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:164
llvm::CoroBeginInst
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:420
CoroInstr.h
llvm::coro::Shape::FramePtr
Value * FramePtr
Definition: CoroInternal.h:112
DerivedTypes.h
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1474
llvm::CastInst::isBitCastable
static bool isBitCastable(Type *SrcTy, Type *DestTy)
Check whether a bitcast between these types is valid.
Definition: Instructions.cpp:3400
llvm::AMDGPU::VGPRIndexMode::Id
Id
Definition: SIDefines.h:235
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
llvm::FunctionType
Class to represent function types.
Definition: DerivedTypes.h:103
addCallToCallGraph
static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee)
Definition: Coroutines.cpp:442