LLVM  13.0.0git
Coroutines.cpp
Go to the documentation of this file.
1 //===- Coroutines.cpp -----------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the common infrastructure for Coroutine Passes.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "CoroInstr.h"
15 #include "CoroInternal.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
21 #include "llvm/IR/Attributes.h"
22 #include "llvm/IR/Constants.h"
23 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/IR/Function.h"
25 #include "llvm/IR/InstIterator.h"
26 #include "llvm/IR/Instructions.h"
27 #include "llvm/IR/IntrinsicInst.h"
28 #include "llvm/IR/Intrinsics.h"
30 #include "llvm/IR/Module.h"
31 #include "llvm/IR/Type.h"
32 #include "llvm/InitializePasses.h"
33 #include "llvm/Support/Casting.h"
35 #include "llvm/Transforms/IPO.h"
38 #include <cassert>
39 #include <cstddef>
40 #include <utility>
41 
42 using namespace llvm;
43 
49 }
50 
55 
58 }
59 
63 }
64 
68 }
69 
72  PM.add(createCoroSplitLegacyPass(Builder.OptLevel != 0));
73 }
74 
78 }
79 
91 }
92 
93 // Construct the lowerer base class and initialize its members.
95  : TheModule(M), Context(M.getContext()),
96  Int8Ptr(Type::getInt8PtrTy(Context)),
97  ResumeFnType(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
98  /*isVarArg=*/false)),
99  NullPtr(ConstantPointerNull::get(Int8Ptr)) {}
100 
101 // Creates a sequence of instructions to obtain a resume function address using
102 // llvm.coro.subfn.addr. It generates the following sequence:
103 //
104 // call i8* @llvm.coro.subfn.addr(i8* %Arg, i8 %index)
105 // bitcast i8* %2 to void(i8*)*
106 
108  Instruction *InsertPt) {
109  auto *IndexVal = ConstantInt::get(Type::getInt8Ty(Context), Index);
110  auto *Fn = Intrinsic::getDeclaration(&TheModule, Intrinsic::coro_subfn_addr);
111 
114  "makeSubFnCall: Index value out of range");
115  auto *Call = CallInst::Create(Fn, {Arg, IndexVal}, "", InsertPt);
116 
117  auto *Bitcast =
118  new BitCastInst(Call, ResumeFnType->getPointerTo(), "", InsertPt);
119  return Bitcast;
120 }
121 
122 #ifndef NDEBUG
124  // NOTE: Must be sorted!
125  static const char *const CoroIntrinsics[] = {
126  "llvm.coro.alloc",
127  "llvm.coro.async.context.alloc",
128  "llvm.coro.async.context.dealloc",
129  "llvm.coro.async.size.replace",
130  "llvm.coro.async.store_resume",
131  "llvm.coro.begin",
132  "llvm.coro.destroy",
133  "llvm.coro.done",
134  "llvm.coro.end",
135  "llvm.coro.end.async",
136  "llvm.coro.frame",
137  "llvm.coro.free",
138  "llvm.coro.id",
139  "llvm.coro.id.async",
140  "llvm.coro.id.retcon",
141  "llvm.coro.id.retcon.once",
142  "llvm.coro.noop",
143  "llvm.coro.param",
144  "llvm.coro.prepare.async",
145  "llvm.coro.prepare.retcon",
146  "llvm.coro.promise",
147  "llvm.coro.resume",
148  "llvm.coro.save",
149  "llvm.coro.size",
150  "llvm.coro.subfn.addr",
151  "llvm.coro.suspend",
152  "llvm.coro.suspend.async",
153  "llvm.coro.suspend.retcon",
154  };
155  return Intrinsic::lookupLLVMIntrinsicByName(CoroIntrinsics, Name) != -1;
156 }
157 #endif
158 
159 // Verifies if a module has named values listed. Also, in debug mode verifies
160 // that names are intrinsic names.
162  const std::initializer_list<StringRef> List) {
163  for (StringRef Name : List) {
164  assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
165  if (M.getNamedValue(Name))
166  return true;
167  }
168 
169  return false;
170 }
171 
172 // Replace all coro.frees associated with the provided CoroId either with 'null'
173 // if Elide is true and with its frame parameter otherwise.
174 void coro::replaceCoroFree(CoroIdInst *CoroId, bool Elide) {
176  for (User *U : CoroId->users())
177  if (auto CF = dyn_cast<CoroFreeInst>(U))
178  CoroFrees.push_back(CF);
179 
180  if (CoroFrees.empty())
181  return;
182 
183  Value *Replacement =
185  : CoroFrees.front()->getFrame();
186 
187  for (CoroFreeInst *CF : CoroFrees) {
188  CF->replaceAllUsesWith(Replacement);
189  CF->eraseFromParent();
190  }
191 }
192 
193 // FIXME: This code is stolen from CallGraph::addToCallGraph(Function *F), which
194 // happens to be private. It is better for this functionality exposed by the
195 // CallGraph.
196 static void buildCGN(CallGraph &CG, CallGraphNode *Node) {
197  Function *F = Node->getFunction();
198 
199  // Look for calls by this function.
200  for (Instruction &I : instructions(F))
201  if (auto *Call = dyn_cast<CallBase>(&I)) {
202  const Function *Callee = Call->getCalledFunction();
203  if (!Callee || !Intrinsic::isLeaf(Callee->getIntrinsicID()))
204  // Indirect calls of intrinsics are not allowed so no need to check.
205  // We can be more precise here by using TargetArg returned by
206  // Intrinsic::isLeaf.
207  Node->addCalledFunction(Call, CG.getCallsExternalNode());
208  else if (!Callee->isIntrinsic())
209  Node->addCalledFunction(Call, CG.getOrInsertFunction(Callee));
210  }
211 }
212 
213 // Rebuild CGN after we extracted parts of the code from ParentFunc into
214 // NewFuncs. Builds CGNs for the NewFuncs and adds them to the current SCC.
216  CallGraph &CG, CallGraphSCC &SCC) {
217  // Rebuild CGN from scratch for the ParentFunc
218  auto *ParentNode = CG[&ParentFunc];
219  ParentNode->removeAllCalledFunctions();
220  buildCGN(CG, ParentNode);
221 
222  SmallVector<CallGraphNode *, 8> Nodes(SCC.begin(), SCC.end());
223 
224  for (Function *F : NewFuncs) {
226  Nodes.push_back(Callee);
227  buildCGN(CG, Callee);
228  }
229 
230  SCC.initialize(Nodes);
231 }
232 
233 static void clear(coro::Shape &Shape) {
234  Shape.CoroBegin = nullptr;
235  Shape.CoroEnds.clear();
236  Shape.CoroSizes.clear();
237  Shape.CoroSuspends.clear();
238 
239  Shape.FrameTy = nullptr;
240  Shape.FramePtr = nullptr;
241  Shape.AllocaSpillBlock = nullptr;
242 }
243 
245  CoroSuspendInst *SuspendInst) {
246  Module *M = SuspendInst->getModule();
247  auto *Fn = Intrinsic::getDeclaration(M, Intrinsic::coro_save);
248  auto *SaveInst =
249  cast<CoroSaveInst>(CallInst::Create(Fn, CoroBegin, "", SuspendInst));
250  assert(!SuspendInst->getCoroSave());
251  SuspendInst->setArgOperand(0, SaveInst);
252  return SaveInst;
253 }
254 
255 // Collect "interesting" coroutine intrinsics.
257  bool HasFinalSuspend = false;
258  size_t FinalSuspendIndex = 0;
259  clear(*this);
261  SmallVector<CoroSaveInst *, 2> UnusedCoroSaves;
262 
263  for (Instruction &I : instructions(F)) {
264  if (auto II = dyn_cast<IntrinsicInst>(&I)) {
265  switch (II->getIntrinsicID()) {
266  default:
267  continue;
268  case Intrinsic::coro_size:
269  CoroSizes.push_back(cast<CoroSizeInst>(II));
270  break;
271  case Intrinsic::coro_frame:
272  CoroFrames.push_back(cast<CoroFrameInst>(II));
273  break;
274  case Intrinsic::coro_save:
275  // After optimizations, coro_suspends using this coro_save might have
276  // been removed, remember orphaned coro_saves to remove them later.
277  if (II->use_empty())
278  UnusedCoroSaves.push_back(cast<CoroSaveInst>(II));
279  break;
280  case Intrinsic::coro_suspend_async: {
281  auto *Suspend = cast<CoroSuspendAsyncInst>(II);
282  Suspend->checkWellFormed();
283  CoroSuspends.push_back(Suspend);
284  break;
285  }
286  case Intrinsic::coro_suspend_retcon: {
287  auto Suspend = cast<CoroSuspendRetconInst>(II);
288  CoroSuspends.push_back(Suspend);
289  break;
290  }
291  case Intrinsic::coro_suspend: {
292  auto Suspend = cast<CoroSuspendInst>(II);
293  CoroSuspends.push_back(Suspend);
294  if (Suspend->isFinal()) {
295  if (HasFinalSuspend)
297  "Only one suspend point can be marked as final");
298  HasFinalSuspend = true;
299  FinalSuspendIndex = CoroSuspends.size() - 1;
300  }
301  break;
302  }
303  case Intrinsic::coro_begin: {
304  auto CB = cast<CoroBeginInst>(II);
305 
306  // Ignore coro id's that aren't pre-split.
307  auto Id = dyn_cast<CoroIdInst>(CB->getId());
308  if (Id && !Id->getInfo().isPreSplit())
309  break;
310 
311  if (CoroBegin)
313  "coroutine should have exactly one defining @llvm.coro.begin");
314  CB->addAttribute(AttributeList::ReturnIndex, Attribute::NonNull);
315  CB->addAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
316  CB->removeAttribute(AttributeList::FunctionIndex,
317  Attribute::NoDuplicate);
318  CoroBegin = CB;
319  break;
320  }
321  case Intrinsic::coro_end_async:
322  case Intrinsic::coro_end:
323  CoroEnds.push_back(cast<AnyCoroEndInst>(II));
324  if (auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(II)) {
325  AsyncEnd->checkWellFormed();
326  }
327  if (CoroEnds.back()->isFallthrough() && isa<CoroEndInst>(II)) {
328  // Make sure that the fallthrough coro.end is the first element in the
329  // CoroEnds vector.
330  // Note: I don't think this is neccessary anymore.
331  if (CoroEnds.size() > 1) {
332  if (CoroEnds.front()->isFallthrough())
334  "Only one coro.end can be marked as fallthrough");
335  std::swap(CoroEnds.front(), CoroEnds.back());
336  }
337  }
338  break;
339  }
340  }
341  }
342 
343  // If for some reason, we were not able to find coro.begin, bailout.
344  if (!CoroBegin) {
345  // Replace coro.frame which are supposed to be lowered to the result of
346  // coro.begin with undef.
347  auto *Undef = UndefValue::get(Type::getInt8PtrTy(F.getContext()));
348  for (CoroFrameInst *CF : CoroFrames) {
349  CF->replaceAllUsesWith(Undef);
350  CF->eraseFromParent();
351  }
352 
353  // Replace all coro.suspend with undef and remove related coro.saves if
354  // present.
355  for (AnyCoroSuspendInst *CS : CoroSuspends) {
356  CS->replaceAllUsesWith(UndefValue::get(CS->getType()));
357  CS->eraseFromParent();
358  if (auto *CoroSave = CS->getCoroSave())
359  CoroSave->eraseFromParent();
360  }
361 
362  // Replace all coro.ends with unreachable instruction.
363  for (AnyCoroEndInst *CE : CoroEnds)
364  changeToUnreachable(CE, /*UseLLVMTrap=*/false);
365 
366  return;
367  }
368 
369  auto Id = CoroBegin->getId();
370  switch (auto IdIntrinsic = Id->getIntrinsicID()) {
371  case Intrinsic::coro_id: {
372  auto SwitchId = cast<CoroIdInst>(Id);
373  this->ABI = coro::ABI::Switch;
374  this->SwitchLowering.HasFinalSuspend = HasFinalSuspend;
375  this->SwitchLowering.ResumeSwitch = nullptr;
376  this->SwitchLowering.PromiseAlloca = SwitchId->getPromise();
377  this->SwitchLowering.ResumeEntryBlock = nullptr;
378 
379  for (auto AnySuspend : CoroSuspends) {
380  auto Suspend = dyn_cast<CoroSuspendInst>(AnySuspend);
381  if (!Suspend) {
382 #ifndef NDEBUG
383  AnySuspend->dump();
384 #endif
385  report_fatal_error("coro.id must be paired with coro.suspend");
386  }
387 
388  if (!Suspend->getCoroSave())
389  createCoroSave(CoroBegin, Suspend);
390  }
391  break;
392  }
393  case Intrinsic::coro_id_async: {
394  auto *AsyncId = cast<CoroIdAsyncInst>(Id);
395  AsyncId->checkWellFormed();
396  this->ABI = coro::ABI::Async;
397  this->AsyncLowering.Context = AsyncId->getStorage();
398  this->AsyncLowering.ContextArgNo = AsyncId->getStorageArgumentIndex();
399  this->AsyncLowering.ContextHeaderSize = AsyncId->getStorageSize();
400  this->AsyncLowering.ContextAlignment =
401  AsyncId->getStorageAlignment().value();
402  this->AsyncLowering.AsyncFuncPointer = AsyncId->getAsyncFunctionPointer();
403  this->AsyncLowering.AsyncCC = F.getCallingConv();
404  break;
405  };
406  case Intrinsic::coro_id_retcon:
407  case Intrinsic::coro_id_retcon_once: {
408  auto ContinuationId = cast<AnyCoroIdRetconInst>(Id);
409  ContinuationId->checkWellFormed();
410  this->ABI = (IdIntrinsic == Intrinsic::coro_id_retcon
413  auto Prototype = ContinuationId->getPrototype();
414  this->RetconLowering.ResumePrototype = Prototype;
415  this->RetconLowering.Alloc = ContinuationId->getAllocFunction();
416  this->RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
417  this->RetconLowering.ReturnBlock = nullptr;
418  this->RetconLowering.IsFrameInlineInStorage = false;
419 
420  // Determine the result value types, and make sure they match up with
421  // the values passed to the suspends.
422  auto ResultTys = getRetconResultTypes();
423  auto ResumeTys = getRetconResumeTypes();
424 
425  for (auto AnySuspend : CoroSuspends) {
426  auto Suspend = dyn_cast<CoroSuspendRetconInst>(AnySuspend);
427  if (!Suspend) {
428 #ifndef NDEBUG
429  AnySuspend->dump();
430 #endif
431  report_fatal_error("coro.id.retcon.* must be paired with "
432  "coro.suspend.retcon");
433  }
434 
435  // Check that the argument types of the suspend match the results.
436  auto SI = Suspend->value_begin(), SE = Suspend->value_end();
437  auto RI = ResultTys.begin(), RE = ResultTys.end();
438  for (; SI != SE && RI != RE; ++SI, ++RI) {
439  auto SrcTy = (*SI)->getType();
440  if (SrcTy != *RI) {
441  // The optimizer likes to eliminate bitcasts leading into variadic
442  // calls, but that messes with our invariants. Re-insert the
443  // bitcast and ignore this type mismatch.
444  if (CastInst::isBitCastable(SrcTy, *RI)) {
445  auto BCI = new BitCastInst(*SI, *RI, "", Suspend);
446  SI->set(BCI);
447  continue;
448  }
449 
450 #ifndef NDEBUG
451  Suspend->dump();
452  Prototype->getFunctionType()->dump();
453 #endif
454  report_fatal_error("argument to coro.suspend.retcon does not "
455  "match corresponding prototype function result");
456  }
457  }
458  if (SI != SE || RI != RE) {
459 #ifndef NDEBUG
460  Suspend->dump();
461  Prototype->getFunctionType()->dump();
462 #endif
463  report_fatal_error("wrong number of arguments to coro.suspend.retcon");
464  }
465 
466  // Check that the result type of the suspend matches the resume types.
467  Type *SResultTy = Suspend->getType();
468  ArrayRef<Type*> SuspendResultTys;
469  if (SResultTy->isVoidTy()) {
470  // leave as empty array
471  } else if (auto SResultStructTy = dyn_cast<StructType>(SResultTy)) {
472  SuspendResultTys = SResultStructTy->elements();
473  } else {
474  // forms an ArrayRef using SResultTy, be careful
475  SuspendResultTys = SResultTy;
476  }
477  if (SuspendResultTys.size() != ResumeTys.size()) {
478 #ifndef NDEBUG
479  Suspend->dump();
480  Prototype->getFunctionType()->dump();
481 #endif
482  report_fatal_error("wrong number of results from coro.suspend.retcon");
483  }
484  for (size_t I = 0, E = ResumeTys.size(); I != E; ++I) {
485  if (SuspendResultTys[I] != ResumeTys[I]) {
486 #ifndef NDEBUG
487  Suspend->dump();
488  Prototype->getFunctionType()->dump();
489 #endif
490  report_fatal_error("result from coro.suspend.retcon does not "
491  "match corresponding prototype function param");
492  }
493  }
494  }
495  break;
496  }
497 
498  default:
499  llvm_unreachable("coro.begin is not dependent on a coro.id call");
500  }
501 
502  // The coro.free intrinsic is always lowered to the result of coro.begin.
503  for (CoroFrameInst *CF : CoroFrames) {
504  CF->replaceAllUsesWith(CoroBegin);
505  CF->eraseFromParent();
506  }
507 
508  // Move final suspend to be the last element in the CoroSuspends vector.
509  if (ABI == coro::ABI::Switch &&
510  SwitchLowering.HasFinalSuspend &&
511  FinalSuspendIndex != CoroSuspends.size() - 1)
512  std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());
513 
514  // Remove orphaned coro.saves.
515  for (CoroSaveInst *CoroSave : UnusedCoroSaves)
516  CoroSave->eraseFromParent();
517 }
518 
519 static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee) {
520  Call->setCallingConv(Callee->getCallingConv());
521  // TODO: attributes?
522 }
523 
524 static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee){
525  if (CG)
526  (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[Callee]);
527 }
528 
530  CallGraph *CG) const {
531  switch (ABI) {
532  case coro::ABI::Switch:
533  llvm_unreachable("can't allocate memory in coro switch-lowering");
534 
535  case coro::ABI::Retcon:
536  case coro::ABI::RetconOnce: {
537  auto Alloc = RetconLowering.Alloc;
538  Size = Builder.CreateIntCast(Size,
539  Alloc->getFunctionType()->getParamType(0),
540  /*is signed*/ false);
541  auto *Call = Builder.CreateCall(Alloc, Size);
542  propagateCallAttrsFromCallee(Call, Alloc);
543  addCallToCallGraph(CG, Call, Alloc);
544  return Call;
545  }
546  case coro::ABI::Async:
547  llvm_unreachable("can't allocate memory in coro async-lowering");
548  }
549  llvm_unreachable("Unknown coro::ABI enum");
550 }
551 
553  CallGraph *CG) const {
554  switch (ABI) {
555  case coro::ABI::Switch:
556  llvm_unreachable("can't allocate memory in coro switch-lowering");
557 
558  case coro::ABI::Retcon:
559  case coro::ABI::RetconOnce: {
560  auto Dealloc = RetconLowering.Dealloc;
561  Ptr = Builder.CreateBitCast(Ptr,
562  Dealloc->getFunctionType()->getParamType(0));
563  auto *Call = Builder.CreateCall(Dealloc, Ptr);
564  propagateCallAttrsFromCallee(Call, Dealloc);
565  addCallToCallGraph(CG, Call, Dealloc);
566  return;
567  }
568  case coro::ABI::Async:
569  llvm_unreachable("can't allocate memory in coro async-lowering");
570  }
571  llvm_unreachable("Unknown coro::ABI enum");
572 }
573 
575 static void fail(const Instruction *I, const char *Reason, Value *V) {
576 #ifndef NDEBUG
577  I->dump();
578  if (V) {
579  errs() << " Value: ";
581  errs() << '\n';
582  }
583 #endif
584  report_fatal_error(Reason);
585 }
586 
587 /// Check that the given value is a well-formed prototype for the
588 /// llvm.coro.id.retcon.* intrinsics.
590  auto F = dyn_cast<Function>(V->stripPointerCasts());
591  if (!F)
592  fail(I, "llvm.coro.id.retcon.* prototype not a Function", V);
593 
594  auto FT = F->getFunctionType();
595 
596  if (isa<CoroIdRetconInst>(I)) {
597  bool ResultOkay;
598  if (FT->getReturnType()->isPointerTy()) {
599  ResultOkay = true;
600  } else if (auto SRetTy = dyn_cast<StructType>(FT->getReturnType())) {
601  ResultOkay = (!SRetTy->isOpaque() &&
602  SRetTy->getNumElements() > 0 &&
603  SRetTy->getElementType(0)->isPointerTy());
604  } else {
605  ResultOkay = false;
606  }
607  if (!ResultOkay)
608  fail(I, "llvm.coro.id.retcon prototype must return pointer as first "
609  "result", F);
610 
611  if (FT->getReturnType() !=
612  I->getFunction()->getFunctionType()->getReturnType())
613  fail(I, "llvm.coro.id.retcon prototype return type must be same as"
614  "current function return type", F);
615  } else {
616  // No meaningful validation to do here for llvm.coro.id.unique.once.
617  }
618 
619  if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
620  fail(I, "llvm.coro.id.retcon.* prototype must take pointer as "
621  "its first parameter", F);
622 }
623 
624 /// Check that the given value is a well-formed allocator.
625 static void checkWFAlloc(const Instruction *I, Value *V) {
626  auto F = dyn_cast<Function>(V->stripPointerCasts());
627  if (!F)
628  fail(I, "llvm.coro.* allocator not a Function", V);
629 
630  auto FT = F->getFunctionType();
631  if (!FT->getReturnType()->isPointerTy())
632  fail(I, "llvm.coro.* allocator must return a pointer", F);
633 
634  if (FT->getNumParams() != 1 ||
635  !FT->getParamType(0)->isIntegerTy())
636  fail(I, "llvm.coro.* allocator must take integer as only param", F);
637 }
638 
639 /// Check that the given value is a well-formed deallocator.
640 static void checkWFDealloc(const Instruction *I, Value *V) {
641  auto F = dyn_cast<Function>(V->stripPointerCasts());
642  if (!F)
643  fail(I, "llvm.coro.* deallocator not a Function", V);
644 
645  auto FT = F->getFunctionType();
646  if (!FT->getReturnType()->isVoidTy())
647  fail(I, "llvm.coro.* deallocator must return void", F);
648 
649  if (FT->getNumParams() != 1 ||
650  !FT->getParamType(0)->isPointerTy())
651  fail(I, "llvm.coro.* deallocator must take pointer as only param", F);
652 }
653 
654 static void checkConstantInt(const Instruction *I, Value *V,
655  const char *Reason) {
656  if (!isa<ConstantInt>(V)) {
657  fail(I, Reason, V);
658  }
659 }
660 
662  checkConstantInt(this, getArgOperand(SizeArg),
663  "size argument to coro.id.retcon.* must be constant");
664  checkConstantInt(this, getArgOperand(AlignArg),
665  "alignment argument to coro.id.retcon.* must be constant");
666  checkWFRetconPrototype(this, getArgOperand(PrototypeArg));
667  checkWFAlloc(this, getArgOperand(AllocArg));
668  checkWFDealloc(this, getArgOperand(DeallocArg));
669 }
670 
671 static void checkAsyncFuncPointer(const Instruction *I, Value *V) {
672  auto *AsyncFuncPtrAddr = dyn_cast<GlobalVariable>(V->stripPointerCasts());
673  if (!AsyncFuncPtrAddr)
674  fail(I, "llvm.coro.id.async async function pointer not a global", V);
675 
676  auto *StructTy =
677  cast<StructType>(AsyncFuncPtrAddr->getType()->getPointerElementType());
678  if (StructTy->isOpaque() || !StructTy->isPacked() ||
679  StructTy->getNumElements() != 2 ||
680  !StructTy->getElementType(0)->isIntegerTy(32) ||
681  !StructTy->getElementType(1)->isIntegerTy(32))
682  fail(I,
683  "llvm.coro.id.async async function pointer argument's type is not "
684  "<{i32, i32}>",
685  V);
686 }
687 
689  checkConstantInt(this, getArgOperand(SizeArg),
690  "size argument to coro.id.async must be constant");
691  checkConstantInt(this, getArgOperand(AlignArg),
692  "alignment argument to coro.id.async must be constant");
693  checkConstantInt(this, getArgOperand(StorageArg),
694  "storage argument offset to coro.id.async must be constant");
695  checkAsyncFuncPointer(this, getArgOperand(AsyncFuncPtrArg));
696 }
697 
699  Function *F) {
700  auto *FunTy = cast<FunctionType>(F->getType()->getPointerElementType());
701  if (!FunTy->getReturnType()->isPointerTy() ||
702  !FunTy->getReturnType()->getPointerElementType()->isIntegerTy(8))
703  fail(I,
704  "llvm.coro.suspend.async resume function projection function must "
705  "return an i8* type",
706  F);
707  if (FunTy->getNumParams() != 1 || !FunTy->getParamType(0)->isPointerTy() ||
708  !FunTy->getParamType(0)->getPointerElementType()->isIntegerTy(8))
709  fail(I,
710  "llvm.coro.suspend.async resume function projection function must "
711  "take one i8* type as parameter",
712  F);
713 }
714 
716  checkAsyncContextProjectFunction(this, getAsyncContextProjectionFunction());
717 }
718 
720  auto *MustTailCallFunc = getMustTailCallFunction();
721  if (!MustTailCallFunc)
722  return;
723  auto *FnTy =
724  cast<FunctionType>(MustTailCallFunc->getType()->getPointerElementType());
725  if (FnTy->getNumParams() != (getNumArgOperands() - 3))
726  fail(this,
727  "llvm.coro.end.async must tail call function argument type must "
728  "match the tail arguments",
729  MustTailCallFunc);
730 }
731 
733  unwrap(PM)->add(createCoroEarlyLegacyPass());
734 }
735 
737  unwrap(PM)->add(createCoroSplitLegacyPass());
738 }
739 
741  unwrap(PM)->add(createCoroElideLegacyPass());
742 }
743 
746 }
747 
748 void
752 }
llvm::coro::Shape::CoroSizes
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:101
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::PassManagerBuilder::EP_OptimizerLast
@ EP_OptimizerLast
EP_OptimizerLast – This extension point allows adding passes that run after everything else.
Definition: PassManagerBuilder.h:89
llvm::CoroIdAsyncInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:688
llvm::createCoroElideLegacyPass
Pass * createCoroElideLegacyPass()
Analyze coroutines use sites, devirtualize resume/destroy calls and elide heap allocation for corouti...
Definition: CoroElide.cpp:450
llvm
Definition: AllocatorList.h:23
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:66
LLVMAddCoroElidePass
void LLVMAddCoroElidePass(LLVMPassManagerRef PM)
See llvm::createCoroElideLegacyPass function.
Definition: Coroutines.cpp:740
llvm::AnyCoroSuspendInst
Definition: CoroInstr.h:477
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1295
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:256
llvm::initializeCoroSplitLegacyPass
void initializeCoroSplitLegacyPass(PassRegistry &)
IntrinsicInst.h
llvm::coro::Shape::FrameTy
StructType * FrameTy
Definition: CoroInternal.h:122
InstIterator.h
llvm::AnyCoroIdRetconInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:661
llvm::initializeCoroutines
void initializeCoroutines(PassRegistry &)
Initialize all passes linked into the Coroutines library.
Definition: Coroutines.cpp:44
llvm::Function
Definition: Function.h:61
StringRef.h
createCoroSave
static CoroSaveInst * createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst)
Definition: Coroutines.cpp:244
llvm::BitCastInst
This class represents a no-op cast from one type to another.
Definition: Instructions.h:5136
checkWFAlloc
static void checkWFAlloc(const Instruction *I, Value *V)
Check that the given value is a well-formed allocator.
Definition: Coroutines.cpp:625
llvm::CoroSubFnInst::IndexFirst
@ IndexFirst
Definition: CoroInstr.h:45
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
ErrorHandling.h
llvm::IRBuilder<>
llvm::coro::ABI::Retcon
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
llvm::CoroSaveInst
This represents the llvm.coro.save instruction.
Definition: CoroInstr.h:440
Local.h
llvm::CallGraph
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:73
llvm::coro::ABI::Switch
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
llvm::coro::LowererBase::makeSubFnCall
Value * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
Definition: Coroutines.cpp:107
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
llvm::Intrinsic::lookupLLVMIntrinsicByName
int lookupLLVMIntrinsicByName(ArrayRef< const char * > NameTable, StringRef Name)
Looks up Name in NameTable via binary search.
Definition: IntrinsicInst.cpp:127
Module.h
llvm::coro::declaresIntrinsics
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:161
addCoroutineOpt0Passes
static void addCoroutineOpt0Passes(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Definition: Coroutines.cpp:51
checkAsyncContextProjectFunction
static void checkAsyncContextProjectFunction(const Instruction *I, Function *F)
Definition: Coroutines.cpp:698
llvm::createBarrierNoopPass
ModulePass * createBarrierNoopPass()
createBarrierNoopPass - This pass is purely a module pass barrier in a pass manager.
Definition: BarrierNoopPass.cpp:43
llvm::CoroFreeInst
This represents the llvm.coro.free instruction.
Definition: CoroInstr.h:404
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:892
LLVMAddCoroEarlyPass
void LLVMAddCoroEarlyPass(LLVMPassManagerRef PM)
See llvm::createCoroEarlyLegacyPass function.
Definition: Coroutines.cpp:732
LegacyPassManager.h
addCoroutineEarlyPasses
static void addCoroutineEarlyPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Definition: Coroutines.cpp:60
PassManagerBuilder.h
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:202
llvm::addCoroutinePassesToExtensionPoints
void addCoroutinePassesToExtensionPoints(PassManagerBuilder &Builder)
Add all coroutine passes to appropriate extension points.
Definition: Coroutines.cpp:80
llvm::initializeCoroEarlyLegacyPass
void initializeCoroEarlyLegacyPass(PassRegistry &)
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::CoroSubFnInst::IndexLast
@ IndexLast
Definition: CoroInstr.h:44
llvm::coro::Shape::emitDealloc
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:552
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
checkAsyncFuncPointer
static void checkAsyncFuncPointer(const Instruction *I, Value *V)
Definition: Coroutines.cpp:671
clear
static void clear(coro::Shape &Shape)
Definition: Coroutines.cpp:233
llvm::CallGraph::getOrInsertFunction
CallGraphNode * getOrInsertFunction(const Function *F)
Similar to operator[], but this will insert a new CallGraphNode for F if one does not already exist.
Definition: CallGraph.cpp:175
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:205
llvm::ConstantPointerNull
A constant pointer value that points to null.
Definition: Constants.h:533
llvm::coro::Shape::AllocaSpillBlock
BasicBlock * AllocaSpillBlock
Definition: CoroInternal.h:126
llvm::initializeCoroCleanupLegacyPass
void initializeCoroCleanupLegacyPass(PassRegistry &)
llvm::CallGraphSCC
CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on.
Definition: CallGraphSCCPass.h:87
Constants.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::User
Definition: User.h:44
Intrinsics.h
llvm::CallInst::Create
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Definition: Instructions.h:1493
SI
@ SI
Definition: SIInstrInfo.cpp:7342
llvm::CoroSuspendInst
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:493
false
Definition: StackSlotColoring.cpp:142
llvm::coro::Shape::CoroSuspends
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:102
llvm::coro::Shape::CoroEnds
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:100
llvm::coro::replaceCoroFree
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:174
llvm::Instruction
Definition: Instruction.h:45
llvm::CoroSuspendAsyncInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:715
llvm::PassRegistry
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
Definition: PassRegistry.h:38
llvm::unwrap
Attribute unwrap(LLVMAttributeRef Attr)
Definition: Attributes.h:244
llvm::report_fatal_error
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
fail
static LLVM_ATTRIBUTE_NORETURN void fail(const Instruction *I, const char *Reason, Value *V)
Definition: Coroutines.cpp:575
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1770
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:885
llvm::CallGraphNode
A node in the call graph for a module.
Definition: CallGraph.h:167
Type.h
llvm::legacy::PassManagerBase::add
virtual void add(Pass *P)=0
Add a pass to the queue of passes to run.
llvm::coro::LowererBase::LowererBase
LowererBase(Module &M)
Definition: Coroutines.cpp:94
llvm::PassManagerBuilder
PassManagerBuilder - This class is used to set up a standard optimization sequence for languages like...
Definition: PassManagerBuilder.h:59
llvm::coro::Shape
Definition: CoroInternal.h:98
Coroutines.h
CoroInternal.h
checkWFRetconPrototype
static void checkWFRetconPrototype(const AnyCoroIdRetconInst *I, Value *V)
Check that the given value is a well-formed prototype for the llvm.coro.id.retcon.
Definition: Coroutines.cpp:589
llvm::AttributeList::ReturnIndex
@ ReturnIndex
Definition: Attributes.h:388
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
llvm::PassManagerBuilder::EP_ScalarOptimizerLate
@ EP_ScalarOptimizerLate
EP_ScalarOptimizerLate - This extension point allows adding optimization passes after most of the mai...
Definition: PassManagerBuilder.h:85
llvm::ConstantPointerNull::get
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1756
IPO.h
llvm::changeToUnreachable
unsigned changeToUnreachable(Instruction *I, bool UseLLVMTrap, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2131
llvm::LegalizeActions::Bitcast
@ Bitcast
Perform the operation on a different, but equivalently sized type.
Definition: LegalizerInfo.h:72
propagateCallAttrsFromCallee
static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee)
Definition: Coroutines.cpp:519
I
#define I(x, y, z)
Definition: MD5.cpp:59
LLVMPassManagerBuilderRef
struct LLVMOpaquePassManagerBuilder * LLVMPassManagerBuilderRef
Definition: PassManagerBuilder.h:20
llvm::coro::Shape::FramePtr
Instruction * FramePtr
Definition: CoroInternal.h:125
llvm::Intrinsic::isLeaf
bool isLeaf(ID id)
Returns true if the intrinsic is a leaf, i.e.
Definition: Function.cpp:1278
buildCGN
static void buildCGN(CallGraph &CG, CallGraphNode *Node)
Definition: Coroutines.cpp:196
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVMPassManagerBuilderAddCoroutinePassesToExtensionPoints
void LLVMPassManagerBuilderAddCoroutinePassesToExtensionPoints(LLVMPassManagerBuilderRef PMB)
See llvm::addCoroutinePassesToExtensionPoints.
Definition: Coroutines.cpp:749
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:840
llvm::coro::ABI::Async
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::CoroFrameInst
This represents the llvm.coro.frame instruction.
Definition: CoroInstr.h:392
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
llvm::initializeCoroElideLegacyPass
void initializeCoroElideLegacyPass(PassRegistry &)
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:649
llvm::Value::printAsOperand
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4691
LLVMPassManagerRef
struct LLVMOpaquePassManager * LLVMPassManagerRef
Definition: Types.h:127
llvm::createCoroEarlyLegacyPass
Pass * createCoroEarlyLegacyPass()
Lower coroutine intrinsics that are not needed by later passes.
llvm::AMDGPU::CPol::SCC
@ SCC
Definition: SIDefines.h:285
LLVM_ATTRIBUTE_NORETURN
#define LLVM_ATTRIBUTE_NORETURN
Definition: Compiler.h:250
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:33
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
llvm::NVPTXISD::Prototype
@ Prototype
Definition: NVPTXISelLowering.h:46
Coroutines.h
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:940
llvm::coro::ABI
ABI
Definition: CoroInternal.h:71
LLVMAddCoroSplitPass
void LLVMAddCoroSplitPass(LLVMPassManagerRef PM)
See llvm::createCoroSplitLegacyPass function.
Definition: Coroutines.cpp:736
llvm::coro::Shape::CoroBegin
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:99
addCoroutineSCCPasses
static void addCoroutineSCCPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Definition: Coroutines.cpp:70
CallGraphSCCPass.h
llvm::AnyCoroEndInst
Definition: CoroInstr.h:602
llvm::CallBase::setArgOperand
void setArgOperand(unsigned i, Value *v)
Definition: InstrTypes.h:1346
llvm::Value::stripPointerCasts
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:636
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:205
llvm::createCoroCleanupLegacyPass
Pass * createCoroCleanupLegacyPass()
Lower all remaining coroutine intrinsics.
Attributes.h
llvm::coro::ABI::RetconOnce
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
llvm::coro::Shape::buildFrom
void buildFrom(Function &F)
Definition: Coroutines.cpp:256
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:53
Casting.h
Function.h
isCoroutineIntrinsicName
static bool isCoroutineIntrinsicName(StringRef Name)
Definition: Coroutines.cpp:123
llvm::coro::Shape::emitAlloc
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:529
llvm::createCoroSplitLegacyPass
Pass * createCoroSplitLegacyPass(bool IsOptimizing=false)
Split up coroutines into multiple functions driving their state machines.
Definition: CoroSplit.cpp:2249
llvm::CoroAsyncEndInst::checkWellFormed
void checkWellFormed() const
Definition: Coroutines.cpp:719
llvm::Type::dump
void dump() const
Definition: AsmWriter.cpp:4768
checkConstantInt
static void checkConstantInt(const Instruction *I, Value *V, const char *Reason)
Definition: Coroutines.cpp:654
CallGraph.h
llvm::AnyCoroIdRetconInst
This represents either the llvm.coro.id.retcon or llvm.coro.id.retcon.once instruction.
Definition: CoroInstr.h:204
llvm::AttributeList::FunctionIndex
@ FunctionIndex
Definition: Attributes.h:389
Instructions.h
llvm::CoroSuspendInst::getCoroSave
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:497
llvm::PassManagerBuilder::EP_EarlyAsPossible
@ EP_EarlyAsPossible
EP_EarlyAsPossible - This extension point allows adding passes before any other transformations,...
Definition: PassManagerBuilder.h:72
llvm::Registry
A global registry used in conjunction with static constructors to make pluggable components (like tar...
Definition: Registry.h:44
SmallVector.h
List
const NodeList & List
Definition: RDFGraph.cpp:201
checkWFDealloc
static void checkWFDealloc(const Instruction *I, Value *V)
Check that the given value is a well-formed deallocator.
Definition: Coroutines.cpp:640
llvm::coro::updateCallGraph
void updateCallGraph(Function &Caller, ArrayRef< Function * > Funcs, CallGraph &CG, CallGraphSCC &SCC)
Definition: Coroutines.cpp:215
llvm::PassManagerBuilder::EP_EnabledOnOptLevel0
@ EP_EnabledOnOptLevel0
EP_EnabledOnOptLevel0 - This extension point allows adding passes that should not be disabled by O0 o...
Definition: PassManagerBuilder.h:99
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
llvm::legacy::PassManagerBase
PassManagerBase - An abstract interface to allow code to add passes to a pass manager without having ...
Definition: LegacyPassManager.h:39
llvm::CoroBeginInst
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:420
CoroInstr.h
DerivedTypes.h
LLVMAddCoroCleanupPass
void LLVMAddCoroCleanupPass(LLVMPassManagerRef PM)
See llvm::createCoroCleanupLegacyPass function.
Definition: Coroutines.cpp:744
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1450
llvm::CastInst::isBitCastable
static bool isBitCastable(Type *SrcTy, Type *DestTy)
Check whether a bitcast between these types is valid.
Definition: Instructions.cpp:3165
addCoroutineOptimizerLastPasses
static void addCoroutineOptimizerLastPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Definition: Coroutines.cpp:75
llvm::AMDGPU::VGPRIndexMode::Id
Id
Definition: SIDefines.h:221
llvm::PassManagerBuilder::EP_CGSCCOptimizerLate
@ EP_CGSCCOptimizerLate
EP_CGSCCOptimizerLate - This extension point allows adding CallGraphSCC passes at the end of the main...
Definition: PassManagerBuilder.h:117
llvm::CallGraph::getCallsExternalNode
CallGraphNode * getCallsExternalNode() const
Definition: CallGraph.h:130
InitializePasses.h
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
addCoroutineScalarOptimizerPasses
static void addCoroutineScalarOptimizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Definition: Coroutines.cpp:65
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:434
llvm::FunctionType
Class to represent function types.
Definition: DerivedTypes.h:102
addCallToCallGraph
static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee)
Definition: Coroutines.cpp:524