LLVM  13.0.0git
CoroSplit.cpp
Go to the documentation of this file.
1 //===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // This pass builds the coroutine frame and outlines resume and destroy parts
9 // of the coroutine into separate functions.
10 //
11 // We present a coroutine to an LLVM as an ordinary function with suspension
12 // points marked up with intrinsics. We let the optimizer party on the coroutine
13 // as a single function for as long as possible. Shortly before the coroutine is
14 // eligible to be inlined into its callers, we split up the coroutine into parts
15 // corresponding to an initial, resume and destroy invocations of the coroutine,
16 // add them to the current SCC and restart the IPO pipeline to optimize the
17 // coroutine subfunctions we extracted before proceeding to the caller of the
18 // coroutine.
19 //===----------------------------------------------------------------------===//
20 
22 #include "CoroInstr.h"
23 #include "CoroInternal.h"
24 #include "llvm/ADT/DenseMap.h"
25 #include "llvm/ADT/SmallPtrSet.h"
26 #include "llvm/ADT/SmallVector.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/ADT/Twine.h"
29 #include "llvm/Analysis/CFG.h"
33 #include "llvm/IR/Argument.h"
34 #include "llvm/IR/Attributes.h"
35 #include "llvm/IR/BasicBlock.h"
36 #include "llvm/IR/CFG.h"
37 #include "llvm/IR/CallingConv.h"
38 #include "llvm/IR/Constants.h"
39 #include "llvm/IR/DataLayout.h"
40 #include "llvm/IR/DerivedTypes.h"
41 #include "llvm/IR/Dominators.h"
42 #include "llvm/IR/Function.h"
43 #include "llvm/IR/GlobalValue.h"
44 #include "llvm/IR/GlobalVariable.h"
45 #include "llvm/IR/IRBuilder.h"
46 #include "llvm/IR/InstIterator.h"
47 #include "llvm/IR/InstrTypes.h"
48 #include "llvm/IR/Instruction.h"
49 #include "llvm/IR/Instructions.h"
50 #include "llvm/IR/IntrinsicInst.h"
51 #include "llvm/IR/LLVMContext.h"
53 #include "llvm/IR/Module.h"
54 #include "llvm/IR/Type.h"
55 #include "llvm/IR/Value.h"
56 #include "llvm/IR/Verifier.h"
57 #include "llvm/InitializePasses.h"
58 #include "llvm/Pass.h"
59 #include "llvm/Support/Casting.h"
60 #include "llvm/Support/Debug.h"
63 #include "llvm/Transforms/Scalar.h"
69 #include <cassert>
70 #include <cstddef>
71 #include <cstdint>
72 #include <initializer_list>
73 #include <iterator>
74 
75 using namespace llvm;
76 
77 #define DEBUG_TYPE "coro-split"
78 
79 namespace {
80 
81 /// A little helper class for building
82 class CoroCloner {
83 public:
84  enum class Kind {
85  /// The shared resume function for a switch lowering.
86  SwitchResume,
87 
88  /// The shared unwind function for a switch lowering.
89  SwitchUnwind,
90 
91  /// The shared cleanup function for a switch lowering.
92  SwitchCleanup,
93 
94  /// An individual continuation function.
95  Continuation,
96 
97  /// An async resume function.
98  Async,
99  };
100 
101 private:
102  Function &OrigF;
103  Function *NewF;
104  const Twine &Suffix;
105  coro::Shape &Shape;
106  Kind FKind;
107  ValueToValueMapTy VMap;
109  Value *NewFramePtr = nullptr;
110 
111  /// The active suspend instruction; meaningful only for continuation and async
112  /// ABIs.
113  AnyCoroSuspendInst *ActiveSuspend = nullptr;
114 
115 public:
116  /// Create a cloner for a switch lowering.
117  CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
118  Kind FKind)
119  : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape),
120  FKind(FKind), Builder(OrigF.getContext()) {
121  assert(Shape.ABI == coro::ABI::Switch);
122  }
123 
124  /// Create a cloner for a continuation lowering.
125  CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
126  Function *NewF, AnyCoroSuspendInst *ActiveSuspend)
127  : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape),
128  FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation),
129  Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend) {
130  assert(Shape.ABI == coro::ABI::Retcon ||
131  Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async);
132  assert(NewF && "need existing function for continuation");
133  assert(ActiveSuspend && "need active suspend point for continuation");
134  }
135 
136  Function *getFunction() const {
137  assert(NewF != nullptr && "declaration not yet set");
138  return NewF;
139  }
140 
141  void create();
142 
143 private:
144  bool isSwitchDestroyFunction() {
145  switch (FKind) {
146  case Kind::Async:
147  case Kind::Continuation:
148  case Kind::SwitchResume:
149  return false;
150  case Kind::SwitchUnwind:
151  case Kind::SwitchCleanup:
152  return true;
153  }
154  llvm_unreachable("Unknown CoroCloner::Kind enum");
155  }
156 
157  void replaceEntryBlock();
158  Value *deriveNewFramePointer();
159  void replaceRetconOrAsyncSuspendUses();
160  void replaceCoroSuspends();
161  void replaceCoroEnds();
162  void replaceSwiftErrorOps();
163  void salvageDebugInfo();
164  void handleFinalSuspend();
165 };
166 
167 } // end anonymous namespace
168 
170  const coro::Shape &Shape, Value *FramePtr,
171  CallGraph *CG) {
172  assert(Shape.ABI == coro::ABI::Retcon ||
173  Shape.ABI == coro::ABI::RetconOnce);
175  return;
176 
177  Shape.emitDealloc(Builder, FramePtr, CG);
178 }
179 
180 /// Replace an llvm.coro.end.async.
181 /// Will inline the must tail call function call if there is one.
182 /// \returns true if cleanup of the coro.end block is needed, false otherwise.
184  IRBuilder<> Builder(End);
185 
186  auto *EndAsync = dyn_cast<CoroAsyncEndInst>(End);
187  if (!EndAsync) {
188  Builder.CreateRetVoid();
189  return true /*needs cleanup of coro.end block*/;
190  }
191 
192  auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();
193  if (!MustTailCallFunc) {
194  Builder.CreateRetVoid();
195  return true /*needs cleanup of coro.end block*/;
196  }
197 
198  // Move the must tail call from the predecessor block into the end block.
199  auto *CoroEndBlock = End->getParent();
200  auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();
201  assert(MustTailCallFuncBlock && "Must have a single predecessor block");
202  auto It = MustTailCallFuncBlock->getTerminator()->getIterator();
203  auto *MustTailCall = cast<CallInst>(&*std::prev(It));
204  CoroEndBlock->getInstList().splice(
205  End->getIterator(), MustTailCallFuncBlock->getInstList(), MustTailCall);
206 
207  // Insert the return instruction.
208  Builder.SetInsertPoint(End);
209  Builder.CreateRetVoid();
210  InlineFunctionInfo FnInfo;
211 
212  // Remove the rest of the block, by splitting it into an unreachable block.
213  auto *BB = End->getParent();
214  BB->splitBasicBlock(End);
215  BB->getTerminator()->eraseFromParent();
216 
217  auto InlineRes = InlineFunction(*MustTailCall, FnInfo);
218  assert(InlineRes.isSuccess() && "Expected inlining to succeed");
219  (void)InlineRes;
220 
221  // We have cleaned up the coro.end block above.
222  return false;
223 }
224 
225 /// Replace a non-unwind call to llvm.coro.end.
227  const coro::Shape &Shape, Value *FramePtr,
228  bool InResume, CallGraph *CG) {
229  // Start inserting right before the coro.end.
230  IRBuilder<> Builder(End);
231 
232  // Create the return instruction.
233  switch (Shape.ABI) {
234  // The cloned functions in switch-lowering always return void.
235  case coro::ABI::Switch:
236  // coro.end doesn't immediately end the coroutine in the main function
237  // in this lowering, because we need to deallocate the coroutine.
238  if (!InResume)
239  return;
240  Builder.CreateRetVoid();
241  break;
242 
243  // In async lowering this returns.
244  case coro::ABI::Async: {
245  bool CoroEndBlockNeedsCleanup = replaceCoroEndAsync(End);
246  if (!CoroEndBlockNeedsCleanup)
247  return;
248  break;
249  }
250 
251  // In unique continuation lowering, the continuations always return void.
252  // But we may have implicitly allocated storage.
254  maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
255  Builder.CreateRetVoid();
256  break;
257 
258  // In non-unique continuation lowering, we signal completion by returning
259  // a null continuation.
260  case coro::ABI::Retcon: {
261  maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
262  auto RetTy = Shape.getResumeFunctionType()->getReturnType();
263  auto RetStructTy = dyn_cast<StructType>(RetTy);
264  PointerType *ContinuationTy =
265  cast<PointerType>(RetStructTy ? RetStructTy->getElementType(0) : RetTy);
266 
267  Value *ReturnValue = ConstantPointerNull::get(ContinuationTy);
268  if (RetStructTy) {
269  ReturnValue = Builder.CreateInsertValue(UndefValue::get(RetStructTy),
270  ReturnValue, 0);
271  }
272  Builder.CreateRet(ReturnValue);
273  break;
274  }
275  }
276 
277  // Remove the rest of the block, by splitting it into an unreachable block.
278  auto *BB = End->getParent();
279  BB->splitBasicBlock(End);
280  BB->getTerminator()->eraseFromParent();
281 }
282 
283 /// Replace an unwind call to llvm.coro.end.
284 static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
285  Value *FramePtr, bool InResume,
286  CallGraph *CG) {
287  IRBuilder<> Builder(End);
288 
289  switch (Shape.ABI) {
290  // In switch-lowering, this does nothing in the main function.
291  case coro::ABI::Switch:
292  if (!InResume)
293  return;
294  break;
295  // In async lowering this does nothing.
296  case coro::ABI::Async:
297  break;
298  // In continuation-lowering, this frees the continuation storage.
299  case coro::ABI::Retcon:
301  maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
302  break;
303  }
304 
305  // If coro.end has an associated bundle, add cleanupret instruction.
306  if (auto Bundle = End->getOperandBundle(LLVMContext::OB_funclet)) {
307  auto *FromPad = cast<CleanupPadInst>(Bundle->Inputs[0]);
308  auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);
309  End->getParent()->splitBasicBlock(End);
310  CleanupRet->getParent()->getTerminator()->eraseFromParent();
311  }
312 }
313 
314 static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
315  Value *FramePtr, bool InResume, CallGraph *CG) {
316  if (End->isUnwind())
317  replaceUnwindCoroEnd(End, Shape, FramePtr, InResume, CG);
318  else
319  replaceFallthroughCoroEnd(End, Shape, FramePtr, InResume, CG);
320 
321  auto &Context = End->getContext();
322  End->replaceAllUsesWith(InResume ? ConstantInt::getTrue(Context)
324  End->eraseFromParent();
325 }
326 
327 // Create an entry block for a resume function with a switch that will jump to
328 // suspend points.
330  assert(Shape.ABI == coro::ABI::Switch);
331  LLVMContext &C = F.getContext();
332 
333  // resume.entry:
334  // %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0,
335  // i32 2
336  // % index = load i32, i32* %index.addr
337  // switch i32 %index, label %unreachable [
338  // i32 0, label %resume.0
339  // i32 1, label %resume.1
340  // ...
341  // ]
342 
343  auto *NewEntry = BasicBlock::Create(C, "resume.entry", &F);
344  auto *UnreachBB = BasicBlock::Create(C, "unreachable", &F);
345 
346  IRBuilder<> Builder(NewEntry);
347  auto *FramePtr = Shape.FramePtr;
348  auto *FrameTy = Shape.FrameTy;
349  auto *GepIndex = Builder.CreateStructGEP(
350  FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
351  auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
352  auto *Switch =
353  Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
354  Shape.SwitchLowering.ResumeSwitch = Switch;
355 
356  size_t SuspendIndex = 0;
357  for (auto *AnyS : Shape.CoroSuspends) {
358  auto *S = cast<CoroSuspendInst>(AnyS);
359  ConstantInt *IndexVal = Shape.getIndex(SuspendIndex);
360 
361  // Replace CoroSave with a store to Index:
362  // %index.addr = getelementptr %f.frame... (index field number)
363  // store i32 0, i32* %index.addr1
364  auto *Save = S->getCoroSave();
365  Builder.SetInsertPoint(Save);
366  if (S->isFinal()) {
367  // Final suspend point is represented by storing zero in ResumeFnAddr.
368  auto *GepIndex = Builder.CreateStructGEP(FrameTy, FramePtr,
370  "ResumeFn.addr");
371  auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
372  FrameTy->getTypeAtIndex(coro::Shape::SwitchFieldIndex::Resume)));
373  Builder.CreateStore(NullPtr, GepIndex);
374  } else {
375  auto *GepIndex = Builder.CreateStructGEP(
376  FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
377  Builder.CreateStore(IndexVal, GepIndex);
378  }
379  Save->replaceAllUsesWith(ConstantTokenNone::get(C));
380  Save->eraseFromParent();
381 
382  // Split block before and after coro.suspend and add a jump from an entry
383  // switch:
384  //
385  // whateverBB:
386  // whatever
387  // %0 = call i8 @llvm.coro.suspend(token none, i1 false)
388  // switch i8 %0, label %suspend[i8 0, label %resume
389  // i8 1, label %cleanup]
390  // becomes:
391  //
392  // whateverBB:
393  // whatever
394  // br label %resume.0.landing
395  //
396  // resume.0: ; <--- jump from the switch in the resume.entry
397  // %0 = tail call i8 @llvm.coro.suspend(token none, i1 false)
398  // br label %resume.0.landing
399  //
400  // resume.0.landing:
401  // %1 = phi i8[-1, %whateverBB], [%0, %resume.0]
402  // switch i8 % 1, label %suspend [i8 0, label %resume
403  // i8 1, label %cleanup]
404 
405  auto *SuspendBB = S->getParent();
406  auto *ResumeBB =
407  SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));
408  auto *LandingBB = ResumeBB->splitBasicBlock(
409  S->getNextNode(), ResumeBB->getName() + Twine(".landing"));
410  Switch->addCase(IndexVal, ResumeBB);
411 
412  cast<BranchInst>(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);
413  auto *PN = PHINode::Create(Builder.getInt8Ty(), 2, "", &LandingBB->front());
414  S->replaceAllUsesWith(PN);
415  PN->addIncoming(Builder.getInt8(-1), SuspendBB);
416  PN->addIncoming(S, ResumeBB);
417 
418  ++SuspendIndex;
419  }
420 
421  Builder.SetInsertPoint(UnreachBB);
422  Builder.CreateUnreachable();
423 
424  Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
425 }
426 
427 
428 // Rewrite final suspend point handling. We do not use suspend index to
429 // represent the final suspend point. Instead we zero-out ResumeFnAddr in the
430 // coroutine frame, since it is undefined behavior to resume a coroutine
431 // suspended at the final suspend point. Thus, in the resume function, we can
432 // simply remove the last case (when coro::Shape is built, the final suspend
433 // point (if present) is always the last element of CoroSuspends array).
434 // In the destroy function, we add a code sequence to check if ResumeFnAddress
435 // is Null, and if so, jump to the appropriate label to handle cleanup from the
436 // final suspend point.
437 void CoroCloner::handleFinalSuspend() {
438  assert(Shape.ABI == coro::ABI::Switch &&
439  Shape.SwitchLowering.HasFinalSuspend);
440  auto *Switch = cast<SwitchInst>(VMap[Shape.SwitchLowering.ResumeSwitch]);
441  auto FinalCaseIt = std::prev(Switch->case_end());
442  BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();
443  Switch->removeCase(FinalCaseIt);
444  if (isSwitchDestroyFunction()) {
445  BasicBlock *OldSwitchBB = Switch->getParent();
446  auto *NewSwitchBB = OldSwitchBB->splitBasicBlock(Switch, "Switch");
447  Builder.SetInsertPoint(OldSwitchBB->getTerminator());
448  auto *GepIndex = Builder.CreateStructGEP(Shape.FrameTy, NewFramePtr,
450  "ResumeFn.addr");
451  auto *Load = Builder.CreateLoad(Shape.getSwitchResumePointerType(),
452  GepIndex);
453  auto *Cond = Builder.CreateIsNull(Load);
454  Builder.CreateCondBr(Cond, ResumeBB, NewSwitchBB);
455  OldSwitchBB->getTerminator()->eraseFromParent();
456  }
457 }
458 
459 static FunctionType *
461  auto *AsyncSuspend = cast<CoroSuspendAsyncInst>(Suspend);
462  auto *StructTy = cast<StructType>(AsyncSuspend->getType());
463  auto &Context = Suspend->getParent()->getParent()->getContext();
464  auto *VoidTy = Type::getVoidTy(Context);
465  return FunctionType::get(VoidTy, StructTy->elements(), false);
466 }
467 
469  const Twine &Suffix,
470  Module::iterator InsertBefore,
471  AnyCoroSuspendInst *ActiveSuspend) {
472  Module *M = OrigF.getParent();
473  auto *FnTy = (Shape.ABI != coro::ABI::Async)
474  ? Shape.getResumeFunctionType()
475  : getFunctionTypeFromAsyncSuspend(ActiveSuspend);
476 
477  Function *NewF =
478  Function::Create(FnTy, GlobalValue::LinkageTypes::InternalLinkage,
479  OrigF.getName() + Suffix);
480  if (Shape.ABI != coro::ABI::Async)
481  NewF->addParamAttr(0, Attribute::NonNull);
482 
483  // For the async lowering ABI we can't guarantee that the context argument is
484  // not access via a different pointer not based on the argument.
485  if (Shape.ABI != coro::ABI::Async)
486  NewF->addParamAttr(0, Attribute::NoAlias);
487 
488  M->getFunctionList().insert(InsertBefore, NewF);
489 
490  return NewF;
491 }
492 
493 /// Replace uses of the active llvm.coro.suspend.retcon/async call with the
494 /// arguments to the continuation function.
495 ///
496 /// This assumes that the builder has a meaningful insertion point.
497 void CoroCloner::replaceRetconOrAsyncSuspendUses() {
498  assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||
499  Shape.ABI == coro::ABI::Async);
500 
501  auto NewS = VMap[ActiveSuspend];
502  if (NewS->use_empty()) return;
503 
504  // Copy out all the continuation arguments after the buffer pointer into
505  // an easily-indexed data structure for convenience.
507  // The async ABI includes all arguments -- including the first argument.
508  bool IsAsyncABI = Shape.ABI == coro::ABI::Async;
509  for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),
510  E = NewF->arg_end();
511  I != E; ++I)
512  Args.push_back(&*I);
513 
514  // If the suspend returns a single scalar value, we can just do a simple
515  // replacement.
516  if (!isa<StructType>(NewS->getType())) {
517  assert(Args.size() == 1);
518  NewS->replaceAllUsesWith(Args.front());
519  return;
520  }
521 
522  // Try to peephole extracts of an aggregate return.
523  for (auto UI = NewS->use_begin(), UE = NewS->use_end(); UI != UE; ) {
524  auto EVI = dyn_cast<ExtractValueInst>((UI++)->getUser());
525  if (!EVI || EVI->getNumIndices() != 1)
526  continue;
527 
528  EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);
529  EVI->eraseFromParent();
530  }
531 
532  // If we have no remaining uses, we're done.
533  if (NewS->use_empty()) return;
534 
535  // Otherwise, we need to create an aggregate.
536  Value *Agg = UndefValue::get(NewS->getType());
537  for (size_t I = 0, E = Args.size(); I != E; ++I)
538  Agg = Builder.CreateInsertValue(Agg, Args[I], I);
539 
540  NewS->replaceAllUsesWith(Agg);
541 }
542 
543 void CoroCloner::replaceCoroSuspends() {
544  Value *SuspendResult;
545 
546  switch (Shape.ABI) {
547  // In switch lowering, replace coro.suspend with the appropriate value
548  // for the type of function we're extracting.
549  // Replacing coro.suspend with (0) will result in control flow proceeding to
550  // a resume label associated with a suspend point, replacing it with (1) will
551  // result in control flow proceeding to a cleanup label associated with this
552  // suspend point.
553  case coro::ABI::Switch:
554  SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);
555  break;
556 
557  // In async lowering there are no uses of the result.
558  case coro::ABI::Async:
559  return;
560 
561  // In returned-continuation lowering, the arguments from earlier
562  // continuations are theoretically arbitrary, and they should have been
563  // spilled.
565  case coro::ABI::Retcon:
566  return;
567  }
568 
569  for (AnyCoroSuspendInst *CS : Shape.CoroSuspends) {
570  // The active suspend was handled earlier.
571  if (CS == ActiveSuspend) continue;
572 
573  auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[CS]);
574  MappedCS->replaceAllUsesWith(SuspendResult);
575  MappedCS->eraseFromParent();
576  }
577 }
578 
579 void CoroCloner::replaceCoroEnds() {
580  for (AnyCoroEndInst *CE : Shape.CoroEnds) {
581  // We use a null call graph because there's no call graph node for
582  // the cloned function yet. We'll just be rebuilding that later.
583  auto *NewCE = cast<AnyCoroEndInst>(VMap[CE]);
584  replaceCoroEnd(NewCE, Shape, NewFramePtr, /*in resume*/ true, nullptr);
585  }
586 }
587 
589  ValueToValueMapTy *VMap) {
590  if (Shape.ABI == coro::ABI::Async && Shape.CoroSuspends.empty())
591  return;
592  Value *CachedSlot = nullptr;
593  auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {
594  if (CachedSlot) {
595  assert(CachedSlot->getType()->getPointerElementType() == ValueTy &&
596  "multiple swifterror slots in function with different types");
597  return CachedSlot;
598  }
599 
600  // Check if the function has a swifterror argument.
601  for (auto &Arg : F.args()) {
602  if (Arg.isSwiftError()) {
603  CachedSlot = &Arg;
604  assert(Arg.getType()->getPointerElementType() == ValueTy &&
605  "swifterror argument does not have expected type");
606  return &Arg;
607  }
608  }
609 
610  // Create a swifterror alloca.
611  IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
612  auto Alloca = Builder.CreateAlloca(ValueTy);
613  Alloca->setSwiftError(true);
614 
615  CachedSlot = Alloca;
616  return Alloca;
617  };
618 
619  for (CallInst *Op : Shape.SwiftErrorOps) {
620  auto MappedOp = VMap ? cast<CallInst>((*VMap)[Op]) : Op;
621  IRBuilder<> Builder(MappedOp);
622 
623  // If there are no arguments, this is a 'get' operation.
624  Value *MappedResult;
625  if (Op->getNumArgOperands() == 0) {
626  auto ValueTy = Op->getType();
627  auto Slot = getSwiftErrorSlot(ValueTy);
628  MappedResult = Builder.CreateLoad(ValueTy, Slot);
629  } else {
630  assert(Op->getNumArgOperands() == 1);
631  auto Value = MappedOp->getArgOperand(0);
632  auto ValueTy = Value->getType();
633  auto Slot = getSwiftErrorSlot(ValueTy);
634  Builder.CreateStore(Value, Slot);
635  MappedResult = Slot;
636  }
637 
638  MappedOp->replaceAllUsesWith(MappedResult);
639  MappedOp->eraseFromParent();
640  }
641 
642  // If we're updating the original function, we've invalidated SwiftErrorOps.
643  if (VMap == nullptr) {
644  Shape.SwiftErrorOps.clear();
645  }
646 }
647 
649  ::replaceSwiftErrorOps(*NewF, Shape, &VMap);
650 }
651 
655  for (auto &BB : *NewF)
656  for (auto &I : BB)
657  if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
658  Worklist.push_back(DVI);
659  for (DbgVariableIntrinsic *DVI : Worklist)
660  coro::salvageDebugInfo(DbgPtrAllocaCache, DVI, Shape.ReuseFrameSlot);
661 
662  // Remove all salvaged dbg.declare intrinsics that became
663  // either unreachable or stale due to the CoroSplit transformation.
664  DominatorTree DomTree(*NewF);
665  auto IsUnreachableBlock = [&](BasicBlock *BB) {
666  return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
667  &DomTree);
668  };
669  for (DbgVariableIntrinsic *DVI : Worklist) {
670  if (IsUnreachableBlock(DVI->getParent()))
671  DVI->eraseFromParent();
672  else if (dyn_cast_or_null<AllocaInst>(DVI->getVariableLocationOp(0))) {
673  // Count all non-debuginfo uses in reachable blocks.
674  unsigned Uses = 0;
675  for (auto *User : DVI->getVariableLocationOp(0)->users())
676  if (auto *I = dyn_cast<Instruction>(User))
677  if (!isa<AllocaInst>(I) && !IsUnreachableBlock(I->getParent()))
678  ++Uses;
679  if (!Uses)
680  DVI->eraseFromParent();
681  }
682  }
683 }
684 
685 void CoroCloner::replaceEntryBlock() {
686  // In the original function, the AllocaSpillBlock is a block immediately
687  // following the allocation of the frame object which defines GEPs for
688  // all the allocas that have been moved into the frame, and it ends by
689  // branching to the original beginning of the coroutine. Make this
690  // the entry block of the cloned function.
691  auto *Entry = cast<BasicBlock>(VMap[Shape.AllocaSpillBlock]);
692  auto *OldEntry = &NewF->getEntryBlock();
693  Entry->setName("entry" + Suffix);
694  Entry->moveBefore(OldEntry);
695  Entry->getTerminator()->eraseFromParent();
696 
697  // Clear all predecessors of the new entry block. There should be
698  // exactly one predecessor, which we created when splitting out
699  // AllocaSpillBlock to begin with.
700  assert(Entry->hasOneUse());
701  auto BranchToEntry = cast<BranchInst>(Entry->user_back());
702  assert(BranchToEntry->isUnconditional());
703  Builder.SetInsertPoint(BranchToEntry);
704  Builder.CreateUnreachable();
705  BranchToEntry->eraseFromParent();
706 
707  // Branch from the entry to the appropriate place.
708  Builder.SetInsertPoint(Entry);
709  switch (Shape.ABI) {
710  case coro::ABI::Switch: {
711  // In switch-lowering, we built a resume-entry block in the original
712  // function. Make the entry block branch to this.
713  auto *SwitchBB =
714  cast<BasicBlock>(VMap[Shape.SwitchLowering.ResumeEntryBlock]);
715  Builder.CreateBr(SwitchBB);
716  break;
717  }
718  case coro::ABI::Async:
719  case coro::ABI::Retcon:
720  case coro::ABI::RetconOnce: {
721  // In continuation ABIs, we want to branch to immediately after the
722  // active suspend point. Earlier phases will have put the suspend in its
723  // own basic block, so just thread our jump directly to its successor.
724  assert((Shape.ABI == coro::ABI::Async &&
725  isa<CoroSuspendAsyncInst>(ActiveSuspend)) ||
726  ((Shape.ABI == coro::ABI::Retcon ||
727  Shape.ABI == coro::ABI::RetconOnce) &&
728  isa<CoroSuspendRetconInst>(ActiveSuspend)));
729  auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[ActiveSuspend]);
730  auto Branch = cast<BranchInst>(MappedCS->getNextNode());
731  assert(Branch->isUnconditional());
732  Builder.CreateBr(Branch->getSuccessor(0));
733  break;
734  }
735  }
736 
737  // Any static alloca that's still being used but not reachable from the new
738  // entry needs to be moved to the new entry.
739  Function *F = OldEntry->getParent();
740  DominatorTree DT{*F};
741  for (auto IT = inst_begin(F), End = inst_end(F); IT != End;) {
742  Instruction &I = *IT++;
743  auto *Alloca = dyn_cast<AllocaInst>(&I);
744  if (!Alloca || I.use_empty())
745  continue;
746  if (DT.isReachableFromEntry(I.getParent()) ||
747  !isa<ConstantInt>(Alloca->getArraySize()))
748  continue;
749  I.moveBefore(*Entry, Entry->getFirstInsertionPt());
750  }
751 }
752 
753 /// Derive the value of the new frame pointer.
754 Value *CoroCloner::deriveNewFramePointer() {
755  // Builder should be inserting to the front of the new entry block.
756 
757  switch (Shape.ABI) {
758  // In switch-lowering, the argument is the frame pointer.
759  case coro::ABI::Switch:
760  return &*NewF->arg_begin();
761  // In async-lowering, one of the arguments is an async context as determined
762  // by the `llvm.coro.id.async` intrinsic. We can retrieve the async context of
763  // the resume function from the async context projection function associated
764  // with the active suspend. The frame is located as a tail to the async
765  // context header.
766  case coro::ABI::Async: {
767  auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
768  auto *CalleeContext =
769  NewF->getArg(ActiveAsyncSuspend->getStorageArgumentIndex());
770  auto *FramePtrTy = Shape.FrameTy->getPointerTo();
771  auto *ProjectionFunc =
772  ActiveAsyncSuspend->getAsyncContextProjectionFunction();
773  auto DbgLoc =
774  cast<CoroSuspendAsyncInst>(VMap[ActiveSuspend])->getDebugLoc();
775  // Calling i8* (i8*)
776  auto *CallerContext = Builder.CreateCall(
777  cast<FunctionType>(ProjectionFunc->getType()->getPointerElementType()),
778  ProjectionFunc, CalleeContext);
779  CallerContext->setCallingConv(ProjectionFunc->getCallingConv());
780  CallerContext->setDebugLoc(DbgLoc);
781  // The frame is located after the async_context header.
782  auto &Context = Builder.getContext();
783  auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(
784  Type::getInt8Ty(Context), CallerContext,
785  Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr");
786  // Inline the projection function.
788  auto InlineRes = InlineFunction(*CallerContext, InlineInfo);
789  assert(InlineRes.isSuccess());
790  (void)InlineRes;
791  return Builder.CreateBitCast(FramePtrAddr, FramePtrTy);
792  }
793  // In continuation-lowering, the argument is the opaque storage.
794  case coro::ABI::Retcon:
795  case coro::ABI::RetconOnce: {
796  Argument *NewStorage = &*NewF->arg_begin();
797  auto FramePtrTy = Shape.FrameTy->getPointerTo();
798 
799  // If the storage is inline, just bitcast to the storage to the frame type.
800  if (Shape.RetconLowering.IsFrameInlineInStorage)
801  return Builder.CreateBitCast(NewStorage, FramePtrTy);
802 
803  // Otherwise, load the real frame from the opaque storage.
804  auto FramePtrPtr =
805  Builder.CreateBitCast(NewStorage, FramePtrTy->getPointerTo());
806  return Builder.CreateLoad(FramePtrTy, FramePtrPtr);
807  }
808  }
809  llvm_unreachable("bad ABI");
810 }
811 
813  unsigned ParamIndex,
814  uint64_t Size, Align Alignment) {
815  AttrBuilder ParamAttrs;
816  ParamAttrs.addAttribute(Attribute::NonNull);
817  ParamAttrs.addAttribute(Attribute::NoAlias);
818  ParamAttrs.addAlignmentAttr(Alignment);
819  ParamAttrs.addDereferenceableAttr(Size);
820  Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
821 }
822 
824  unsigned ParamIndex) {
825  AttrBuilder ParamAttrs;
826  ParamAttrs.addAttribute(Attribute::SwiftAsync);
827  Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
828 }
829 
830 /// Clone the body of the original function into a resume function of
831 /// some sort.
832 void CoroCloner::create() {
833  // Create the new function if we don't already have one.
834  if (!NewF) {
835  NewF = createCloneDeclaration(OrigF, Shape, Suffix,
836  OrigF.getParent()->end(), ActiveSuspend);
837  }
838 
839  // Replace all args with undefs. The buildCoroutineFrame algorithm already
840  // rewritten access to the args that occurs after suspend points with loads
841  // and stores to/from the coroutine frame.
842  for (Argument &A : OrigF.args())
843  VMap[&A] = UndefValue::get(A.getType());
844 
846 
847  // Ignore attempts to change certain attributes of the function.
848  // TODO: maybe there should be a way to suppress this during cloning?
849  auto savedVisibility = NewF->getVisibility();
850  auto savedUnnamedAddr = NewF->getUnnamedAddr();
851  auto savedDLLStorageClass = NewF->getDLLStorageClass();
852 
853  // NewF's linkage (which CloneFunctionInto does *not* change) might not
854  // be compatible with the visibility of OrigF (which it *does* change),
855  // so protect against that.
856  auto savedLinkage = NewF->getLinkage();
857  NewF->setLinkage(llvm::GlobalValue::ExternalLinkage);
858 
859  CloneFunctionInto(NewF, &OrigF, VMap,
861 
862  auto &Context = NewF->getContext();
863 
864  // For async functions / continuations, adjust the scope line of the
865  // clone to the line number of the suspend point. However, only
866  // adjust the scope line when the files are the same. This ensures
867  // line number and file name belong together. The scope line is
868  // associated with all pre-prologue instructions. This avoids a jump
869  // in the linetable from the function declaration to the suspend point.
870  if (DISubprogram *SP = NewF->getSubprogram()) {
871  assert(SP != OrigF.getSubprogram() && SP->isDistinct());
872  if (ActiveSuspend)
873  if (auto DL = ActiveSuspend->getDebugLoc())
874  if (SP->getFile() == DL->getFile())
875  SP->setScopeLine(DL->getLine());
876  // Update the linkage name to reflect the modified symbol name. It
877  // is necessary to update the linkage name in Swift, since the
878  // mangling changes for resume functions. It might also be the
879  // right thing to do in C++, but due to a limitation in LLVM's
880  // AsmPrinter we can only do this if the function doesn't have an
881  // abstract specification, since the DWARF backend expects the
882  // abstract specification to contain the linkage name and asserts
883  // that they are identical.
884  if (!SP->getDeclaration() && SP->getUnit() &&
885  SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift)
886  SP->replaceLinkageName(MDString::get(Context, NewF->getName()));
887  }
888 
889  NewF->setLinkage(savedLinkage);
890  NewF->setVisibility(savedVisibility);
891  NewF->setUnnamedAddr(savedUnnamedAddr);
892  NewF->setDLLStorageClass(savedDLLStorageClass);
893 
894  // Replace the attributes of the new function:
895  auto OrigAttrs = NewF->getAttributes();
896  auto NewAttrs = AttributeList();
897 
898  switch (Shape.ABI) {
899  case coro::ABI::Switch:
900  // Bootstrap attributes by copying function attributes from the
901  // original function. This should include optimization settings and so on.
902  NewAttrs = NewAttrs.addAttributes(Context, AttributeList::FunctionIndex,
903  OrigAttrs.getFnAttributes());
904 
905  addFramePointerAttrs(NewAttrs, Context, 0,
906  Shape.FrameSize, Shape.FrameAlign);
907  break;
908  case coro::ABI::Async: {
909  // Transfer the original function's attributes.
910  auto FnAttrs = OrigF.getAttributes().getFnAttributes();
911  NewAttrs =
912  NewAttrs.addAttributes(Context, AttributeList::FunctionIndex, FnAttrs);
913  break;
914  }
915  case coro::ABI::Retcon:
917  // If we have a continuation prototype, just use its attributes,
918  // full-stop.
919  NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes();
920 
921  addFramePointerAttrs(NewAttrs, Context, 0,
922  Shape.getRetconCoroId()->getStorageSize(),
923  Shape.getRetconCoroId()->getStorageAlignment());
924  break;
925  }
926 
927  switch (Shape.ABI) {
928  // In these ABIs, the cloned functions always return 'void', and the
929  // existing return sites are meaningless. Note that for unique
930  // continuations, this includes the returns associated with suspends;
931  // this is fine because we can't suspend twice.
932  case coro::ABI::Switch:
934  // Remove old returns.
935  for (ReturnInst *Return : Returns)
936  changeToUnreachable(Return, /*UseLLVMTrap=*/false);
937  break;
938 
939  // With multi-suspend continuations, we'll already have eliminated the
940  // original returns and inserted returns before all the suspend points,
941  // so we want to leave any returns in place.
942  case coro::ABI::Retcon:
943  break;
944  // Async lowering will insert musttail call functions at all suspend points
945  // followed by a return.
946  // Don't change returns to unreachable because that will trip up the verifier.
947  // These returns should be unreachable from the clone.
948  case coro::ABI::Async: {
949  auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
950  if (OrigF.hasParamAttribute(Shape.AsyncLowering.ContextArgNo,
951  Attribute::SwiftAsync)) {
952  auto ContextArgIndex = ActiveAsyncSuspend->getStorageArgumentIndex();
953  addAsyncContextAttrs(NewAttrs, Context, ContextArgIndex);
954  }
955  break;
956  }
957  }
958 
959  NewF->setAttributes(NewAttrs);
960  NewF->setCallingConv(Shape.getResumeFunctionCC());
961 
962  // Set up the new entry block.
963  replaceEntryBlock();
964 
965  Builder.SetInsertPoint(&NewF->getEntryBlock().front());
966  NewFramePtr = deriveNewFramePointer();
967 
968  // Remap frame pointer.
969  Value *OldFramePtr = VMap[Shape.FramePtr];
970  NewFramePtr->takeName(OldFramePtr);
971  OldFramePtr->replaceAllUsesWith(NewFramePtr);
972 
973  // Remap vFrame pointer.
974  auto *NewVFrame = Builder.CreateBitCast(
975  NewFramePtr, Type::getInt8PtrTy(Builder.getContext()), "vFrame");
976  Value *OldVFrame = cast<Value>(VMap[Shape.CoroBegin]);
977  OldVFrame->replaceAllUsesWith(NewVFrame);
978 
979  switch (Shape.ABI) {
980  case coro::ABI::Switch:
981  // Rewrite final suspend handling as it is not done via switch (allows to
982  // remove final case from the switch, since it is undefined behavior to
983  // resume the coroutine suspended at the final suspend point.
984  if (Shape.SwitchLowering.HasFinalSuspend)
985  handleFinalSuspend();
986  break;
987  case coro::ABI::Async:
988  case coro::ABI::Retcon:
990  // Replace uses of the active suspend with the corresponding
991  // continuation-function arguments.
992  assert(ActiveSuspend != nullptr &&
993  "no active suspend when lowering a continuation-style coroutine");
994  replaceRetconOrAsyncSuspendUses();
995  break;
996  }
997 
998  // Handle suspends.
999  replaceCoroSuspends();
1000 
1001  // Handle swifterror.
1003 
1004  // Remove coro.end intrinsics.
1005  replaceCoroEnds();
1006 
1007  // Salvage debug info that points into the coroutine frame.
1008  salvageDebugInfo();
1009 
1010  // Eliminate coro.free from the clones, replacing it with 'null' in cleanup,
1011  // to suppress deallocation code.
1012  if (Shape.ABI == coro::ABI::Switch)
1013  coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]),
1014  /*Elide=*/ FKind == CoroCloner::Kind::SwitchCleanup);
1015 }
1016 
1017 // Create a resume clone by cloning the body of the original function, setting
1018 // new entry block and replacing coro.suspend an appropriate value to force
1019 // resume or cleanup pass for every suspend point.
1020 static Function *createClone(Function &F, const Twine &Suffix,
1021  coro::Shape &Shape, CoroCloner::Kind FKind) {
1022  CoroCloner Cloner(F, Suffix, Shape, FKind);
1023  Cloner.create();
1024  return Cloner.getFunction();
1025 }
1026 
1027 /// Remove calls to llvm.coro.end in the original function.
1028 static void removeCoroEnds(const coro::Shape &Shape, CallGraph *CG) {
1029  for (auto End : Shape.CoroEnds) {
1030  replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, CG);
1031  }
1032 }
1033 
1035  assert(Shape.ABI == coro::ABI::Async);
1036 
1037  auto *FuncPtrStruct = cast<ConstantStruct>(
1039  auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);
1040  auto *OrigContextSize = FuncPtrStruct->getOperand(1);
1041  auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),
1042  Shape.AsyncLowering.ContextSize);
1043  auto *NewFuncPtrStruct = ConstantStruct::get(
1044  FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);
1045 
1046  Shape.AsyncLowering.AsyncFuncPointer->setInitializer(NewFuncPtrStruct);
1047 }
1048 
1049 static void replaceFrameSize(coro::Shape &Shape) {
1050  if (Shape.ABI == coro::ABI::Async)
1052 
1053  if (Shape.CoroSizes.empty())
1054  return;
1055 
1056  // In the same function all coro.sizes should have the same result type.
1057  auto *SizeIntrin = Shape.CoroSizes.back();
1058  Module *M = SizeIntrin->getModule();
1059  const DataLayout &DL = M->getDataLayout();
1060  auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1061  auto *SizeConstant = ConstantInt::get(SizeIntrin->getType(), Size);
1062 
1063  for (CoroSizeInst *CS : Shape.CoroSizes) {
1064  CS->replaceAllUsesWith(SizeConstant);
1065  CS->eraseFromParent();
1066  }
1067 }
1068 
1069 // Create a global constant array containing pointers to functions provided and
1070 // set Info parameter of CoroBegin to point at this constant. Example:
1071 //
1072 // @f.resumers = internal constant [2 x void(%f.frame*)*]
1073 // [void(%f.frame*)* @f.resume, void(%f.frame*)* @f.destroy]
1074 // define void @f() {
1075 // ...
1076 // call i8* @llvm.coro.begin(i8* null, i32 0, i8* null,
1077 // i8* bitcast([2 x void(%f.frame*)*] * @f.resumers to i8*))
1078 //
1079 // Assumes that all the functions have the same signature.
1080 static void setCoroInfo(Function &F, coro::Shape &Shape,
1081  ArrayRef<Function *> Fns) {
1082  // This only works under the switch-lowering ABI because coro elision
1083  // only works on the switch-lowering ABI.
1084  assert(Shape.ABI == coro::ABI::Switch);
1085 
1086  SmallVector<Constant *, 4> Args(Fns.begin(), Fns.end());
1087  assert(!Args.empty());
1088  Function *Part = *Fns.begin();
1089  Module *M = Part->getParent();
1090  auto *ArrTy = ArrayType::get(Part->getType(), Args.size());
1091 
1092  auto *ConstVal = ConstantArray::get(ArrTy, Args);
1093  auto *GV = new GlobalVariable(*M, ConstVal->getType(), /*isConstant=*/true,
1095  F.getName() + Twine(".resumers"));
1096 
1097  // Update coro.begin instruction to refer to this constant.
1098  LLVMContext &C = F.getContext();
1100  Shape.getSwitchCoroId()->setInfo(BC);
1101 }
1102 
1103 // Store addresses of Resume/Destroy/Cleanup functions in the coroutine frame.
1104 static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
1105  Function *DestroyFn, Function *CleanupFn) {
1106  assert(Shape.ABI == coro::ABI::Switch);
1107 
1109  auto *ResumeAddr = Builder.CreateStructGEP(
1111  "resume.addr");
1112  Builder.CreateStore(ResumeFn, ResumeAddr);
1113 
1114  Value *DestroyOrCleanupFn = DestroyFn;
1115 
1116  CoroIdInst *CoroId = Shape.getSwitchCoroId();
1117  if (CoroAllocInst *CA = CoroId->getCoroAlloc()) {
1118  // If there is a CoroAlloc and it returns false (meaning we elide the
1119  // allocation, use CleanupFn instead of DestroyFn).
1120  DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);
1121  }
1122 
1123  auto *DestroyAddr = Builder.CreateStructGEP(
1125  "destroy.addr");
1126  Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);
1127 }
1128 
1129 static void postSplitCleanup(Function &F) {
1131 
1132  // For now, we do a mandatory verification step because we don't
1133  // entirely trust this pass. Note that we don't want to add a verifier
1134  // pass to FPM below because it will also verify all the global data.
1135  if (verifyFunction(F, &errs()))
1136  report_fatal_error("Broken function");
1137 
1138  legacy::FunctionPassManager FPM(F.getParent());
1139 
1140  FPM.add(createSCCPPass());
1142  FPM.add(createEarlyCSEPass());
1144 
1145  FPM.doInitialization();
1146  FPM.run(F);
1147  FPM.doFinalization();
1148 }
1149 
1150 // Assuming we arrived at the block NewBlock from Prev instruction, store
1151 // PHI's incoming values in the ResolvedValues map.
1152 static void
1154  DenseMap<Value *, Value *> &ResolvedValues) {
1155  auto *PrevBB = Prev->getParent();
1156  for (PHINode &PN : NewBlock->phis()) {
1157  auto V = PN.getIncomingValueForBlock(PrevBB);
1158  // See if we already resolved it.
1159  auto VI = ResolvedValues.find(V);
1160  if (VI != ResolvedValues.end())
1161  V = VI->second;
1162  // Remember the value.
1163  ResolvedValues[&PN] = V;
1164  }
1165 }
1166 
1167 // Replace a sequence of branches leading to a ret, with a clone of a ret
1168 // instruction. Suspend instruction represented by a switch, track the PHI
1169 // values and select the correct case successor when possible.
1170 static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) {
1171  DenseMap<Value *, Value *> ResolvedValues;
1172  BasicBlock *UnconditionalSucc = nullptr;
1173 
1174  Instruction *I = InitialInst;
1175  while (I->isTerminator() ||
1176  (isa<CmpInst>(I) && I->getNextNode()->isTerminator())) {
1177  if (isa<ReturnInst>(I)) {
1178  if (I != InitialInst) {
1179  // If InitialInst is an unconditional branch,
1180  // remove PHI values that come from basic block of InitialInst
1181  if (UnconditionalSucc)
1182  UnconditionalSucc->removePredecessor(InitialInst->getParent(), true);
1183  ReplaceInstWithInst(InitialInst, I->clone());
1184  }
1185  return true;
1186  }
1187  if (auto *BR = dyn_cast<BranchInst>(I)) {
1188  if (BR->isUnconditional()) {
1189  BasicBlock *BB = BR->getSuccessor(0);
1190  if (I == InitialInst)
1191  UnconditionalSucc = BB;
1192  scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
1193  I = BB->getFirstNonPHIOrDbgOrLifetime();
1194  continue;
1195  }
1196  } else if (auto *CondCmp = dyn_cast<CmpInst>(I)) {
1197  auto *BR = dyn_cast<BranchInst>(I->getNextNode());
1198  if (BR && BR->isConditional() && CondCmp == BR->getCondition()) {
1199  // If the case number of suspended switch instruction is reduced to
1200  // 1, then it is simplified to CmpInst in llvm::ConstantFoldTerminator.
1201  // And the comparsion looks like : %cond = icmp eq i8 %V, constant.
1202  ConstantInt *CondConst = dyn_cast<ConstantInt>(CondCmp->getOperand(1));
1203  if (CondConst && CondCmp->getPredicate() == CmpInst::ICMP_EQ) {
1204  Value *V = CondCmp->getOperand(0);
1205  auto it = ResolvedValues.find(V);
1206  if (it != ResolvedValues.end())
1207  V = it->second;
1208 
1209  if (ConstantInt *Cond0 = dyn_cast<ConstantInt>(V)) {
1210  BasicBlock *BB = Cond0->equalsInt(CondConst->getZExtValue())
1211  ? BR->getSuccessor(0)
1212  : BR->getSuccessor(1);
1213  scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
1214  I = BB->getFirstNonPHIOrDbgOrLifetime();
1215  continue;
1216  }
1217  }
1218  }
1219  } else if (auto *SI = dyn_cast<SwitchInst>(I)) {
1220  Value *V = SI->getCondition();
1221  auto it = ResolvedValues.find(V);
1222  if (it != ResolvedValues.end())
1223  V = it->second;
1224  if (ConstantInt *Cond = dyn_cast<ConstantInt>(V)) {
1225  BasicBlock *BB = SI->findCaseValue(Cond)->getCaseSuccessor();
1226  scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
1227  I = BB->getFirstNonPHIOrDbgOrLifetime();
1228  continue;
1229  }
1230  }
1231  return false;
1232  }
1233  return false;
1234 }
1235 
1236 // Check whether CI obeys the rules of musttail attribute.
1237 static bool shouldBeMustTail(const CallInst &CI, const Function &F) {
1238  if (CI.isInlineAsm())
1239  return false;
1240 
1241  // Match prototypes and calling conventions of resume function.
1242  FunctionType *CalleeTy = CI.getFunctionType();
1243  if (!CalleeTy->getReturnType()->isVoidTy() || (CalleeTy->getNumParams() != 1))
1244  return false;
1245 
1246  Type *CalleeParmTy = CalleeTy->getParamType(0);
1247  if (!CalleeParmTy->isPointerTy() ||
1248  (CalleeParmTy->getPointerAddressSpace() != 0))
1249  return false;
1250 
1251  if (CI.getCallingConv() != F.getCallingConv())
1252  return false;
1253 
1254  // CI should not has any ABI-impacting function attributes.
1255  static const Attribute::AttrKind ABIAttrs[] = {
1256  Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
1257  Attribute::Preallocated, Attribute::InReg, Attribute::Returned,
1258  Attribute::SwiftSelf, Attribute::SwiftError};
1260  for (auto AK : ABIAttrs)
1261  if (Attrs.hasParamAttribute(0, AK))
1262  return false;
1263 
1264  return true;
1265 }
1266 
1267 // Add musttail to any resume instructions that is immediately followed by a
1268 // suspend (i.e. ret). We do this even in -O0 to support guaranteed tail call
1269 // for symmetrical coroutine control transfer (C++ Coroutines TS extension).
1270 // This transformation is done only in the resume part of the coroutine that has
1271 // identical signature and calling convention as the coro.resume call.
1273  bool changed = false;
1274 
1275  // Collect potential resume instructions.
1277  for (auto &I : instructions(F))
1278  if (auto *Call = dyn_cast<CallInst>(&I))
1279  if (shouldBeMustTail(*Call, F))
1280  Resumes.push_back(Call);
1281 
1282  // Set musttail on those that are followed by a ret instruction.
1283  for (CallInst *Call : Resumes)
1284  if (simplifyTerminatorLeadingToRet(Call->getNextNode())) {
1285  Call->setTailCallKind(CallInst::TCK_MustTail);
1286  changed = true;
1287  }
1288 
1289  if (changed)
1291 }
1292 
1293 // Coroutine has no suspend points. Remove heap allocation for the coroutine
1294 // frame if possible.
1296  auto *CoroBegin = Shape.CoroBegin;
1297  auto *CoroId = CoroBegin->getId();
1298  auto *AllocInst = CoroId->getCoroAlloc();
1299  switch (Shape.ABI) {
1300  case coro::ABI::Switch: {
1301  auto SwitchId = cast<CoroIdInst>(CoroId);
1302  coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr);
1303  if (AllocInst) {
1304  IRBuilder<> Builder(AllocInst);
1305  auto *Frame = Builder.CreateAlloca(Shape.FrameTy);
1306  Frame->setAlignment(Shape.FrameAlign);
1307  auto *VFrame = Builder.CreateBitCast(Frame, Builder.getInt8PtrTy());
1308  AllocInst->replaceAllUsesWith(Builder.getFalse());
1309  AllocInst->eraseFromParent();
1310  CoroBegin->replaceAllUsesWith(VFrame);
1311  } else {
1312  CoroBegin->replaceAllUsesWith(CoroBegin->getMem());
1313  }
1314 
1315  break;
1316  }
1317  case coro::ABI::Async:
1318  case coro::ABI::Retcon:
1319  case coro::ABI::RetconOnce:
1320  CoroBegin->replaceAllUsesWith(UndefValue::get(CoroBegin->getType()));
1321  break;
1322  }
1323 
1324  CoroBegin->eraseFromParent();
1325 }
1326 
1327 // SimplifySuspendPoint needs to check that there is no calls between
1328 // coro_save and coro_suspend, since any of the calls may potentially resume
1329 // the coroutine and if that is the case we cannot eliminate the suspend point.
1331  for (Instruction *I = From; I != To; I = I->getNextNode()) {
1332  // Assume that no intrinsic can resume the coroutine.
1333  if (isa<IntrinsicInst>(I))
1334  continue;
1335 
1336  if (isa<CallBase>(I))
1337  return true;
1338  }
1339  return false;
1340 }
1341 
1342 static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) {
1345 
1346  Set.insert(SaveBB);
1347  Worklist.push_back(ResDesBB);
1348 
1349  // Accumulate all blocks between SaveBB and ResDesBB. Because CoroSaveIntr
1350  // returns a token consumed by suspend instruction, all blocks in between
1351  // will have to eventually hit SaveBB when going backwards from ResDesBB.
1352  while (!Worklist.empty()) {
1353  auto *BB = Worklist.pop_back_val();
1354  Set.insert(BB);
1355  for (auto *Pred : predecessors(BB))
1356  if (Set.count(Pred) == 0)
1357  Worklist.push_back(Pred);
1358  }
1359 
1360  // SaveBB and ResDesBB are checked separately in hasCallsBetween.
1361  Set.erase(SaveBB);
1362  Set.erase(ResDesBB);
1363 
1364  for (auto *BB : Set)
1365  if (hasCallsInBlockBetween(BB->getFirstNonPHI(), nullptr))
1366  return true;
1367 
1368  return false;
1369 }
1370 
1371 static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy) {
1372  auto *SaveBB = Save->getParent();
1373  auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();
1374 
1375  if (SaveBB == ResumeOrDestroyBB)
1376  return hasCallsInBlockBetween(Save->getNextNode(), ResumeOrDestroy);
1377 
1378  // Any calls from Save to the end of the block?
1379  if (hasCallsInBlockBetween(Save->getNextNode(), nullptr))
1380  return true;
1381 
1382  // Any calls from begging of the block up to ResumeOrDestroy?
1383  if (hasCallsInBlockBetween(ResumeOrDestroyBB->getFirstNonPHI(),
1384  ResumeOrDestroy))
1385  return true;
1386 
1387  // Any calls in all of the blocks between SaveBB and ResumeOrDestroyBB?
1388  if (hasCallsInBlocksBetween(SaveBB, ResumeOrDestroyBB))
1389  return true;
1390 
1391  return false;
1392 }
1393 
1394 // If a SuspendIntrin is preceded by Resume or Destroy, we can eliminate the
1395 // suspend point and replace it with nornal control flow.
1397  CoroBeginInst *CoroBegin) {
1398  Instruction *Prev = Suspend->getPrevNode();
1399  if (!Prev) {
1400  auto *Pred = Suspend->getParent()->getSinglePredecessor();
1401  if (!Pred)
1402  return false;
1403  Prev = Pred->getTerminator();
1404  }
1405 
1406  CallBase *CB = dyn_cast<CallBase>(Prev);
1407  if (!CB)
1408  return false;
1409 
1410  auto *Callee = CB->getCalledOperand()->stripPointerCasts();
1411 
1412  // See if the callsite is for resumption or destruction of the coroutine.
1413  auto *SubFn = dyn_cast<CoroSubFnInst>(Callee);
1414  if (!SubFn)
1415  return false;
1416 
1417  // Does not refer to the current coroutine, we cannot do anything with it.
1418  if (SubFn->getFrame() != CoroBegin)
1419  return false;
1420 
1421  // See if the transformation is safe. Specifically, see if there are any
1422  // calls in between Save and CallInstr. They can potenitally resume the
1423  // coroutine rendering this optimization unsafe.
1424  auto *Save = Suspend->getCoroSave();
1425  if (hasCallsBetween(Save, CB))
1426  return false;
1427 
1428  // Replace llvm.coro.suspend with the value that results in resumption over
1429  // the resume or cleanup path.
1430  Suspend->replaceAllUsesWith(SubFn->getRawIndex());
1431  Suspend->eraseFromParent();
1432  Save->eraseFromParent();
1433 
1434  // No longer need a call to coro.resume or coro.destroy.
1435  if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
1436  BranchInst::Create(Invoke->getNormalDest(), Invoke);
1437  }
1438 
1439  // Grab the CalledValue from CB before erasing the CallInstr.
1440  auto *CalledValue = CB->getCalledOperand();
1441  CB->eraseFromParent();
1442 
1443  // If no more users remove it. Usually it is a bitcast of SubFn.
1444  if (CalledValue != SubFn && CalledValue->user_empty())
1445  if (auto *I = dyn_cast<Instruction>(CalledValue))
1446  I->eraseFromParent();
1447 
1448  // Now we are good to remove SubFn.
1449  if (SubFn->user_empty())
1450  SubFn->eraseFromParent();
1451 
1452  return true;
1453 }
1454 
1455 // Remove suspend points that are simplified.
1456 static void simplifySuspendPoints(coro::Shape &Shape) {
1457  // Currently, the only simplification we do is switch-lowering-specific.
1458  if (Shape.ABI != coro::ABI::Switch)
1459  return;
1460 
1461  auto &S = Shape.CoroSuspends;
1462  size_t I = 0, N = S.size();
1463  if (N == 0)
1464  return;
1465  while (true) {
1466  auto SI = cast<CoroSuspendInst>(S[I]);
1467  // Leave final.suspend to handleFinalSuspend since it is undefined behavior
1468  // to resume a coroutine suspended at the final suspend point.
1469  if (!SI->isFinal() && simplifySuspendPoint(SI, Shape.CoroBegin)) {
1470  if (--N == I)
1471  break;
1472  std::swap(S[I], S[N]);
1473  continue;
1474  }
1475  if (++I == N)
1476  break;
1477  }
1478  S.resize(N);
1479 }
1480 
1482  SmallVectorImpl<Function *> &Clones) {
1483  assert(Shape.ABI == coro::ABI::Switch);
1484 
1485  createResumeEntryBlock(F, Shape);
1486  auto ResumeClone = createClone(F, ".resume", Shape,
1487  CoroCloner::Kind::SwitchResume);
1488  auto DestroyClone = createClone(F, ".destroy", Shape,
1489  CoroCloner::Kind::SwitchUnwind);
1490  auto CleanupClone = createClone(F, ".cleanup", Shape,
1491  CoroCloner::Kind::SwitchCleanup);
1492 
1493  postSplitCleanup(*ResumeClone);
1494  postSplitCleanup(*DestroyClone);
1495  postSplitCleanup(*CleanupClone);
1496 
1497  addMustTailToCoroResumes(*ResumeClone);
1498 
1499  // Store addresses resume/destroy/cleanup functions in the coroutine frame.
1500  updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
1501 
1502  assert(Clones.empty());
1503  Clones.push_back(ResumeClone);
1504  Clones.push_back(DestroyClone);
1505  Clones.push_back(CleanupClone);
1506 
1507  // Create a constant array referring to resume/destroy/clone functions pointed
1508  // by the last argument of @llvm.coro.info, so that CoroElide pass can
1509  // determined correct function to call.
1510  setCoroInfo(F, Shape, Clones);
1511 }
1512 
1514  Value *Continuation) {
1515  auto *ResumeIntrinsic = Suspend->getResumeFunction();
1516  auto &Context = Suspend->getParent()->getParent()->getContext();
1517  auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1518 
1519  IRBuilder<> Builder(ResumeIntrinsic);
1520  auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy);
1521  ResumeIntrinsic->replaceAllUsesWith(Val);
1522  ResumeIntrinsic->eraseFromParent();
1524  UndefValue::get(Int8PtrTy));
1525 }
1526 
1527 /// Coerce the arguments in \p FnArgs according to \p FnTy in \p CallArgs.
1529  ArrayRef<Value *> FnArgs,
1530  SmallVectorImpl<Value *> &CallArgs) {
1531  size_t ArgIdx = 0;
1532  for (auto paramTy : FnTy->params()) {
1533  assert(ArgIdx < FnArgs.size());
1534  if (paramTy != FnArgs[ArgIdx]->getType())
1535  CallArgs.push_back(
1536  Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy));
1537  else
1538  CallArgs.push_back(FnArgs[ArgIdx]);
1539  ++ArgIdx;
1540  }
1541 }
1542 
1545  IRBuilder<> &Builder) {
1546  auto *FnTy =
1547  cast<FunctionType>(MustTailCallFn->getType()->getPointerElementType());
1548  // Coerce the arguments, llvm optimizations seem to ignore the types in
1549  // vaarg functions and throws away casts in optimized mode.
1550  SmallVector<Value *, 8> CallArgs;
1551  coerceArguments(Builder, FnTy, Arguments, CallArgs);
1552 
1553  auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);
1554  TailCall->setTailCallKind(CallInst::TCK_MustTail);
1555  TailCall->setDebugLoc(Loc);
1556  TailCall->setCallingConv(MustTailCallFn->getCallingConv());
1557  return TailCall;
1558 }
1559 
1561  SmallVectorImpl<Function *> &Clones) {
1562  assert(Shape.ABI == coro::ABI::Async);
1563  assert(Clones.empty());
1564  // Reset various things that the optimizer might have decided it
1565  // "knows" about the coroutine function due to not seeing a return.
1566  F.removeFnAttr(Attribute::NoReturn);
1567  F.removeAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
1568  F.removeAttribute(AttributeList::ReturnIndex, Attribute::NonNull);
1569 
1570  auto &Context = F.getContext();
1571  auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1572 
1573  auto *Id = cast<CoroIdAsyncInst>(Shape.CoroBegin->getId());
1575 
1576  auto *FramePtr = Id->getStorage();
1577  FramePtr = Builder.CreateBitOrPointerCast(FramePtr, Int8PtrTy);
1578  FramePtr = Builder.CreateConstInBoundsGEP1_32(
1580  "async.ctx.frameptr");
1581 
1582  // Map all uses of llvm.coro.begin to the allocated frame pointer.
1583  {
1584  // Make sure we don't invalidate Shape.FramePtr.
1585  TrackingVH<Instruction> Handle(Shape.FramePtr);
1586  Shape.CoroBegin->replaceAllUsesWith(FramePtr);
1587  Shape.FramePtr = Handle.getValPtr();
1588  }
1589 
1590  // Create all the functions in order after the main function.
1591  auto NextF = std::next(F.getIterator());
1592 
1593  // Create a continuation function for each of the suspend points.
1594  Clones.reserve(Shape.CoroSuspends.size());
1595  for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1596  auto *Suspend = cast<CoroSuspendAsyncInst>(Shape.CoroSuspends[Idx]);
1597 
1598  // Create the clone declaration.
1599  auto *Continuation = createCloneDeclaration(
1600  F, Shape, ".resume." + Twine(Idx), NextF, Suspend);
1601  Clones.push_back(Continuation);
1602 
1603  // Insert a branch to a new return block immediately before the suspend
1604  // point.
1605  auto *SuspendBB = Suspend->getParent();
1606  auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1607  auto *Branch = cast<BranchInst>(SuspendBB->getTerminator());
1608 
1609  // Place it before the first suspend.
1610  auto *ReturnBB =
1611  BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
1612  Branch->setSuccessor(0, ReturnBB);
1613 
1614  IRBuilder<> Builder(ReturnBB);
1615 
1616  // Insert the call to the tail call function and inline it.
1617  auto *Fn = Suspend->getMustTailCallFunction();
1618  SmallVector<Value *, 8> Args(Suspend->args());
1619  auto FnArgs = ArrayRef<Value *>(Args).drop_front(
1621  auto *TailCall =
1622  coro::createMustTailCall(Suspend->getDebugLoc(), Fn, FnArgs, Builder);
1623  Builder.CreateRetVoid();
1624  InlineFunctionInfo FnInfo;
1625  auto InlineRes = InlineFunction(*TailCall, FnInfo);
1626  assert(InlineRes.isSuccess() && "Expected inlining to succeed");
1627  (void)InlineRes;
1628 
1629  // Replace the lvm.coro.async.resume intrisic call.
1630  replaceAsyncResumeFunction(Suspend, Continuation);
1631  }
1632 
1633  assert(Clones.size() == Shape.CoroSuspends.size());
1634  for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1635  auto *Suspend = Shape.CoroSuspends[Idx];
1636  auto *Clone = Clones[Idx];
1637 
1638  CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend).create();
1639  }
1640 }
1641 
1643  SmallVectorImpl<Function *> &Clones) {
1644  assert(Shape.ABI == coro::ABI::Retcon ||
1645  Shape.ABI == coro::ABI::RetconOnce);
1646  assert(Clones.empty());
1647 
1648  // Reset various things that the optimizer might have decided it
1649  // "knows" about the coroutine function due to not seeing a return.
1650  F.removeFnAttr(Attribute::NoReturn);
1651  F.removeAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
1652  F.removeAttribute(AttributeList::ReturnIndex, Attribute::NonNull);
1653 
1654  // Allocate the frame.
1655  auto *Id = cast<AnyCoroIdRetconInst>(Shape.CoroBegin->getId());
1656  Value *RawFramePtr;
1658  RawFramePtr = Id->getStorage();
1659  } else {
1661 
1662  // Determine the size of the frame.
1663  const DataLayout &DL = F.getParent()->getDataLayout();
1664  auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1665 
1666  // Allocate. We don't need to update the call graph node because we're
1667  // going to recompute it from scratch after splitting.
1668  // FIXME: pass the required alignment
1669  RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr);
1670  RawFramePtr =
1671  Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType());
1672 
1673  // Stash the allocated frame pointer in the continuation storage.
1674  auto Dest = Builder.CreateBitCast(Id->getStorage(),
1675  RawFramePtr->getType()->getPointerTo());
1676  Builder.CreateStore(RawFramePtr, Dest);
1677  }
1678 
1679  // Map all uses of llvm.coro.begin to the allocated frame pointer.
1680  {
1681  // Make sure we don't invalidate Shape.FramePtr.
1682  TrackingVH<Instruction> Handle(Shape.FramePtr);
1683  Shape.CoroBegin->replaceAllUsesWith(RawFramePtr);
1684  Shape.FramePtr = Handle.getValPtr();
1685  }
1686 
1687  // Create a unique return block.
1688  BasicBlock *ReturnBB = nullptr;
1689  SmallVector<PHINode *, 4> ReturnPHIs;
1690 
1691  // Create all the functions in order after the main function.
1692  auto NextF = std::next(F.getIterator());
1693 
1694  // Create a continuation function for each of the suspend points.
1695  Clones.reserve(Shape.CoroSuspends.size());
1696  for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1697  auto Suspend = cast<CoroSuspendRetconInst>(Shape.CoroSuspends[i]);
1698 
1699  // Create the clone declaration.
1700  auto Continuation =
1701  createCloneDeclaration(F, Shape, ".resume." + Twine(i), NextF, nullptr);
1702  Clones.push_back(Continuation);
1703 
1704  // Insert a branch to the unified return block immediately before
1705  // the suspend point.
1706  auto SuspendBB = Suspend->getParent();
1707  auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1708  auto Branch = cast<BranchInst>(SuspendBB->getTerminator());
1709 
1710  // Create the unified return block.
1711  if (!ReturnBB) {
1712  // Place it before the first suspend.
1713  ReturnBB = BasicBlock::Create(F.getContext(), "coro.return", &F,
1714  NewSuspendBB);
1715  Shape.RetconLowering.ReturnBlock = ReturnBB;
1716 
1717  IRBuilder<> Builder(ReturnBB);
1718 
1719  // Create PHIs for all the return values.
1720  assert(ReturnPHIs.empty());
1721 
1722  // First, the continuation.
1723  ReturnPHIs.push_back(Builder.CreatePHI(Continuation->getType(),
1724  Shape.CoroSuspends.size()));
1725 
1726  // Next, all the directly-yielded values.
1727  for (auto ResultTy : Shape.getRetconResultTypes())
1728  ReturnPHIs.push_back(Builder.CreatePHI(ResultTy,
1729  Shape.CoroSuspends.size()));
1730 
1731  // Build the return value.
1732  auto RetTy = F.getReturnType();
1733 
1734  // Cast the continuation value if necessary.
1735  // We can't rely on the types matching up because that type would
1736  // have to be infinite.
1737  auto CastedContinuationTy =
1738  (ReturnPHIs.size() == 1 ? RetTy : RetTy->getStructElementType(0));
1739  auto *CastedContinuation =
1740  Builder.CreateBitCast(ReturnPHIs[0], CastedContinuationTy);
1741 
1742  Value *RetV;
1743  if (ReturnPHIs.size() == 1) {
1744  RetV = CastedContinuation;
1745  } else {
1746  RetV = UndefValue::get(RetTy);
1747  RetV = Builder.CreateInsertValue(RetV, CastedContinuation, 0);
1748  for (size_t I = 1, E = ReturnPHIs.size(); I != E; ++I)
1749  RetV = Builder.CreateInsertValue(RetV, ReturnPHIs[I], I);
1750  }
1751 
1752  Builder.CreateRet(RetV);
1753  }
1754 
1755  // Branch to the return block.
1756  Branch->setSuccessor(0, ReturnBB);
1757  ReturnPHIs[0]->addIncoming(Continuation, SuspendBB);
1758  size_t NextPHIIndex = 1;
1759  for (auto &VUse : Suspend->value_operands())
1760  ReturnPHIs[NextPHIIndex++]->addIncoming(&*VUse, SuspendBB);
1761  assert(NextPHIIndex == ReturnPHIs.size());
1762  }
1763 
1764  assert(Clones.size() == Shape.CoroSuspends.size());
1765  for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1766  auto Suspend = Shape.CoroSuspends[i];
1767  auto Clone = Clones[i];
1768 
1769  CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend).create();
1770  }
1771 }
1772 
1773 namespace {
1774  class PrettyStackTraceFunction : public PrettyStackTraceEntry {
1775  Function &F;
1776  public:
1777  PrettyStackTraceFunction(Function &F) : F(F) {}
1778  void print(raw_ostream &OS) const override {
1779  OS << "While splitting coroutine ";
1780  F.printAsOperand(OS, /*print type*/ false, F.getParent());
1781  OS << "\n";
1782  }
1783  };
1784 }
1785 
1788  bool ReuseFrameSlot) {
1789  PrettyStackTraceFunction prettyStackTrace(F);
1790 
1791  // The suspend-crossing algorithm in buildCoroutineFrame get tripped
1792  // up by uses in unreachable blocks, so remove them as a first pass.
1794 
1795  coro::Shape Shape(F, ReuseFrameSlot);
1796  if (!Shape.CoroBegin)
1797  return Shape;
1798 
1799  simplifySuspendPoints(Shape);
1800  buildCoroutineFrame(F, Shape);
1801  replaceFrameSize(Shape);
1802 
1803  // If there are no suspend points, no split required, just remove
1804  // the allocation and deallocation blocks, they are not needed.
1805  if (Shape.CoroSuspends.empty()) {
1806  handleNoSuspendCoroutine(Shape);
1807  } else {
1808  switch (Shape.ABI) {
1809  case coro::ABI::Switch:
1810  splitSwitchCoroutine(F, Shape, Clones);
1811  break;
1812  case coro::ABI::Async:
1813  splitAsyncCoroutine(F, Shape, Clones);
1814  break;
1815  case coro::ABI::Retcon:
1816  case coro::ABI::RetconOnce:
1817  splitRetconCoroutine(F, Shape, Clones);
1818  break;
1819  }
1820  }
1821 
1822  // Replace all the swifterror operations in the original function.
1823  // This invalidates SwiftErrorOps in the Shape.
1824  replaceSwiftErrorOps(F, Shape, nullptr);
1825 
1826  return Shape;
1827 }
1828 
1829 static void
1831  const SmallVectorImpl<Function *> &Clones,
1832  CallGraph &CG, CallGraphSCC &SCC) {
1833  if (!Shape.CoroBegin)
1834  return;
1835 
1836  removeCoroEnds(Shape, &CG);
1838 
1839  // Update call graph and add the functions we created to the SCC.
1840  coro::updateCallGraph(F, Clones, CG, SCC);
1841 }
1842 
1844  LazyCallGraph::Node &N, const coro::Shape &Shape,
1848  if (!Shape.CoroBegin)
1849  return;
1850 
1851  for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) {
1852  auto &Context = End->getContext();
1853  End->replaceAllUsesWith(ConstantInt::getFalse(Context));
1854  End->eraseFromParent();
1855  }
1856 
1857  if (!Clones.empty()) {
1858  switch (Shape.ABI) {
1859  case coro::ABI::Switch:
1860  // Each clone in the Switch lowering is independent of the other clones.
1861  // Let the LazyCallGraph know about each one separately.
1862  for (Function *Clone : Clones)
1863  CG.addSplitFunction(N.getFunction(), *Clone);
1864  break;
1865  case coro::ABI::Async:
1866  case coro::ABI::Retcon:
1867  case coro::ABI::RetconOnce:
1868  // Each clone in the Async/Retcon lowering references of the other clones.
1869  // Let the LazyCallGraph know about all of them at once.
1870  if (!Clones.empty())
1871  CG.addSplitRefRecursiveFunctions(N.getFunction(), Clones);
1872  break;
1873  }
1874 
1875  // Let the CGSCC infra handle the changes to the original function.
1877  }
1878 
1879  // Do some cleanup and let the CGSCC infra see if we've cleaned up any edges
1880  // to the split functions.
1881  postSplitCleanup(N.getFunction());
1883 }
1884 
1885 // When we see the coroutine the first time, we insert an indirect call to a
1886 // devirt trigger function and mark the coroutine that it is now ready for
1887 // split.
1888 // Async lowering uses this after it has split the function to restart the
1889 // pipeline.
1891  bool MarkForAsyncRestart = false) {
1892  Module &M = *F.getParent();
1893  LLVMContext &Context = F.getContext();
1894 #ifndef NDEBUG
1895  Function *DevirtFn = M.getFunction(CORO_DEVIRT_TRIGGER_FN);
1896  assert(DevirtFn && "coro.devirt.trigger function not found");
1897 #endif
1898 
1899  F.addFnAttr(CORO_PRESPLIT_ATTR, MarkForAsyncRestart
1901  : PREPARED_FOR_SPLIT);
1902 
1903  // Insert an indirect call sequence that will be devirtualized by CoroElide
1904  // pass:
1905  // %0 = call i8* @llvm.coro.subfn.addr(i8* null, i8 -1)
1906  // %1 = bitcast i8* %0 to void(i8*)*
1907  // call void %1(i8* null)
1908  coro::LowererBase Lowerer(M);
1909  Instruction *InsertPt =
1910  MarkForAsyncRestart ? F.getEntryBlock().getFirstNonPHIOrDbgOrLifetime()
1911  : F.getEntryBlock().getTerminator();
1913  auto *DevirtFnAddr =
1914  Lowerer.makeSubFnCall(Null, CoroSubFnInst::RestartTrigger, InsertPt);
1916  {Type::getInt8PtrTy(Context)}, false);
1917  auto *IndirectCall = CallInst::Create(FnTy, DevirtFnAddr, Null, "", InsertPt);
1918 
1919  // Update CG graph with an indirect call we just added.
1920  CG[&F]->addCalledFunction(IndirectCall, CG.getCallsExternalNode());
1921 }
1922 
1923 // Make sure that there is a devirtualization trigger function that the
1924 // coro-split pass uses to force a restart of the CGSCC pipeline. If the devirt
1925 // trigger function is not found, we will create one and add it to the current
1926 // SCC.
1928  Module &M = CG.getModule();
1929  if (M.getFunction(CORO_DEVIRT_TRIGGER_FN))
1930  return;
1931 
1932  LLVMContext &C = M.getContext();
1934  /*isVarArg=*/false);
1935  Function *DevirtFn =
1936  Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
1938  DevirtFn->addFnAttr(Attribute::AlwaysInline);
1939  auto *Entry = BasicBlock::Create(C, "entry", DevirtFn);
1940  ReturnInst::Create(C, Entry);
1941 
1942  auto *Node = CG.getOrInsertFunction(DevirtFn);
1943 
1944  SmallVector<CallGraphNode *, 8> Nodes(SCC.begin(), SCC.end());
1945  Nodes.push_back(Node);
1946  SCC.initialize(Nodes);
1947 }
1948 
1949 /// Replace a call to llvm.coro.prepare.retcon.
1950 static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG,
1951  LazyCallGraph::SCC &C) {
1952  auto CastFn = Prepare->getArgOperand(0); // as an i8*
1953  auto Fn = CastFn->stripPointerCasts(); // as its original type
1954 
1955  // Attempt to peephole this pattern:
1956  // %0 = bitcast [[TYPE]] @some_function to i8*
1957  // %1 = call @llvm.coro.prepare.retcon(i8* %0)
1958  // %2 = bitcast %1 to [[TYPE]]
1959  // ==>
1960  // %2 = @some_function
1961  for (auto UI = Prepare->use_begin(), UE = Prepare->use_end(); UI != UE;) {
1962  // Look for bitcasts back to the original function type.
1963  auto *Cast = dyn_cast<BitCastInst>((UI++)->getUser());
1964  if (!Cast || Cast->getType() != Fn->getType())
1965  continue;
1966 
1967  // Replace and remove the cast.
1968  Cast->replaceAllUsesWith(Fn);
1969  Cast->eraseFromParent();
1970  }
1971 
1972  // Replace any remaining uses with the function as an i8*.
1973  // This can never directly be a callee, so we don't need to update CG.
1974  Prepare->replaceAllUsesWith(CastFn);
1975  Prepare->eraseFromParent();
1976 
1977  // Kill dead bitcasts.
1978  while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
1979  if (!Cast->use_empty())
1980  break;
1981  CastFn = Cast->getOperand(0);
1982  Cast->eraseFromParent();
1983  }
1984 }
1985 /// Replace a call to llvm.coro.prepare.retcon.
1986 static void replacePrepare(CallInst *Prepare, CallGraph &CG) {
1987  auto CastFn = Prepare->getArgOperand(0); // as an i8*
1988  auto Fn = CastFn->stripPointerCasts(); // as its original type
1989 
1990  // Find call graph nodes for the preparation.
1991  CallGraphNode *PrepareUserNode = nullptr, *FnNode = nullptr;
1992  if (auto ConcreteFn = dyn_cast<Function>(Fn)) {
1993  PrepareUserNode = CG[Prepare->getFunction()];
1994  FnNode = CG[ConcreteFn];
1995  }
1996 
1997  // Attempt to peephole this pattern:
1998  // %0 = bitcast [[TYPE]] @some_function to i8*
1999  // %1 = call @llvm.coro.prepare.retcon(i8* %0)
2000  // %2 = bitcast %1 to [[TYPE]]
2001  // ==>
2002  // %2 = @some_function
2003  for (auto UI = Prepare->use_begin(), UE = Prepare->use_end();
2004  UI != UE; ) {
2005  // Look for bitcasts back to the original function type.
2006  auto *Cast = dyn_cast<BitCastInst>((UI++)->getUser());
2007  if (!Cast || Cast->getType() != Fn->getType()) continue;
2008 
2009  // Check whether the replacement will introduce new direct calls.
2010  // If so, we'll need to update the call graph.
2011  if (PrepareUserNode) {
2012  for (auto &Use : Cast->uses()) {
2013  if (auto *CB = dyn_cast<CallBase>(Use.getUser())) {
2014  if (!CB->isCallee(&Use))
2015  continue;
2016  PrepareUserNode->removeCallEdgeFor(*CB);
2017  PrepareUserNode->addCalledFunction(CB, FnNode);
2018  }
2019  }
2020  }
2021 
2022  // Replace and remove the cast.
2023  Cast->replaceAllUsesWith(Fn);
2024  Cast->eraseFromParent();
2025  }
2026 
2027  // Replace any remaining uses with the function as an i8*.
2028  // This can never directly be a callee, so we don't need to update CG.
2029  Prepare->replaceAllUsesWith(CastFn);
2030  Prepare->eraseFromParent();
2031 
2032  // Kill dead bitcasts.
2033  while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
2034  if (!Cast->use_empty()) break;
2035  CastFn = Cast->getOperand(0);
2036  Cast->eraseFromParent();
2037  }
2038 }
2039 
2040 static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG,
2041  LazyCallGraph::SCC &C) {
2042  bool Changed = false;
2043  for (auto PI = PrepareFn->use_begin(), PE = PrepareFn->use_end(); PI != PE;) {
2044  // Intrinsics can only be used in calls.
2045  auto *Prepare = cast<CallInst>((PI++)->getUser());
2046  replacePrepare(Prepare, CG, C);
2047  Changed = true;
2048  }
2049 
2050  return Changed;
2051 }
2052 
2053 /// Remove calls to llvm.coro.prepare.retcon, a barrier meant to prevent
2054 /// IPO from operating on calls to a retcon coroutine before it's been
2055 /// split. This is only safe to do after we've split all retcon
2056 /// coroutines in the module. We can do that this in this pass because
2057 /// this pass does promise to split all retcon coroutines (as opposed to
2058 /// switch coroutines, which are lowered in multiple stages).
2059 static bool replaceAllPrepares(Function *PrepareFn, CallGraph &CG) {
2060  bool Changed = false;
2061  for (auto PI = PrepareFn->use_begin(), PE = PrepareFn->use_end();
2062  PI != PE; ) {
2063  // Intrinsics can only be used in calls.
2064  auto *Prepare = cast<CallInst>((PI++)->getUser());
2065  replacePrepare(Prepare, CG);
2066  Changed = true;
2067  }
2068 
2069  return Changed;
2070 }
2071 
2072 static bool declaresCoroSplitIntrinsics(const Module &M) {
2073  return coro::declaresIntrinsics(M, {"llvm.coro.begin",
2074  "llvm.coro.prepare.retcon",
2075  "llvm.coro.prepare.async"});
2076 }
2077 
2078 static void addPrepareFunction(const Module &M,
2080  StringRef Name) {
2081  auto *PrepareFn = M.getFunction(Name);
2082  if (PrepareFn && !PrepareFn->use_empty())
2083  Fns.push_back(PrepareFn);
2084 }
2085 
2088  LazyCallGraph &CG, CGSCCUpdateResult &UR) {
2089  // NB: One invariant of a valid LazyCallGraph::SCC is that it must contain a
2090  // non-zero number of nodes, so we assume that here and grab the first
2091  // node's function's module.
2092  Module &M = *C.begin()->getFunction().getParent();
2093  auto &FAM =
2094  AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
2095 
2097  return PreservedAnalyses::all();
2098 
2099  // Check for uses of llvm.coro.prepare.retcon/async.
2100  SmallVector<Function *, 2> PrepareFns;
2101  addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
2102  addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
2103 
2104  // Find coroutines for processing.
2106  for (LazyCallGraph::Node &N : C)
2107  if (N.getFunction().hasFnAttribute(CORO_PRESPLIT_ATTR))
2108  Coroutines.push_back(&N);
2109 
2110  if (Coroutines.empty() && PrepareFns.empty())
2111  return PreservedAnalyses::all();
2112 
2113  if (Coroutines.empty()) {
2114  for (auto *PrepareFn : PrepareFns) {
2115  replaceAllPrepares(PrepareFn, CG, C);
2116  }
2117  }
2118 
2119  // Split all the coroutines.
2120  for (LazyCallGraph::Node *N : Coroutines) {
2121  Function &F = N->getFunction();
2122  Attribute Attr = F.getFnAttribute(CORO_PRESPLIT_ATTR);
2123  StringRef Value = Attr.getValueAsString();
2124  LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()
2125  << "' state: " << Value << "\n");
2126  if (Value == UNPREPARED_FOR_SPLIT) {
2127  // Enqueue a second iteration of the CGSCC pipeline on this SCC.
2128  UR.CWorklist.insert(&C);
2130  continue;
2131  }
2132  F.removeFnAttr(CORO_PRESPLIT_ATTR);
2133 
2135  const coro::Shape Shape = splitCoroutine(F, Clones, ReuseFrameSlot);
2136  updateCallGraphAfterCoroutineSplit(*N, Shape, Clones, C, CG, AM, UR, FAM);
2137 
2138  if ((Shape.ABI == coro::ABI::Async || Shape.ABI == coro::ABI::Retcon ||
2139  Shape.ABI == coro::ABI::RetconOnce) &&
2140  !Shape.CoroSuspends.empty()) {
2141  // Run the CGSCC pipeline on the newly split functions.
2142  // All clones will be in the same RefSCC, so choose a random clone.
2143  UR.RCWorklist.insert(CG.lookupRefSCC(CG.get(*Clones[0])));
2144  }
2145  }
2146 
2147  if (!PrepareFns.empty()) {
2148  for (auto *PrepareFn : PrepareFns) {
2149  replaceAllPrepares(PrepareFn, CG, C);
2150  }
2151  }
2152 
2153  return PreservedAnalyses::none();
2154 }
2155 
2156 namespace {
2157 
2158 // We present a coroutine to LLVM as an ordinary function with suspension
2159 // points marked up with intrinsics. We let the optimizer party on the coroutine
2160 // as a single function for as long as possible. Shortly before the coroutine is
2161 // eligible to be inlined into its callers, we split up the coroutine into parts
2162 // corresponding to initial, resume and destroy invocations of the coroutine,
2163 // add them to the current SCC and restart the IPO pipeline to optimize the
2164 // coroutine subfunctions we extracted before proceeding to the caller of the
2165 // coroutine.
2166 struct CoroSplitLegacy : public CallGraphSCCPass {
2167  static char ID; // Pass identification, replacement for typeid
2168 
2169  CoroSplitLegacy(bool ReuseFrameSlot = false)
2170  : CallGraphSCCPass(ID), ReuseFrameSlot(ReuseFrameSlot) {
2172  }
2173 
2174  bool Run = false;
2175  bool ReuseFrameSlot;
2176 
2177  // A coroutine is identified by the presence of coro.begin intrinsic, if
2178  // we don't have any, this pass has nothing to do.
2179  bool doInitialization(CallGraph &CG) override {
2182  }
2183 
2184  bool runOnSCC(CallGraphSCC &SCC) override {
2185  if (!Run)
2186  return false;
2187 
2188  // Check for uses of llvm.coro.prepare.retcon.
2189  SmallVector<Function *, 2> PrepareFns;
2190  auto &M = SCC.getCallGraph().getModule();
2191  addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
2192  addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
2193 
2194  // Find coroutines for processing.
2195  SmallVector<Function *, 4> Coroutines;
2196  for (CallGraphNode *CGN : SCC)
2197  if (auto *F = CGN->getFunction())
2198  if (F->hasFnAttribute(CORO_PRESPLIT_ATTR))
2199  Coroutines.push_back(F);
2200 
2201  if (Coroutines.empty() && PrepareFns.empty())
2202  return false;
2203 
2204  CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
2205 
2206  if (Coroutines.empty()) {
2207  bool Changed = false;
2208  for (auto *PrepareFn : PrepareFns)
2209  Changed |= replaceAllPrepares(PrepareFn, CG);
2210  return Changed;
2211  }
2212 
2214 
2215  // Split all the coroutines.
2216  for (Function *F : Coroutines) {
2217  Attribute Attr = F->getFnAttribute(CORO_PRESPLIT_ATTR);
2218  StringRef Value = Attr.getValueAsString();
2219  LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F->getName()
2220  << "' state: " << Value << "\n");
2221  // Async lowering marks coroutines to trigger a restart of the pipeline
2222  // after it has split them.
2224  F->removeFnAttr(CORO_PRESPLIT_ATTR);
2225  continue;
2226  }
2227  if (Value == UNPREPARED_FOR_SPLIT) {
2228  prepareForSplit(*F, CG);
2229  continue;
2230  }
2231  F->removeFnAttr(CORO_PRESPLIT_ATTR);
2232 
2234  const coro::Shape Shape = splitCoroutine(*F, Clones, ReuseFrameSlot);
2235  updateCallGraphAfterCoroutineSplit(*F, Shape, Clones, CG, SCC);
2236  if (Shape.ABI == coro::ABI::Async) {
2237  // Restart SCC passes.
2238  // Mark function for CoroElide pass. It will devirtualize causing a
2239  // restart of the SCC pipeline.
2240  prepareForSplit(*F, CG, true /*MarkForAsyncRestart*/);
2241  }
2242  }
2243 
2244  for (auto *PrepareFn : PrepareFns)
2245  replaceAllPrepares(PrepareFn, CG);
2246 
2247  return true;
2248  }
2249 
2250  void getAnalysisUsage(AnalysisUsage &AU) const override {
2252  }
2253 
2254  StringRef getPassName() const override { return "Coroutine Splitting"; }
2255 };
2256 
2257 } // end anonymous namespace
2258 
2259 char CoroSplitLegacy::ID = 0;
2260 
2262  CoroSplitLegacy, "coro-split",
2263  "Split coroutine into a set of functions driving its state machine", false,
2264  false)
2267  CoroSplitLegacy, "coro-split",
2268  "Split coroutine into a set of functions driving its state machine", false,
2269  false)
2270 
2271 Pass *llvm::createCoroSplitLegacyPass(bool ReuseFrameSlot) {
2272  return new CoroSplitLegacy(ReuseFrameSlot);
2273 }
llvm::coro::Shape::CoroSizes
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:101
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
postSplitCleanup
static void postSplitCleanup(Function &F)
Definition: CoroSplit.cpp:1129
i
i
Definition: README.txt:29
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
set
We currently generate a but we really shouldn eax ecx xorl edx divl ecx eax divl ecx movl eax ret A similar code sequence works for division We currently compile i32 v2 eax eax jo LBB1_2 atomic and others It is also currently not done for read modify write instructions It is also current not done if the OF or CF flags are needed The shift operators have the complication that when the shift count is EFLAGS is not set
Definition: README.txt:1277
functions
amdgpu propagate attributes Late propagate attributes from kernels to functions
Definition: AMDGPUPropagateAttributes.cpp:216
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
llvm::coro::createMustTailCall
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, ArrayRef< Value * > Arguments, IRBuilder<> &)
Definition: CoroSplit.cpp:1543
addPrepareFunction
static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)
Definition: CoroSplit.cpp:2078
Attrs
Function Attrs
Definition: README_ALTIVEC.txt:215
llvm::predecessors
pred_range predecessors(BasicBlock *BB)
Definition: CFG.h:127
llvm::coro::Shape::AsyncLoweringStorage::FrameOffset
uint64_t FrameOffset
Definition: CoroInternal.h:156
PREPARED_FOR_SPLIT
#define PREPARED_FOR_SPLIT
Definition: CoroInternal.h:41
createCloneDeclaration
static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)
Definition: CoroSplit.cpp:468
shouldBeMustTail
static bool shouldBeMustTail(const CallInst &CI, const Function &F)
Definition: CoroSplit.cpp:1237
llvm
Definition: AllocatorList.h:23
llvm::DILocalScope::getSubprogram
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
Definition: DebugInfoMetadata.cpp:812
llvm::coro::salvageDebugInfo
void salvageDebugInfo(SmallDenseMap< llvm::Value *, llvm::AllocaInst *, 4 > &DbgPtrAllocaCache, DbgVariableIntrinsic *DVI, bool ReuseFrameSlot)
Recover a dbg.declare prepared by the frontend and emit an alloca holding a pointer to the coroutine ...
Definition: CoroFrame.cpp:2440
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
updateCoroFrame
static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn, Function *DestroyFn, Function *CleanupFn)
Definition: CoroSplit.cpp:1104
llvm::CmpInst::ICMP_EQ
@ ICMP_EQ
equal
Definition: InstrTypes.h:743
llvm::ReturnInst
Return a value (possibly void), from a function.
Definition: Instructions.h:2953
llvm::SmallPtrSetImpl::erase
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:378
ValueMapper.h
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:112
llvm::AnyCoroSuspendInst
Definition: CoroInstr.h:477
print
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
Definition: ArchiveWriter.cpp:147
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:249
llvm::coro::Shape::SwitchLowering
SwitchLoweringStorage SwitchLowering
Definition: CoroInternal.h:164
llvm::initializeCoroSplitLegacyPass
void initializeCoroSplitLegacyPass(PassRegistry &)
updateCallGraphAfterCoroutineSplit
static void updateCallGraphAfterCoroutineSplit(Function &F, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, CallGraph &CG, CallGraphSCC &SCC)
Definition: CoroSplit.cpp:1830
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:107
IntrinsicInst.h
llvm::Type::isPointerTy
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:229
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:769
llvm::Module::iterator
FunctionListType::iterator iterator
The Function iterators.
Definition: Module.h:92
Scalar.h
llvm::coro::Shape::FrameTy
StructType * FrameTy
Definition: CoroInternal.h:122
InstIterator.h
llvm::Function
Definition: Function.h:61
getFunction
static Function * getFunction(Constant *C)
Definition: Evaluator.cpp:253
llvm::Attribute
Definition: Attributes.h:52
StringRef.h
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1325
Pass.h
llvm::TrackingVH::getValPtr
ValueTy * getValPtr() const
Definition: ValueHandle.h:335
it
Reference model for inliner Oz decision policy Note this model is also referenced by test Transforms Inline ML tests if replacing it
Definition: README.txt:3
llvm::createCFGSimplificationPass
FunctionPass * createCFGSimplificationPass(SimplifyCFGOptions Options=SimplifyCFGOptions(), std::function< bool(const Function &)> Ftor=nullptr)
Definition: SimplifyCFGPass.cpp:384
addFramePointerAttrs
static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment)
Definition: CoroSplit.cpp:812
hasCallsInBlockBetween
static bool hasCallsInBlockBetween(Instruction *From, Instruction *To)
Definition: CoroSplit.cpp:1330
removeCoroEnds
static void removeCoroEnds(const coro::Shape &Shape, CallGraph *CG)
Remove calls to llvm.coro.end in the original function.
Definition: CoroSplit.cpp:1028
llvm::ilist_node_with_parent::getNextNode
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:288
llvm::SmallVector< Value *, 8 >
llvm::CallGraphNode::removeCallEdgeFor
void removeCallEdgeFor(CallBase &Call)
Removes the edge in the node for the specified call site.
Definition: CallGraph.cpp:214
llvm::CallBase::isInlineAsm
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1461
prepareForSplit
static void prepareForSplit(Function &F, CallGraph &CG, bool MarkForAsyncRestart=false)
Definition: CoroSplit.cpp:1890
llvm::Type::getPointerAddressSpace
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:728
llvm::coro::ABI::Retcon
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:40
llvm::SmallDenseMap
Definition: DenseMap.h:880
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:321
Local.h
llvm::CallGraph
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:73
llvm::CoroSplitPass::ReuseFrameSlot
bool ReuseFrameSlot
Definition: CoroSplit.h:32
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
FAM
FunctionAnalysisManager FAM
Definition: PassBuilderBindings.cpp:59
llvm::createEarlyCSEPass
FunctionPass * createEarlyCSEPass(bool UseMemorySSA=false)
Definition: EarlyCSE.cpp:1719
llvm::verifyFunction
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
Definition: Verifier.cpp:5782
hasCallsBetween
static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)
Definition: CoroSplit.cpp:1371
llvm::coro::ABI::Switch
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
llvm::PreservedAnalyses::none
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:158
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:327
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
scanPHIsAndUpdateValueMap
static void scanPHIsAndUpdateValueMap(Instruction *Prev, BasicBlock *NewBlock, DenseMap< Value *, Value * > &ResolvedValues)
Definition: CoroSplit.cpp:1153
llvm::coro::LowererBase::makeSubFnCall
Value * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
Definition: Coroutines.cpp:107
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
llvm::coro::Shape::ABI
coro::ABI ABI
Definition: CoroInternal.h:120
DenseMap.h
llvm::removeUnreachableBlocks
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:2443
Module.h
llvm::CoroSuspendAsyncInst::getResumeFunction
CoroAsyncResumeInst * getResumeFunction() const
Definition: CoroInstr.h:546
llvm::AttributeList
Definition: Attributes.h:385
llvm::CoroBeginInst::getId
AnyCoroIdInst * getId() const
Definition: CoroInstr.h:424
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1471
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1244
llvm::coro::declaresIntrinsics
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:161
replaceCoroEndAsync
static bool replaceCoroEndAsync(AnyCoroEndInst *End)
Replace an llvm.coro.end.async.
Definition: CoroSplit.cpp:183
llvm::BasicBlock::splitBasicBlock
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:381
llvm::AnyCoroIdInst::getCoroAlloc
CoroAllocInst * getCoroAlloc()
Definition: CoroInstr.h:84
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
llvm::CloneFunctionChangeType::LocalChangesOnly
@ LocalChangesOnly
declaresCoroSplitIntrinsics
static bool declaresCoroSplitIntrinsics(const Module &M)
Definition: CoroSplit.cpp:2072
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:892
handleNoSuspendCoroutine
static void handleNoSuspendCoroutine(coro::Shape &Shape)
Definition: CoroSplit.cpp:1295
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:138
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:264
llvm::SmallVectorImpl::pop_back_val
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:634
llvm::coro::Shape::SwitchLoweringStorage::ResumeSwitch
SwitchInst * ResumeSwitch
Definition: CoroInternal.h:132
replaceCoroEnd
static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Definition: CoroSplit.cpp:314
llvm::coro::buildCoroutineFrame
void buildCoroutineFrame(Function &F, Shape &Shape)
Definition: CoroFrame.cpp:2526
llvm::Function::addFnAttr
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
Definition: Function.h:255
LegacyPassManager.h
llvm::CallInst::TCK_MustTail
@ TCK_MustTail
Definition: Instructions.h:1658
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:195
llvm::CallGraphNode::addCalledFunction
void addCalledFunction(CallBase *Call, CallGraphNode *M)
Adds a function to the list of functions called by this one.
Definition: CallGraph.h:243
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:122
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::CoroSuspendAsyncInst::MustTailCallFuncArg
@ MustTailCallFuncArg
Definition: CoroInstr.h:531
llvm::ConstantExpr::getPointerCast
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition: Constants.cpp:2032
Uses
SmallPtrSet< MachineInstr *, 2 > Uses
Definition: ARMLowOverheadLoops.cpp:583
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
a
=0.0 ? 0.0 :(a > 0.0 ? 1.0 :-1.0) a
Definition: README.txt:489
llvm::coro::Shape::emitDealloc
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:552
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
llvm::PseudoProbeType::IndirectCall
@ IndirectCall
llvm::CallGraph::getOrInsertFunction
CallGraphNode * getOrInsertFunction(const Function *F)
Similar to operator[], but this will insert a new CallGraphNode for F if one does not already exist.
Definition: CallGraph.cpp:175
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:206
llvm::PrettyStackTraceEntry
PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...
Definition: PrettyStackTrace.h:52
updateAsyncFuncPointerContextSize
static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)
Definition: CoroSplit.cpp:1034
addAsyncContextAttrs
static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:823
Instruction.h
llvm::CoroIdInst::setInfo
void setInfo(Constant *C)
Definition: CoroInstr.h:180
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:77
llvm::Intrinsic::getType
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
Definition: Function.cpp:1285
llvm::coro::Shape::RetconLoweringStorage::IsFrameInlineInStorage
bool IsFrameInlineInStorage
Definition: CoroInternal.h:146
GlobalValue.h
llvm::CallGraphSCCPass::doInitialization
virtual bool doInitialization(CallGraph &CG)
doInitialization - This method is called before the SCC's of the program has been processed,...
Definition: CallGraphSCCPass.h:48
llvm::CallGraphSCC
CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on.
Definition: CallGraphSCCPass.h:87
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::LazyCallGraph::SCC
An SCC of the call graph.
Definition: LazyCallGraph.h:421
llvm::coro::Shape::AsyncLoweringStorage::AsyncFuncPointer
GlobalVariable * AsyncFuncPointer
Definition: CoroInternal.h:158
llvm::legacy::FunctionPassManager::doFinalization
bool doFinalization()
doFinalization - Run all of the finalizers for the function passes.
Definition: LegacyPassManager.cpp:1368
Constants.h
splitAsyncCoroutine
static void splitAsyncCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1560
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::User
Definition: User.h:44
createClone
static Function * createClone(Function &F, const Twine &Suffix, coro::Shape &Shape, CoroCloner::Kind FKind)
Definition: CoroSplit.cpp:1020
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::ARM_PROC::A
@ A
Definition: ARMBaseInfo.h:34
Twine.h
InstrTypes.h
llvm::coro::Shape::getRetconResultTypes
ArrayRef< Type * > getRetconResultTypes() const
Definition: CoroInternal.h:222
llvm::CallInst::Create
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Definition: Instructions.h:1521
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::CoroSuspendInst
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:493
llvm::ReplaceInstWithInst
void ReplaceInstWithInst(BasicBlock::InstListType &BIL, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Definition: BasicBlockUtils.cpp:470
splitRetconCoroutine
static void splitRetconCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1642
llvm::Value::uses
iterator_range< use_iterator > uses()
Definition: Value.h:377
false
Definition: StackSlotColoring.cpp:142
llvm::coro::Shape::CoroSuspends
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:102
llvm::isPotentiallyReachable
bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)
Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...
Definition: CFG.cpp:236
llvm::coro::Shape::CoroEnds
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:100
llvm::coro::replaceCoroFree
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:174
llvm::Instruction
Definition: Instruction.h:45
InlineInfo
@ InlineInfo
Definition: FunctionInfo.cpp:24
llvm::report_fatal_error
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::BasicBlock::phis
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:354
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:50
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1783
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:898
llvm::Use::getUser
User * getUser() const
Returns the User that contains this Use.
Definition: Use.h:73
llvm::FunctionType::params
ArrayRef< Type * > params() const
Definition: DerivedTypes.h:129
SmallPtrSet.h
llvm::CallGraphNode
A node in the call graph for a module.
Definition: CallGraph.h:167
splitSwitchCoroutine
static void splitSwitchCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1481
LazyCallGraph.h
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1452
llvm::Attribute::getValueAsString
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:304
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
llvm::AttrBuilder::addAlignmentAttr
AttrBuilder & addAlignmentAttr(MaybeAlign Align)
This turns an alignment into the form used internally in Attribute.
Definition: Attributes.cpp:1845
Type.h
llvm::coro::Shape::FrameAlign
Align FrameAlign
Definition: CoroInternal.h:123
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
ASYNC_RESTART_AFTER_SPLIT
#define ASYNC_RESTART_AFTER_SPLIT
Definition: CoroInternal.h:42
CFG.h
llvm::LazyCallGraph::get
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
Definition: LazyCallGraph.h:974
llvm::coro::Shape::getIndexType
IntegerType * getIndexType() const
Definition: CoroInternal.h:190
into
Clang compiles this into
Definition: README.txt:504
llvm::coro::Shape
Definition: CoroInternal.h:98
llvm::coro::Shape::AsyncLowering
AsyncLoweringStorage AsyncLowering
Definition: CoroInternal.h:166
CoroInternal.h
BasicBlock.h
hasCallsInBlocksBetween
static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)
Definition: CoroSplit.cpp:1342
llvm::DbgVariableIntrinsic
This is the common base class for debug info intrinsics for variables.
Definition: IntrinsicInst.h:148
llvm::AttributeList::ReturnIndex
@ ReturnIndex
Definition: Attributes.h:388
llvm::coro::Shape::SwiftErrorOps
SmallVector< CallInst *, 2 > SwiftErrorOps
Definition: CoroInternal.h:103
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
setCoroInfo
static void setCoroInfo(Function &F, coro::Shape &Shape, ArrayRef< Function * > Fns)
Definition: CoroSplit.cpp:1080
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition: GlobalVariable.h:136
VI
@ VI
Definition: SIInstrInfo.cpp:7494
llvm::ArrayRef::drop_front
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:204
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:78
simplifySuspendPoints
static void simplifySuspendPoints(coro::Shape &Shape)
Definition: CoroSplit.cpp:1456
llvm::CoroSplitPass::run
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: CoroSplit.cpp:2086
llvm::CallGraphSCCPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &Info) const override
getAnalysisUsage - For this class, we declare that we require and preserve the call graph.
Definition: CallGraphSCCPass.cpp:659
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
llvm::updateCGAndAnalysisManagerForFunctionPass
LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a function pass.
Definition: CGSCCPassManager.cpp:1221
llvm::Function::getCallingConv
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:238
llvm::LazyCallGraph::addSplitFunction
void addSplitFunction(Function &OriginalFunction, Function &NewFunction)
Add a new function split/outlined from an existing function.
Definition: LazyCallGraph.cpp:1612
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:572
llvm::LazyCallGraph::lookupRefSCC
RefSCC * lookupRefSCC(Node &N) const
Lookup a function's RefSCC in the graph.
Definition: LazyCallGraph.h:965
CoroSplit.h
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
llvm::ConstantPointerNull::get
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1769
llvm::changeToUnreachable
unsigned changeToUnreachable(Instruction *I, bool UseLLVMTrap, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2096
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::BranchInst::Create
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3091
llvm::DenseMap
Definition: DenseMap.h:714
coerceArguments
static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)
Coerce the arguments in FnArgs according to FnTy in CallArgs.
Definition: CoroSplit.cpp:1528
llvm::CallGraphWrapperPass
The ModulePass which wraps up a CallGraph and the logic to build it.
Definition: CallGraph.h:337
llvm::CoroSuspendAsyncInst
This represents the llvm.coro.suspend.async instruction.
Definition: CoroInstr.h:525
PrettyStackTrace.h
CORO_DEVIRT_TRIGGER_FN
#define CORO_DEVIRT_TRIGGER_FN
Definition: CoroInternal.h:44
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::AttrBuilder
Definition: Attributes.h:804
Cloning.h
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:134
llvm::CGSCCUpdateResult::CWorklist
SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist
Worklist of the SCCs queued for processing.
Definition: CGSCCPassManager.h:257
llvm::Attribute::AttrKind
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
Definition: Attributes.h:71
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:634
llvm::legacy::FunctionPassManager::doInitialization
bool doInitialization()
doInitialization - Run all of the initializers for the function passes.
Definition: LegacyPassManager.cpp:1362
addMustTailToCoroResumes
static void addMustTailToCoroResumes(Function &F)
Definition: CoroSplit.cpp:1272
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(CoroSplitLegacy, "coro-split", "Split coroutine into a set of functions driving its state machine", false, false) INITIALIZE_PASS_END(CoroSplitLegacy
llvm::TrackingVH
Value handle that tracks a Value across RAUW.
Definition: ValueHandle.h:331
llvm::coro::Shape::FramePtr
Instruction * FramePtr
Definition: CoroInternal.h:125
llvm::MDString::get
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:467
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::find
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:137
llvm::ilist_node_with_parent::getPrevNode
NodeTy * getPrevNode()
Definition: ilist_node.h:274
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::Function::hasParamAttribute
bool hasParamAttribute(unsigned ArgNo, Attribute::AttrKind Kind) const
check if an attributes is in the list of attributes.
Definition: Function.h:449
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:840
llvm::coro::Shape::SwitchFieldIndex::Destroy
@ Destroy
Definition: CoroInternal.h:109
llvm::CoroSuspendAsyncInst::ResumeFunctionArg
@ ResumeFunctionArg
Definition: CoroInstr.h:529
SI
StandardInstrumentations SI(Debug, VerifyEach)
replaceUnwindCoroEnd
static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace an unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:284
llvm::coro::ABI::Async
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
llvm::Value::use_begin
use_iterator use_begin()
Definition: Value.h:361
UNPREPARED_FOR_SPLIT
#define UNPREPARED_FOR_SPLIT
Definition: CoroInternal.h:40
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::ArrayType::get
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:598
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
llvm::legacy::FunctionPassManager::add
void add(Pass *P) override
Add a pass to the queue of passes to run.
Definition: LegacyPassManager.cpp:1344
llvm::LazyCallGraph::Node
A node in the call graph.
Definition: LazyCallGraph.h:317
llvm::SmallPtrSetImpl::count
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:382
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::createSCCPPass
FunctionPass * createSCCPPass()
Definition: SCCP.cpp:294
llvm::User::setOperand
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
llvm::coro::Shape::AsyncLoweringStorage::ContextSize
uint64_t ContextSize
Definition: CoroInternal.h:157
replaceAsyncResumeFunction
static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)
Definition: CoroSplit.cpp:1513
llvm::AMDGPU::CPol::SCC
@ SCC
Definition: SIDefines.h:285
CFG.h
replaceFallthroughCoroEnd
static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace a non-unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:226
simplifyTerminatorLeadingToRet
static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst)
Definition: CoroSplit.cpp:1170
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
IT
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::ZeroOrMore, cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate IT block based on arch"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow deprecated IT based on ARMv8"), clEnumValN(NoRestrictedIT, "arm-no-restrict-it", "Allow IT blocks based on ARMv7")))
DataLayout.h
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:167
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
replaceFrameSize
static void replaceFrameSize(coro::Shape &Shape)
Definition: CoroSplit.cpp:1049
createResumeEntryBlock
static void createResumeEntryBlock(Function &F, coro::Shape &Shape)
Definition: CoroSplit.cpp:329
replaceSwiftErrorOps
static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)
Definition: CoroSplit.cpp:588
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:256
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:70
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:517
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:100
llvm::Value::use_end
use_iterator use_end()
Definition: Value.h:369
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::coro::Shape::CoroBegin
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:99
llvm::CoroSubFnInst::RestartTrigger
@ RestartTrigger
Definition: CoroInstr.h:40
llvm::coro::Shape::getResumeFunctionType
FunctionType * getResumeFunctionType() const
Definition: CoroInternal.h:205
CallGraphSCCPass.h
llvm::AnyCoroEndInst
Definition: CoroInstr.h:602
llvm::CoroAllocInst
This represents the llvm.coro.alloc instruction.
Definition: CoroInstr.h:70
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:294
llvm::ValueMap< const Value *, WeakTrackingVH >
CORO_PRESPLIT_ATTR
#define CORO_PRESPLIT_ATTR
Definition: CoroInternal.h:39
llvm::MipsISD::TailCall
@ TailCall
Definition: MipsISelLowering.h:65
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:148
llvm::coro::Shape::SwitchLoweringStorage::ResumeEntryBlock
BasicBlock * ResumeEntryBlock
Definition: CoroInternal.h:134
llvm::coro::Shape::RetconLowering
RetconLoweringStorage RetconLowering
Definition: CoroInternal.h:165
llvm::Value::stripPointerCasts
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:662
llvm::ConstantInt::getFalse
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:853
Argument.h
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:140
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:206
CallingConv.h
Attributes.h
llvm::Function::addParamAttr
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
adds the attribute to the list of attributes for the given arg.
Definition: Function.cpp:556
llvm::coro::ABI::RetconOnce
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
llvm::CGSCCUpdateResult
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Definition: CGSCCPassManager.h:232
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
Verifier.h
splitCoroutine
static coro::Shape splitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, bool ReuseFrameSlot)
Definition: CoroSplit.cpp:1786
llvm::ConstantInt::getTrue
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:846
split
coro split
Definition: CoroSplit.cpp:2267
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
llvm::LazyCallGraph::addSplitRefRecursiveFunctions
void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)
Add new ref-recursive functions split/outlined from an existing function.
Definition: LazyCallGraph.cpp:1691
llvm::legacy::FunctionPassManager::run
bool run(Function &F)
run - Execute all of the passes scheduled for execution.
Definition: LegacyPassManager.cpp:1352
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::end
iterator end()
Definition: DenseMap.h:83
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:314
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
llvm::ISD::BR
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:927
llvm::PHINode::Create
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Definition: Instructions.h:2642
llvm::ArrayRef::begin
iterator begin() const
Definition: ArrayRef.h:153
GlobalVariable.h
llvm::MCID::Branch
@ Branch
Definition: MCInstrDesc.h:156
llvm::ConstantTokenNone::get
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition: Constants.cpp:1440
Casting.h
llvm::coro::Shape::getIndex
ConstantInt * getIndex(uint64_t Value) const
Definition: CoroInternal.h:195
Function.h
llvm::CallGraphSCCPass
Definition: CallGraphSCCPass.h:34
llvm::inst_end
inst_iterator inst_end(Function *F)
Definition: InstIterator.h:132
getFunctionTypeFromAsyncSuspend
static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)
Definition: CoroSplit.cpp:460
Arguments
AMDGPU Lower Kernel Arguments
Definition: AMDGPULowerKernelArguments.cpp:244
llvm::salvageDebugInfo
void salvageDebugInfo(Instruction &I)
Assuming the instruction I is going to be deleted, attempt to salvage debug users of I by writing the...
Definition: Local.cpp:1719
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:728
llvm::InlineFunctionInfo
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:193
llvm::ReturnInst::Create
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:2980
llvm::coro::Shape::emitAlloc
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:529
llvm::ConstantArray::get
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1261
llvm::coro::Shape::getSwitchCoroId
CoroIdInst * getSwitchCoroId() const
Definition: CoroInternal.h:169
llvm::createCoroSplitLegacyPass
Pass * createCoroSplitLegacyPass(bool IsOptimizing=false)
Split up coroutines into multiple functions driving their state machines.
Definition: CoroSplit.cpp:2271
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition: InstrTypes.h:1389
llvm::CallGraph::getModule
Module & getModule() const
Returns the module the call graph corresponds to.
Definition: CallGraph.h:102
replaceAllPrepares
static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2040
llvm::updateCGAndAnalysisManagerForCGSCCPass
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
Definition: CGSCCPassManager.cpp:1228
llvm::coro::Shape::getSwitchIndexField
unsigned getSwitchIndexField() const
Definition: CoroInternal.h:185
llvm::AttrBuilder::addDereferenceableAttr
AttrBuilder & addDereferenceableAttr(uint64_t Bytes)
This turns the number of dereferenceable bytes into the form used internally in Attribute.
Definition: Attributes.cpp:1868
llvm::GlobalValue::ExternalLinkage
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
CallGraph.h
llvm::AttributeList::FunctionIndex
@ FunctionIndex
Definition: Attributes.h:389
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:180
llvm::AttrBuilder::addAttribute
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
Definition: Attributes.h:832
llvm::Pass
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:91
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Instructions.h
llvm::CoroSuspendInst::getCoroSave
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:497
llvm::CoroSizeInst
This represents the llvm.coro.size instruction.
Definition: CoroInstr.h:591
llvm::coro::Shape::RetconLoweringStorage::ReturnBlock
BasicBlock * ReturnBlock
Definition: CoroInternal.h:145
SmallVector.h
Dominators.h
llvm::Type::getPointerElementType
Type * getPointerElementType() const
Definition: Type.h:378
llvm::coro::updateCallGraph
void updateCallGraph(Function &Caller, ArrayRef< Function * > Funcs, CallGraph &CG, CallGraphSCC &SCC)
Definition: Coroutines.cpp:215
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1341
N
#define N
maybeFreeRetconStorage
static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)
Definition: CoroSplit.cpp:169
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::legacy::FunctionPassManager
FunctionPassManager manages FunctionPasses.
Definition: LegacyPassManager.h:71
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
llvm::coro::LowererBase
Definition: CoroInternal.h:60
llvm::PHINode
Definition: Instructions.h:2600
llvm::CoroBeginInst
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:420
llvm::BasicBlock::removePredecessor
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Definition: BasicBlock.cpp:321
CoroInstr.h
llvm::DISubprogram
Subprogram description.
Definition: DebugInfoMetadata.h:1800
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
llvm::InlineFunction
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
Definition: InlineFunction.cpp:1757
llvm::coro::Shape::SwitchFieldIndex::Resume
@ Resume
Definition: CoroInternal.h:108
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1164
llvm::GlobalValue::getType
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:271
DerivedTypes.h
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
replacePrepare
static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Replace a call to llvm.coro.prepare.retcon.
Definition: CoroSplit.cpp:1950
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1478
llvm::CloneFunctionInto
void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.
Definition: CloneFunction.cpp:83
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::inst_begin
inst_iterator inst_begin(Function *F)
Definition: InstIterator.h:131
LLVMContext.h
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:367
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
simplifySuspendPoint
static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)
Definition: CoroSplit.cpp:1396
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:389
CallGraphUpdater.h
llvm::LazyCallGraph
A lazily constructed view of the call graph of a module.
Definition: LazyCallGraph.h:112
raw_ostream.h
llvm::AMDGPU::VGPRIndexMode::Id
Id
Definition: SIDefines.h:221
llvm::CallGraph::getCallsExternalNode
CallGraphNode * getCallsExternalNode() const
Definition: CallGraph.h:130
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition: SmallVector.h:623
llvm::FunctionAnalysisManagerCGSCCProxy
A proxy from a FunctionAnalysisManager to an SCC.
Definition: CGSCCPassManager.h:392
BasicBlockUtils.h
Value.h
createDevirtTriggerFunc
static void createDevirtTriggerFunc(CallGraph &CG, CallGraphSCC &SCC)
Definition: CoroSplit.cpp:1927
InitializePasses.h
llvm::LLVMContext::OB_funclet
@ OB_funclet
Definition: LLVMContext.h:91
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition: DerivedTypes.h:123
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
Debug.h
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:422
llvm::ArrayRef::end
iterator end() const
Definition: ArrayRef.h:154
llvm::GlobalVariable::setInitializer
void setInitializer(Constant *InitVal)
setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...
Definition: Globals.cpp:389
of
Add support for conditional and other related patterns Instead of
Definition: README.txt:134
llvm::FunctionType
Class to represent function types.
Definition: DerivedTypes.h:102
machine
coro Split coroutine into a set of functions driving its state machine
Definition: CoroSplit.cpp:2268
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:44
llvm::CGSCCUpdateResult::RCWorklist
SmallPriorityWorklist< LazyCallGraph::RefSCC *, 1 > & RCWorklist
Worklist of the RefSCCs queued for processing.
Definition: CGSCCPassManager.h:242
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:364
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38