LLVM  16.0.0git
CoroSplit.cpp
Go to the documentation of this file.
1 //===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // This pass builds the coroutine frame and outlines resume and destroy parts
9 // of the coroutine into separate functions.
10 //
11 // We present a coroutine to an LLVM as an ordinary function with suspension
12 // points marked up with intrinsics. We let the optimizer party on the coroutine
13 // as a single function for as long as possible. Shortly before the coroutine is
14 // eligible to be inlined into its callers, we split up the coroutine into parts
15 // corresponding to an initial, resume and destroy invocations of the coroutine,
16 // add them to the current SCC and restart the IPO pipeline to optimize the
17 // coroutine subfunctions we extracted before proceeding to the caller of the
18 // coroutine.
19 //===----------------------------------------------------------------------===//
20 
22 #include "CoroInstr.h"
23 #include "CoroInternal.h"
24 #include "llvm/ADT/DenseMap.h"
26 #include "llvm/ADT/SmallPtrSet.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/StringRef.h"
29 #include "llvm/ADT/Twine.h"
30 #include "llvm/Analysis/CFG.h"
36 #include "llvm/IR/Argument.h"
37 #include "llvm/IR/Attributes.h"
38 #include "llvm/IR/BasicBlock.h"
39 #include "llvm/IR/CFG.h"
40 #include "llvm/IR/CallingConv.h"
41 #include "llvm/IR/Constants.h"
42 #include "llvm/IR/DataLayout.h"
43 #include "llvm/IR/DerivedTypes.h"
44 #include "llvm/IR/Dominators.h"
45 #include "llvm/IR/Function.h"
46 #include "llvm/IR/GlobalValue.h"
47 #include "llvm/IR/GlobalVariable.h"
48 #include "llvm/IR/IRBuilder.h"
49 #include "llvm/IR/InstIterator.h"
50 #include "llvm/IR/InstrTypes.h"
51 #include "llvm/IR/Instruction.h"
52 #include "llvm/IR/Instructions.h"
53 #include "llvm/IR/IntrinsicInst.h"
54 #include "llvm/IR/LLVMContext.h"
55 #include "llvm/IR/Module.h"
56 #include "llvm/IR/Type.h"
57 #include "llvm/IR/Value.h"
58 #include "llvm/IR/Verifier.h"
59 #include "llvm/Support/Casting.h"
60 #include "llvm/Support/Debug.h"
63 #include "llvm/Transforms/Scalar.h"
69 #include <cassert>
70 #include <cstddef>
71 #include <cstdint>
72 #include <initializer_list>
73 #include <iterator>
74 
75 using namespace llvm;
76 
77 #define DEBUG_TYPE "coro-split"
78 
79 namespace {
80 
81 /// A little helper class for building
82 class CoroCloner {
83 public:
84  enum class Kind {
85  /// The shared resume function for a switch lowering.
86  SwitchResume,
87 
88  /// The shared unwind function for a switch lowering.
89  SwitchUnwind,
90 
91  /// The shared cleanup function for a switch lowering.
92  SwitchCleanup,
93 
94  /// An individual continuation function.
95  Continuation,
96 
97  /// An async resume function.
98  Async,
99  };
100 
101 private:
102  Function &OrigF;
103  Function *NewF;
104  const Twine &Suffix;
105  coro::Shape &Shape;
106  Kind FKind;
107  ValueToValueMapTy VMap;
109  Value *NewFramePtr = nullptr;
110 
111  /// The active suspend instruction; meaningful only for continuation and async
112  /// ABIs.
113  AnyCoroSuspendInst *ActiveSuspend = nullptr;
114 
115 public:
116  /// Create a cloner for a switch lowering.
117  CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
118  Kind FKind)
119  : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape),
120  FKind(FKind), Builder(OrigF.getContext()) {
121  assert(Shape.ABI == coro::ABI::Switch);
122  }
123 
124  /// Create a cloner for a continuation lowering.
125  CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
126  Function *NewF, AnyCoroSuspendInst *ActiveSuspend)
127  : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape),
128  FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation),
129  Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend) {
130  assert(Shape.ABI == coro::ABI::Retcon ||
131  Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async);
132  assert(NewF && "need existing function for continuation");
133  assert(ActiveSuspend && "need active suspend point for continuation");
134  }
135 
136  Function *getFunction() const {
137  assert(NewF != nullptr && "declaration not yet set");
138  return NewF;
139  }
140 
141  void create();
142 
143 private:
144  bool isSwitchDestroyFunction() {
145  switch (FKind) {
146  case Kind::Async:
147  case Kind::Continuation:
148  case Kind::SwitchResume:
149  return false;
150  case Kind::SwitchUnwind:
151  case Kind::SwitchCleanup:
152  return true;
153  }
154  llvm_unreachable("Unknown CoroCloner::Kind enum");
155  }
156 
157  void replaceEntryBlock();
158  Value *deriveNewFramePointer();
159  void replaceRetconOrAsyncSuspendUses();
160  void replaceCoroSuspends();
161  void replaceCoroEnds();
162  void replaceSwiftErrorOps();
163  void salvageDebugInfo();
164  void handleFinalSuspend();
165 };
166 
167 } // end anonymous namespace
168 
170  const coro::Shape &Shape, Value *FramePtr,
171  CallGraph *CG) {
172  assert(Shape.ABI == coro::ABI::Retcon ||
173  Shape.ABI == coro::ABI::RetconOnce);
175  return;
176 
177  Shape.emitDealloc(Builder, FramePtr, CG);
178 }
179 
180 /// Replace an llvm.coro.end.async.
181 /// Will inline the must tail call function call if there is one.
182 /// \returns true if cleanup of the coro.end block is needed, false otherwise.
184  IRBuilder<> Builder(End);
185 
186  auto *EndAsync = dyn_cast<CoroAsyncEndInst>(End);
187  if (!EndAsync) {
188  Builder.CreateRetVoid();
189  return true /*needs cleanup of coro.end block*/;
190  }
191 
192  auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();
193  if (!MustTailCallFunc) {
194  Builder.CreateRetVoid();
195  return true /*needs cleanup of coro.end block*/;
196  }
197 
198  // Move the must tail call from the predecessor block into the end block.
199  auto *CoroEndBlock = End->getParent();
200  auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();
201  assert(MustTailCallFuncBlock && "Must have a single predecessor block");
202  auto It = MustTailCallFuncBlock->getTerminator()->getIterator();
203  auto *MustTailCall = cast<CallInst>(&*std::prev(It));
204  CoroEndBlock->getInstList().splice(
205  End->getIterator(), MustTailCallFuncBlock->getInstList(), MustTailCall);
206 
207  // Insert the return instruction.
208  Builder.SetInsertPoint(End);
209  Builder.CreateRetVoid();
210  InlineFunctionInfo FnInfo;
211 
212  // Remove the rest of the block, by splitting it into an unreachable block.
213  auto *BB = End->getParent();
214  BB->splitBasicBlock(End);
215  BB->getTerminator()->eraseFromParent();
216 
217  auto InlineRes = InlineFunction(*MustTailCall, FnInfo);
218  assert(InlineRes.isSuccess() && "Expected inlining to succeed");
219  (void)InlineRes;
220 
221  // We have cleaned up the coro.end block above.
222  return false;
223 }
224 
225 /// Replace a non-unwind call to llvm.coro.end.
227  const coro::Shape &Shape, Value *FramePtr,
228  bool InResume, CallGraph *CG) {
229  // Start inserting right before the coro.end.
230  IRBuilder<> Builder(End);
231 
232  // Create the return instruction.
233  switch (Shape.ABI) {
234  // The cloned functions in switch-lowering always return void.
235  case coro::ABI::Switch:
236  // coro.end doesn't immediately end the coroutine in the main function
237  // in this lowering, because we need to deallocate the coroutine.
238  if (!InResume)
239  return;
240  Builder.CreateRetVoid();
241  break;
242 
243  // In async lowering this returns.
244  case coro::ABI::Async: {
245  bool CoroEndBlockNeedsCleanup = replaceCoroEndAsync(End);
246  if (!CoroEndBlockNeedsCleanup)
247  return;
248  break;
249  }
250 
251  // In unique continuation lowering, the continuations always return void.
252  // But we may have implicitly allocated storage.
254  maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
255  Builder.CreateRetVoid();
256  break;
257 
258  // In non-unique continuation lowering, we signal completion by returning
259  // a null continuation.
260  case coro::ABI::Retcon: {
261  maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
262  auto RetTy = Shape.getResumeFunctionType()->getReturnType();
263  auto RetStructTy = dyn_cast<StructType>(RetTy);
264  PointerType *ContinuationTy =
265  cast<PointerType>(RetStructTy ? RetStructTy->getElementType(0) : RetTy);
266 
267  Value *ReturnValue = ConstantPointerNull::get(ContinuationTy);
268  if (RetStructTy) {
269  ReturnValue = Builder.CreateInsertValue(UndefValue::get(RetStructTy),
270  ReturnValue, 0);
271  }
272  Builder.CreateRet(ReturnValue);
273  break;
274  }
275  }
276 
277  // Remove the rest of the block, by splitting it into an unreachable block.
278  auto *BB = End->getParent();
279  BB->splitBasicBlock(End);
280  BB->getTerminator()->eraseFromParent();
281 }
282 
283 // Mark a coroutine as done, which implies that the coroutine is finished and
284 // never get resumed.
285 //
286 // In resume-switched ABI, the done state is represented by storing zero in
287 // ResumeFnAddr.
288 //
289 // NOTE: We couldn't omit the argument `FramePtr`. It is necessary because the
290 // pointer to the frame in splitted function is not stored in `Shape`.
292  Value *FramePtr) {
293  assert(
294  Shape.ABI == coro::ABI::Switch &&
295  "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");
296  auto *GepIndex = Builder.CreateStructGEP(
298  "ResumeFn.addr");
299  auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
301  Builder.CreateStore(NullPtr, GepIndex);
302 }
303 
304 /// Replace an unwind call to llvm.coro.end.
305 static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
306  Value *FramePtr, bool InResume,
307  CallGraph *CG) {
308  IRBuilder<> Builder(End);
309 
310  switch (Shape.ABI) {
311  // In switch-lowering, this does nothing in the main function.
312  case coro::ABI::Switch: {
313  // In C++'s specification, the coroutine should be marked as done
314  // if promise.unhandled_exception() throws. The frontend will
315  // call coro.end(true) along this path.
316  //
317  // FIXME: We should refactor this once there is other language
318  // which uses Switch-Resumed style other than C++.
319  markCoroutineAsDone(Builder, Shape, FramePtr);
320  if (!InResume)
321  return;
322  break;
323  }
324  // In async lowering this does nothing.
325  case coro::ABI::Async:
326  break;
327  // In continuation-lowering, this frees the continuation storage.
328  case coro::ABI::Retcon:
330  maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
331  break;
332  }
333 
334  // If coro.end has an associated bundle, add cleanupret instruction.
335  if (auto Bundle = End->getOperandBundle(LLVMContext::OB_funclet)) {
336  auto *FromPad = cast<CleanupPadInst>(Bundle->Inputs[0]);
337  auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);
338  End->getParent()->splitBasicBlock(End);
339  CleanupRet->getParent()->getTerminator()->eraseFromParent();
340  }
341 }
342 
343 static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
344  Value *FramePtr, bool InResume, CallGraph *CG) {
345  if (End->isUnwind())
346  replaceUnwindCoroEnd(End, Shape, FramePtr, InResume, CG);
347  else
348  replaceFallthroughCoroEnd(End, Shape, FramePtr, InResume, CG);
349 
350  auto &Context = End->getContext();
351  End->replaceAllUsesWith(InResume ? ConstantInt::getTrue(Context)
353  End->eraseFromParent();
354 }
355 
356 // Create an entry block for a resume function with a switch that will jump to
357 // suspend points.
359  assert(Shape.ABI == coro::ABI::Switch);
360  LLVMContext &C = F.getContext();
361 
362  // resume.entry:
363  // %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0,
364  // i32 2
365  // % index = load i32, i32* %index.addr
366  // switch i32 %index, label %unreachable [
367  // i32 0, label %resume.0
368  // i32 1, label %resume.1
369  // ...
370  // ]
371 
372  auto *NewEntry = BasicBlock::Create(C, "resume.entry", &F);
373  auto *UnreachBB = BasicBlock::Create(C, "unreachable", &F);
374 
375  IRBuilder<> Builder(NewEntry);
376  auto *FramePtr = Shape.FramePtr;
377  auto *FrameTy = Shape.FrameTy;
378  auto *GepIndex = Builder.CreateStructGEP(
379  FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
380  auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
381  auto *Switch =
382  Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
383  Shape.SwitchLowering.ResumeSwitch = Switch;
384 
385  size_t SuspendIndex = 0;
386  for (auto *AnyS : Shape.CoroSuspends) {
387  auto *S = cast<CoroSuspendInst>(AnyS);
388  ConstantInt *IndexVal = Shape.getIndex(SuspendIndex);
389 
390  // Replace CoroSave with a store to Index:
391  // %index.addr = getelementptr %f.frame... (index field number)
392  // store i32 %IndexVal, i32* %index.addr1
393  auto *Save = S->getCoroSave();
394  Builder.SetInsertPoint(Save);
395  if (S->isFinal()) {
396  // The coroutine should be marked done if it reaches the final suspend
397  // point.
398  markCoroutineAsDone(Builder, Shape, FramePtr);
399  }
400 
401  // If the coroutine don't have unwind coro end, we could omit the store to
402  // the final suspend point since we could infer the coroutine is suspended
403  // at the final suspend point by the nullness of ResumeFnAddr.
404  // However, we can't skip it if the coroutine have unwind coro end. Since
405  // the coroutine reaches unwind coro end is considered suspended at the
406  // final suspend point (the ResumeFnAddr is null) but in fact the coroutine
407  // didn't complete yet. We need the IndexVal for the final suspend point
408  // to make the states clear.
409  if (!S->isFinal() || Shape.SwitchLowering.HasUnwindCoroEnd) {
410  auto *GepIndex = Builder.CreateStructGEP(
411  FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
412  Builder.CreateStore(IndexVal, GepIndex);
413  }
414 
415  Save->replaceAllUsesWith(ConstantTokenNone::get(C));
416  Save->eraseFromParent();
417 
418  // Split block before and after coro.suspend and add a jump from an entry
419  // switch:
420  //
421  // whateverBB:
422  // whatever
423  // %0 = call i8 @llvm.coro.suspend(token none, i1 false)
424  // switch i8 %0, label %suspend[i8 0, label %resume
425  // i8 1, label %cleanup]
426  // becomes:
427  //
428  // whateverBB:
429  // whatever
430  // br label %resume.0.landing
431  //
432  // resume.0: ; <--- jump from the switch in the resume.entry
433  // %0 = tail call i8 @llvm.coro.suspend(token none, i1 false)
434  // br label %resume.0.landing
435  //
436  // resume.0.landing:
437  // %1 = phi i8[-1, %whateverBB], [%0, %resume.0]
438  // switch i8 % 1, label %suspend [i8 0, label %resume
439  // i8 1, label %cleanup]
440 
441  auto *SuspendBB = S->getParent();
442  auto *ResumeBB =
443  SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));
444  auto *LandingBB = ResumeBB->splitBasicBlock(
445  S->getNextNode(), ResumeBB->getName() + Twine(".landing"));
446  Switch->addCase(IndexVal, ResumeBB);
447 
448  cast<BranchInst>(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);
449  auto *PN = PHINode::Create(Builder.getInt8Ty(), 2, "", &LandingBB->front());
450  S->replaceAllUsesWith(PN);
451  PN->addIncoming(Builder.getInt8(-1), SuspendBB);
452  PN->addIncoming(S, ResumeBB);
453 
454  ++SuspendIndex;
455  }
456 
457  Builder.SetInsertPoint(UnreachBB);
458  Builder.CreateUnreachable();
459 
460  Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
461 }
462 
463 // In the resume function, we remove the last case (when coro::Shape is built,
464 // the final suspend point (if present) is always the last element of
465 // CoroSuspends array) since it is an undefined behavior to resume a coroutine
466 // suspended at the final suspend point.
467 // In the destroy function, if it isn't possible that the ResumeFnAddr is NULL
468 // and the coroutine doesn't suspend at the final suspend point actually (this
469 // is possible since the coroutine is considered suspended at the final suspend
470 // point if promise.unhandled_exception() exits via an exception), we can
471 // remove the last case.
472 void CoroCloner::handleFinalSuspend() {
473  assert(Shape.ABI == coro::ABI::Switch &&
474  Shape.SwitchLowering.HasFinalSuspend);
475 
476  if (isSwitchDestroyFunction() && Shape.SwitchLowering.HasUnwindCoroEnd)
477  return;
478 
479  auto *Switch = cast<SwitchInst>(VMap[Shape.SwitchLowering.ResumeSwitch]);
480  auto FinalCaseIt = std::prev(Switch->case_end());
481  BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();
482  Switch->removeCase(FinalCaseIt);
483  if (isSwitchDestroyFunction()) {
484  BasicBlock *OldSwitchBB = Switch->getParent();
485  auto *NewSwitchBB = OldSwitchBB->splitBasicBlock(Switch, "Switch");
486  Builder.SetInsertPoint(OldSwitchBB->getTerminator());
487  auto *GepIndex = Builder.CreateStructGEP(Shape.FrameTy, NewFramePtr,
489  "ResumeFn.addr");
490  auto *Load = Builder.CreateLoad(Shape.getSwitchResumePointerType(),
491  GepIndex);
492  auto *Cond = Builder.CreateIsNull(Load);
493  Builder.CreateCondBr(Cond, ResumeBB, NewSwitchBB);
494  OldSwitchBB->getTerminator()->eraseFromParent();
495  }
496 }
497 
498 static FunctionType *
500  auto *AsyncSuspend = cast<CoroSuspendAsyncInst>(Suspend);
501  auto *StructTy = cast<StructType>(AsyncSuspend->getType());
502  auto &Context = Suspend->getParent()->getParent()->getContext();
503  auto *VoidTy = Type::getVoidTy(Context);
504  return FunctionType::get(VoidTy, StructTy->elements(), false);
505 }
506 
508  const Twine &Suffix,
509  Module::iterator InsertBefore,
510  AnyCoroSuspendInst *ActiveSuspend) {
511  Module *M = OrigF.getParent();
512  auto *FnTy = (Shape.ABI != coro::ABI::Async)
513  ? Shape.getResumeFunctionType()
514  : getFunctionTypeFromAsyncSuspend(ActiveSuspend);
515 
516  Function *NewF =
517  Function::Create(FnTy, GlobalValue::LinkageTypes::InternalLinkage,
518  OrigF.getName() + Suffix);
519  if (Shape.ABI != coro::ABI::Async)
520  NewF->addParamAttr(0, Attribute::NonNull);
521 
522  // For the async lowering ABI we can't guarantee that the context argument is
523  // not access via a different pointer not based on the argument.
524  if (Shape.ABI != coro::ABI::Async)
525  NewF->addParamAttr(0, Attribute::NoAlias);
526 
527  M->getFunctionList().insert(InsertBefore, NewF);
528 
529  return NewF;
530 }
531 
532 /// Replace uses of the active llvm.coro.suspend.retcon/async call with the
533 /// arguments to the continuation function.
534 ///
535 /// This assumes that the builder has a meaningful insertion point.
536 void CoroCloner::replaceRetconOrAsyncSuspendUses() {
537  assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||
538  Shape.ABI == coro::ABI::Async);
539 
540  auto NewS = VMap[ActiveSuspend];
541  if (NewS->use_empty()) return;
542 
543  // Copy out all the continuation arguments after the buffer pointer into
544  // an easily-indexed data structure for convenience.
546  // The async ABI includes all arguments -- including the first argument.
547  bool IsAsyncABI = Shape.ABI == coro::ABI::Async;
548  for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),
549  E = NewF->arg_end();
550  I != E; ++I)
551  Args.push_back(&*I);
552 
553  // If the suspend returns a single scalar value, we can just do a simple
554  // replacement.
555  if (!isa<StructType>(NewS->getType())) {
556  assert(Args.size() == 1);
557  NewS->replaceAllUsesWith(Args.front());
558  return;
559  }
560 
561  // Try to peephole extracts of an aggregate return.
562  for (Use &U : llvm::make_early_inc_range(NewS->uses())) {
563  auto *EVI = dyn_cast<ExtractValueInst>(U.getUser());
564  if (!EVI || EVI->getNumIndices() != 1)
565  continue;
566 
567  EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);
568  EVI->eraseFromParent();
569  }
570 
571  // If we have no remaining uses, we're done.
572  if (NewS->use_empty()) return;
573 
574  // Otherwise, we need to create an aggregate.
575  Value *Agg = UndefValue::get(NewS->getType());
576  for (size_t I = 0, E = Args.size(); I != E; ++I)
577  Agg = Builder.CreateInsertValue(Agg, Args[I], I);
578 
579  NewS->replaceAllUsesWith(Agg);
580 }
581 
582 void CoroCloner::replaceCoroSuspends() {
583  Value *SuspendResult;
584 
585  switch (Shape.ABI) {
586  // In switch lowering, replace coro.suspend with the appropriate value
587  // for the type of function we're extracting.
588  // Replacing coro.suspend with (0) will result in control flow proceeding to
589  // a resume label associated with a suspend point, replacing it with (1) will
590  // result in control flow proceeding to a cleanup label associated with this
591  // suspend point.
592  case coro::ABI::Switch:
593  SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);
594  break;
595 
596  // In async lowering there are no uses of the result.
597  case coro::ABI::Async:
598  return;
599 
600  // In returned-continuation lowering, the arguments from earlier
601  // continuations are theoretically arbitrary, and they should have been
602  // spilled.
604  case coro::ABI::Retcon:
605  return;
606  }
607 
608  for (AnyCoroSuspendInst *CS : Shape.CoroSuspends) {
609  // The active suspend was handled earlier.
610  if (CS == ActiveSuspend) continue;
611 
612  auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[CS]);
613  MappedCS->replaceAllUsesWith(SuspendResult);
614  MappedCS->eraseFromParent();
615  }
616 }
617 
618 void CoroCloner::replaceCoroEnds() {
619  for (AnyCoroEndInst *CE : Shape.CoroEnds) {
620  // We use a null call graph because there's no call graph node for
621  // the cloned function yet. We'll just be rebuilding that later.
622  auto *NewCE = cast<AnyCoroEndInst>(VMap[CE]);
623  replaceCoroEnd(NewCE, Shape, NewFramePtr, /*in resume*/ true, nullptr);
624  }
625 }
626 
628  ValueToValueMapTy *VMap) {
629  if (Shape.ABI == coro::ABI::Async && Shape.CoroSuspends.empty())
630  return;
631  Value *CachedSlot = nullptr;
632  auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {
633  if (CachedSlot) {
634  assert(cast<PointerType>(CachedSlot->getType())
635  ->isOpaqueOrPointeeTypeMatches(ValueTy) &&
636  "multiple swifterror slots in function with different types");
637  return CachedSlot;
638  }
639 
640  // Check if the function has a swifterror argument.
641  for (auto &Arg : F.args()) {
642  if (Arg.isSwiftError()) {
643  CachedSlot = &Arg;
644  assert(cast<PointerType>(Arg.getType())
645  ->isOpaqueOrPointeeTypeMatches(ValueTy) &&
646  "swifterror argument does not have expected type");
647  return &Arg;
648  }
649  }
650 
651  // Create a swifterror alloca.
652  IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
653  auto Alloca = Builder.CreateAlloca(ValueTy);
654  Alloca->setSwiftError(true);
655 
656  CachedSlot = Alloca;
657  return Alloca;
658  };
659 
660  for (CallInst *Op : Shape.SwiftErrorOps) {
661  auto MappedOp = VMap ? cast<CallInst>((*VMap)[Op]) : Op;
662  IRBuilder<> Builder(MappedOp);
663 
664  // If there are no arguments, this is a 'get' operation.
665  Value *MappedResult;
666  if (Op->arg_empty()) {
667  auto ValueTy = Op->getType();
668  auto Slot = getSwiftErrorSlot(ValueTy);
669  MappedResult = Builder.CreateLoad(ValueTy, Slot);
670  } else {
671  assert(Op->arg_size() == 1);
672  auto Value = MappedOp->getArgOperand(0);
673  auto ValueTy = Value->getType();
674  auto Slot = getSwiftErrorSlot(ValueTy);
675  Builder.CreateStore(Value, Slot);
676  MappedResult = Slot;
677  }
678 
679  MappedOp->replaceAllUsesWith(MappedResult);
680  MappedOp->eraseFromParent();
681  }
682 
683  // If we're updating the original function, we've invalidated SwiftErrorOps.
684  if (VMap == nullptr) {
685  Shape.SwiftErrorOps.clear();
686  }
687 }
688 
690  ::replaceSwiftErrorOps(*NewF, Shape, &VMap);
691 }
692 
696  for (auto &BB : *NewF)
697  for (auto &I : BB)
698  if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
699  Worklist.push_back(DVI);
700  for (DbgVariableIntrinsic *DVI : Worklist)
701  coro::salvageDebugInfo(DbgPtrAllocaCache, DVI, Shape.OptimizeFrame);
702 
703  // Remove all salvaged dbg.declare intrinsics that became
704  // either unreachable or stale due to the CoroSplit transformation.
705  DominatorTree DomTree(*NewF);
706  auto IsUnreachableBlock = [&](BasicBlock *BB) {
707  return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
708  &DomTree);
709  };
710  for (DbgVariableIntrinsic *DVI : Worklist) {
711  if (IsUnreachableBlock(DVI->getParent()))
712  DVI->eraseFromParent();
713  else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
714  // Count all non-debuginfo uses in reachable blocks.
715  unsigned Uses = 0;
716  for (auto *User : DVI->getVariableLocationOp(0)->users())
717  if (auto *I = dyn_cast<Instruction>(User))
718  if (!isa<AllocaInst>(I) && !IsUnreachableBlock(I->getParent()))
719  ++Uses;
720  if (!Uses)
721  DVI->eraseFromParent();
722  }
723  }
724 }
725 
726 void CoroCloner::replaceEntryBlock() {
727  // In the original function, the AllocaSpillBlock is a block immediately
728  // following the allocation of the frame object which defines GEPs for
729  // all the allocas that have been moved into the frame, and it ends by
730  // branching to the original beginning of the coroutine. Make this
731  // the entry block of the cloned function.
732  auto *Entry = cast<BasicBlock>(VMap[Shape.AllocaSpillBlock]);
733  auto *OldEntry = &NewF->getEntryBlock();
734  Entry->setName("entry" + Suffix);
735  Entry->moveBefore(OldEntry);
736  Entry->getTerminator()->eraseFromParent();
737 
738  // Clear all predecessors of the new entry block. There should be
739  // exactly one predecessor, which we created when splitting out
740  // AllocaSpillBlock to begin with.
741  assert(Entry->hasOneUse());
742  auto BranchToEntry = cast<BranchInst>(Entry->user_back());
743  assert(BranchToEntry->isUnconditional());
744  Builder.SetInsertPoint(BranchToEntry);
745  Builder.CreateUnreachable();
746  BranchToEntry->eraseFromParent();
747 
748  // Branch from the entry to the appropriate place.
749  Builder.SetInsertPoint(Entry);
750  switch (Shape.ABI) {
751  case coro::ABI::Switch: {
752  // In switch-lowering, we built a resume-entry block in the original
753  // function. Make the entry block branch to this.
754  auto *SwitchBB =
755  cast<BasicBlock>(VMap[Shape.SwitchLowering.ResumeEntryBlock]);
756  Builder.CreateBr(SwitchBB);
757  break;
758  }
759  case coro::ABI::Async:
760  case coro::ABI::Retcon:
761  case coro::ABI::RetconOnce: {
762  // In continuation ABIs, we want to branch to immediately after the
763  // active suspend point. Earlier phases will have put the suspend in its
764  // own basic block, so just thread our jump directly to its successor.
765  assert((Shape.ABI == coro::ABI::Async &&
766  isa<CoroSuspendAsyncInst>(ActiveSuspend)) ||
767  ((Shape.ABI == coro::ABI::Retcon ||
768  Shape.ABI == coro::ABI::RetconOnce) &&
769  isa<CoroSuspendRetconInst>(ActiveSuspend)));
770  auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[ActiveSuspend]);
771  auto Branch = cast<BranchInst>(MappedCS->getNextNode());
772  assert(Branch->isUnconditional());
773  Builder.CreateBr(Branch->getSuccessor(0));
774  break;
775  }
776  }
777 
778  // Any static alloca that's still being used but not reachable from the new
779  // entry needs to be moved to the new entry.
780  Function *F = OldEntry->getParent();
781  DominatorTree DT{*F};
783  auto *Alloca = dyn_cast<AllocaInst>(&I);
784  if (!Alloca || I.use_empty())
785  continue;
786  if (DT.isReachableFromEntry(I.getParent()) ||
787  !isa<ConstantInt>(Alloca->getArraySize()))
788  continue;
789  I.moveBefore(*Entry, Entry->getFirstInsertionPt());
790  }
791 }
792 
793 /// Derive the value of the new frame pointer.
794 Value *CoroCloner::deriveNewFramePointer() {
795  // Builder should be inserting to the front of the new entry block.
796 
797  switch (Shape.ABI) {
798  // In switch-lowering, the argument is the frame pointer.
799  case coro::ABI::Switch:
800  return &*NewF->arg_begin();
801  // In async-lowering, one of the arguments is an async context as determined
802  // by the `llvm.coro.id.async` intrinsic. We can retrieve the async context of
803  // the resume function from the async context projection function associated
804  // with the active suspend. The frame is located as a tail to the async
805  // context header.
806  case coro::ABI::Async: {
807  auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
808  auto ContextIdx = ActiveAsyncSuspend->getStorageArgumentIndex() & 0xff;
809  auto *CalleeContext = NewF->getArg(ContextIdx);
810  auto *FramePtrTy = Shape.FrameTy->getPointerTo();
811  auto *ProjectionFunc =
812  ActiveAsyncSuspend->getAsyncContextProjectionFunction();
813  auto DbgLoc =
814  cast<CoroSuspendAsyncInst>(VMap[ActiveSuspend])->getDebugLoc();
815  // Calling i8* (i8*)
816  auto *CallerContext = Builder.CreateCall(ProjectionFunc->getFunctionType(),
817  ProjectionFunc, CalleeContext);
818  CallerContext->setCallingConv(ProjectionFunc->getCallingConv());
819  CallerContext->setDebugLoc(DbgLoc);
820  // The frame is located after the async_context header.
821  auto &Context = Builder.getContext();
822  auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(
823  Type::getInt8Ty(Context), CallerContext,
824  Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr");
825  // Inline the projection function.
827  auto InlineRes = InlineFunction(*CallerContext, InlineInfo);
828  assert(InlineRes.isSuccess());
829  (void)InlineRes;
830  return Builder.CreateBitCast(FramePtrAddr, FramePtrTy);
831  }
832  // In continuation-lowering, the argument is the opaque storage.
833  case coro::ABI::Retcon:
834  case coro::ABI::RetconOnce: {
835  Argument *NewStorage = &*NewF->arg_begin();
836  auto FramePtrTy = Shape.FrameTy->getPointerTo();
837 
838  // If the storage is inline, just bitcast to the storage to the frame type.
839  if (Shape.RetconLowering.IsFrameInlineInStorage)
840  return Builder.CreateBitCast(NewStorage, FramePtrTy);
841 
842  // Otherwise, load the real frame from the opaque storage.
843  auto FramePtrPtr =
844  Builder.CreateBitCast(NewStorage, FramePtrTy->getPointerTo());
845  return Builder.CreateLoad(FramePtrTy, FramePtrPtr);
846  }
847  }
848  llvm_unreachable("bad ABI");
849 }
850 
852  unsigned ParamIndex,
853  uint64_t Size, Align Alignment) {
854  AttrBuilder ParamAttrs(Context);
855  ParamAttrs.addAttribute(Attribute::NonNull);
856  ParamAttrs.addAttribute(Attribute::NoAlias);
857  ParamAttrs.addAlignmentAttr(Alignment);
858  ParamAttrs.addDereferenceableAttr(Size);
859  Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
860 }
861 
863  unsigned ParamIndex) {
864  AttrBuilder ParamAttrs(Context);
865  ParamAttrs.addAttribute(Attribute::SwiftAsync);
866  Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
867 }
868 
870  unsigned ParamIndex) {
871  AttrBuilder ParamAttrs(Context);
872  ParamAttrs.addAttribute(Attribute::SwiftSelf);
873  Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
874 }
875 
876 /// Clone the body of the original function into a resume function of
877 /// some sort.
878 void CoroCloner::create() {
879  // Create the new function if we don't already have one.
880  if (!NewF) {
881  NewF = createCloneDeclaration(OrigF, Shape, Suffix,
882  OrigF.getParent()->end(), ActiveSuspend);
883  }
884 
885  // Replace all args with dummy instructions. If an argument is the old frame
886  // pointer, the dummy will be replaced by the new frame pointer once it is
887  // computed below. Uses of all other arguments should have already been
888  // rewritten by buildCoroutineFrame() to use loads/stores on the coroutine
889  // frame.
890  SmallVector<Instruction *> DummyArgs;
891  for (Argument &A : OrigF.args()) {
892  DummyArgs.push_back(new FreezeInst(UndefValue::get(A.getType())));
893  VMap[&A] = DummyArgs.back();
894  }
895 
897 
898  // Ignore attempts to change certain attributes of the function.
899  // TODO: maybe there should be a way to suppress this during cloning?
900  auto savedVisibility = NewF->getVisibility();
901  auto savedUnnamedAddr = NewF->getUnnamedAddr();
902  auto savedDLLStorageClass = NewF->getDLLStorageClass();
903 
904  // NewF's linkage (which CloneFunctionInto does *not* change) might not
905  // be compatible with the visibility of OrigF (which it *does* change),
906  // so protect against that.
907  auto savedLinkage = NewF->getLinkage();
908  NewF->setLinkage(llvm::GlobalValue::ExternalLinkage);
909 
910  CloneFunctionInto(NewF, &OrigF, VMap,
912 
913  auto &Context = NewF->getContext();
914 
915  // For async functions / continuations, adjust the scope line of the
916  // clone to the line number of the suspend point. However, only
917  // adjust the scope line when the files are the same. This ensures
918  // line number and file name belong together. The scope line is
919  // associated with all pre-prologue instructions. This avoids a jump
920  // in the linetable from the function declaration to the suspend point.
921  if (DISubprogram *SP = NewF->getSubprogram()) {
922  assert(SP != OrigF.getSubprogram() && SP->isDistinct());
923  if (ActiveSuspend)
924  if (auto DL = ActiveSuspend->getDebugLoc())
925  if (SP->getFile() == DL->getFile())
926  SP->setScopeLine(DL->getLine());
927  // Update the linkage name to reflect the modified symbol name. It
928  // is necessary to update the linkage name in Swift, since the
929  // mangling changes for resume functions. It might also be the
930  // right thing to do in C++, but due to a limitation in LLVM's
931  // AsmPrinter we can only do this if the function doesn't have an
932  // abstract specification, since the DWARF backend expects the
933  // abstract specification to contain the linkage name and asserts
934  // that they are identical.
935  if (!SP->getDeclaration() && SP->getUnit() &&
936  SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift)
937  SP->replaceLinkageName(MDString::get(Context, NewF->getName()));
938  }
939 
940  NewF->setLinkage(savedLinkage);
941  NewF->setVisibility(savedVisibility);
942  NewF->setUnnamedAddr(savedUnnamedAddr);
943  NewF->setDLLStorageClass(savedDLLStorageClass);
944  // The function sanitizer metadata needs to match the signature of the
945  // function it is being attached to. However this does not hold for split
946  // functions here. Thus remove the metadata for split functions.
947  if (Shape.ABI == coro::ABI::Switch &&
948  NewF->hasMetadata(LLVMContext::MD_func_sanitize))
949  NewF->eraseMetadata(LLVMContext::MD_func_sanitize);
950 
951  // Replace the attributes of the new function:
952  auto OrigAttrs = NewF->getAttributes();
953  auto NewAttrs = AttributeList();
954 
955  switch (Shape.ABI) {
956  case coro::ABI::Switch:
957  // Bootstrap attributes by copying function attributes from the
958  // original function. This should include optimization settings and so on.
959  NewAttrs = NewAttrs.addFnAttributes(
960  Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));
961 
962  addFramePointerAttrs(NewAttrs, Context, 0,
963  Shape.FrameSize, Shape.FrameAlign);
964  break;
965  case coro::ABI::Async: {
966  auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
967  if (OrigF.hasParamAttribute(Shape.AsyncLowering.ContextArgNo,
968  Attribute::SwiftAsync)) {
969  uint32_t ArgAttributeIndices =
970  ActiveAsyncSuspend->getStorageArgumentIndex();
971  auto ContextArgIndex = ArgAttributeIndices & 0xff;
972  addAsyncContextAttrs(NewAttrs, Context, ContextArgIndex);
973 
974  // `swiftasync` must preceed `swiftself` so 0 is not a valid index for
975  // `swiftself`.
976  auto SwiftSelfIndex = ArgAttributeIndices >> 8;
977  if (SwiftSelfIndex)
978  addSwiftSelfAttrs(NewAttrs, Context, SwiftSelfIndex);
979  }
980 
981  // Transfer the original function's attributes.
982  auto FnAttrs = OrigF.getAttributes().getFnAttrs();
983  NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, FnAttrs));
984  break;
985  }
986  case coro::ABI::Retcon:
988  // If we have a continuation prototype, just use its attributes,
989  // full-stop.
990  NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes();
991 
992  addFramePointerAttrs(NewAttrs, Context, 0,
993  Shape.getRetconCoroId()->getStorageSize(),
994  Shape.getRetconCoroId()->getStorageAlignment());
995  break;
996  }
997 
998  switch (Shape.ABI) {
999  // In these ABIs, the cloned functions always return 'void', and the
1000  // existing return sites are meaningless. Note that for unique
1001  // continuations, this includes the returns associated with suspends;
1002  // this is fine because we can't suspend twice.
1003  case coro::ABI::Switch:
1004  case coro::ABI::RetconOnce:
1005  // Remove old returns.
1006  for (ReturnInst *Return : Returns)
1007  changeToUnreachable(Return);
1008  break;
1009 
1010  // With multi-suspend continuations, we'll already have eliminated the
1011  // original returns and inserted returns before all the suspend points,
1012  // so we want to leave any returns in place.
1013  case coro::ABI::Retcon:
1014  break;
1015  // Async lowering will insert musttail call functions at all suspend points
1016  // followed by a return.
1017  // Don't change returns to unreachable because that will trip up the verifier.
1018  // These returns should be unreachable from the clone.
1019  case coro::ABI::Async:
1020  break;
1021  }
1022 
1023  NewF->setAttributes(NewAttrs);
1024  NewF->setCallingConv(Shape.getResumeFunctionCC());
1025 
1026  // Set up the new entry block.
1027  replaceEntryBlock();
1028 
1029  Builder.SetInsertPoint(&NewF->getEntryBlock().front());
1030  NewFramePtr = deriveNewFramePointer();
1031 
1032  // Remap frame pointer.
1033  Value *OldFramePtr = VMap[Shape.FramePtr];
1034  NewFramePtr->takeName(OldFramePtr);
1035  OldFramePtr->replaceAllUsesWith(NewFramePtr);
1036 
1037  // Remap vFrame pointer.
1038  auto *NewVFrame = Builder.CreateBitCast(
1039  NewFramePtr, Type::getInt8PtrTy(Builder.getContext()), "vFrame");
1040  Value *OldVFrame = cast<Value>(VMap[Shape.CoroBegin]);
1041  if (OldVFrame != NewVFrame)
1042  OldVFrame->replaceAllUsesWith(NewVFrame);
1043 
1044  // All uses of the arguments should have been resolved by this point,
1045  // so we can safely remove the dummy values.
1046  for (Instruction *DummyArg : DummyArgs) {
1047  DummyArg->replaceAllUsesWith(UndefValue::get(DummyArg->getType()));
1048  DummyArg->deleteValue();
1049  }
1050 
1051  switch (Shape.ABI) {
1052  case coro::ABI::Switch:
1053  // Rewrite final suspend handling as it is not done via switch (allows to
1054  // remove final case from the switch, since it is undefined behavior to
1055  // resume the coroutine suspended at the final suspend point.
1056  if (Shape.SwitchLowering.HasFinalSuspend)
1057  handleFinalSuspend();
1058  break;
1059  case coro::ABI::Async:
1060  case coro::ABI::Retcon:
1061  case coro::ABI::RetconOnce:
1062  // Replace uses of the active suspend with the corresponding
1063  // continuation-function arguments.
1064  assert(ActiveSuspend != nullptr &&
1065  "no active suspend when lowering a continuation-style coroutine");
1066  replaceRetconOrAsyncSuspendUses();
1067  break;
1068  }
1069 
1070  // Handle suspends.
1071  replaceCoroSuspends();
1072 
1073  // Handle swifterror.
1075 
1076  // Remove coro.end intrinsics.
1077  replaceCoroEnds();
1078 
1079  // Salvage debug info that points into the coroutine frame.
1080  salvageDebugInfo();
1081 
1082  // Eliminate coro.free from the clones, replacing it with 'null' in cleanup,
1083  // to suppress deallocation code.
1084  if (Shape.ABI == coro::ABI::Switch)
1085  coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]),
1086  /*Elide=*/ FKind == CoroCloner::Kind::SwitchCleanup);
1087 }
1088 
1089 // Create a resume clone by cloning the body of the original function, setting
1090 // new entry block and replacing coro.suspend an appropriate value to force
1091 // resume or cleanup pass for every suspend point.
1092 static Function *createClone(Function &F, const Twine &Suffix,
1093  coro::Shape &Shape, CoroCloner::Kind FKind) {
1094  CoroCloner Cloner(F, Suffix, Shape, FKind);
1095  Cloner.create();
1096  return Cloner.getFunction();
1097 }
1098 
1100  assert(Shape.ABI == coro::ABI::Async);
1101 
1102  auto *FuncPtrStruct = cast<ConstantStruct>(
1104  auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);
1105  auto *OrigContextSize = FuncPtrStruct->getOperand(1);
1106  auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),
1107  Shape.AsyncLowering.ContextSize);
1108  auto *NewFuncPtrStruct = ConstantStruct::get(
1109  FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);
1110 
1111  Shape.AsyncLowering.AsyncFuncPointer->setInitializer(NewFuncPtrStruct);
1112 }
1113 
1115  if (Shape.ABI == coro::ABI::Async)
1117 
1118  for (CoroAlignInst *CA : Shape.CoroAligns) {
1119  CA->replaceAllUsesWith(
1120  ConstantInt::get(CA->getType(), Shape.FrameAlign.value()));
1121  CA->eraseFromParent();
1122  }
1123 
1124  if (Shape.CoroSizes.empty())
1125  return;
1126 
1127  // In the same function all coro.sizes should have the same result type.
1128  auto *SizeIntrin = Shape.CoroSizes.back();
1129  Module *M = SizeIntrin->getModule();
1130  const DataLayout &DL = M->getDataLayout();
1131  auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1132  auto *SizeConstant = ConstantInt::get(SizeIntrin->getType(), Size);
1133 
1134  for (CoroSizeInst *CS : Shape.CoroSizes) {
1135  CS->replaceAllUsesWith(SizeConstant);
1136  CS->eraseFromParent();
1137  }
1138 }
1139 
1140 // Create a global constant array containing pointers to functions provided and
1141 // set Info parameter of CoroBegin to point at this constant. Example:
1142 //
1143 // @f.resumers = internal constant [2 x void(%f.frame*)*]
1144 // [void(%f.frame*)* @f.resume, void(%f.frame*)* @f.destroy]
1145 // define void @f() {
1146 // ...
1147 // call i8* @llvm.coro.begin(i8* null, i32 0, i8* null,
1148 // i8* bitcast([2 x void(%f.frame*)*] * @f.resumers to i8*))
1149 //
1150 // Assumes that all the functions have the same signature.
1151 static void setCoroInfo(Function &F, coro::Shape &Shape,
1152  ArrayRef<Function *> Fns) {
1153  // This only works under the switch-lowering ABI because coro elision
1154  // only works on the switch-lowering ABI.
1155  assert(Shape.ABI == coro::ABI::Switch);
1156 
1157  SmallVector<Constant *, 4> Args(Fns.begin(), Fns.end());
1158  assert(!Args.empty());
1159  Function *Part = *Fns.begin();
1160  Module *M = Part->getParent();
1161  auto *ArrTy = ArrayType::get(Part->getType(), Args.size());
1162 
1163  auto *ConstVal = ConstantArray::get(ArrTy, Args);
1164  auto *GV = new GlobalVariable(*M, ConstVal->getType(), /*isConstant=*/true,
1166  F.getName() + Twine(".resumers"));
1167 
1168  // Update coro.begin instruction to refer to this constant.
1169  LLVMContext &C = F.getContext();
1171  Shape.getSwitchCoroId()->setInfo(BC);
1172 }
1173 
1174 // Store addresses of Resume/Destroy/Cleanup functions in the coroutine frame.
1175 static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
1176  Function *DestroyFn, Function *CleanupFn) {
1177  assert(Shape.ABI == coro::ABI::Switch);
1178 
1180 
1181  auto *ResumeAddr = Builder.CreateStructGEP(
1183  "resume.addr");
1184  Builder.CreateStore(ResumeFn, ResumeAddr);
1185 
1186  Value *DestroyOrCleanupFn = DestroyFn;
1187 
1188  CoroIdInst *CoroId = Shape.getSwitchCoroId();
1189  if (CoroAllocInst *CA = CoroId->getCoroAlloc()) {
1190  // If there is a CoroAlloc and it returns false (meaning we elide the
1191  // allocation, use CleanupFn instead of DestroyFn).
1192  DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);
1193  }
1194 
1195  auto *DestroyAddr = Builder.CreateStructGEP(
1197  "destroy.addr");
1198  Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);
1199 }
1200 
1201 static void postSplitCleanup(Function &F) {
1203 
1204 #ifndef NDEBUG
1205  // For now, we do a mandatory verification step because we don't
1206  // entirely trust this pass. Note that we don't want to add a verifier
1207  // pass to FPM below because it will also verify all the global data.
1208  if (verifyFunction(F, &errs()))
1209  report_fatal_error("Broken function");
1210 #endif
1211 }
1212 
1213 // Assuming we arrived at the block NewBlock from Prev instruction, store
1214 // PHI's incoming values in the ResolvedValues map.
1215 static void
1217  DenseMap<Value *, Value *> &ResolvedValues) {
1218  auto *PrevBB = Prev->getParent();
1219  for (PHINode &PN : NewBlock->phis()) {
1220  auto V = PN.getIncomingValueForBlock(PrevBB);
1221  // See if we already resolved it.
1222  auto VI = ResolvedValues.find(V);
1223  if (VI != ResolvedValues.end())
1224  V = VI->second;
1225  // Remember the value.
1226  ResolvedValues[&PN] = V;
1227  }
1228 }
1229 
1230 // Replace a sequence of branches leading to a ret, with a clone of a ret
1231 // instruction. Suspend instruction represented by a switch, track the PHI
1232 // values and select the correct case successor when possible.
1233 static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) {
1234  DenseMap<Value *, Value *> ResolvedValues;
1235  BasicBlock *UnconditionalSucc = nullptr;
1236  assert(InitialInst->getModule());
1237  const DataLayout &DL = InitialInst->getModule()->getDataLayout();
1238 
1239  auto GetFirstValidInstruction = [](Instruction *I) {
1240  while (I) {
1241  // BitCastInst wouldn't generate actual code so that we could skip it.
1242  if (isa<BitCastInst>(I) || I->isDebugOrPseudoInst() ||
1243  I->isLifetimeStartOrEnd())
1244  I = I->getNextNode();
1245  else if (isInstructionTriviallyDead(I))
1246  // Duing we are in the middle of the transformation, we need to erase
1247  // the dead instruction manually.
1248  I = &*I->eraseFromParent();
1249  else
1250  break;
1251  }
1252  return I;
1253  };
1254 
1255  auto TryResolveConstant = [&ResolvedValues](Value *V) {
1256  auto It = ResolvedValues.find(V);
1257  if (It != ResolvedValues.end())
1258  V = It->second;
1259  return dyn_cast<ConstantInt>(V);
1260  };
1261 
1262  Instruction *I = InitialInst;
1263  while (I->isTerminator() || isa<CmpInst>(I)) {
1264  if (isa<ReturnInst>(I)) {
1265  if (I != InitialInst) {
1266  // If InitialInst is an unconditional branch,
1267  // remove PHI values that come from basic block of InitialInst
1268  if (UnconditionalSucc)
1269  UnconditionalSucc->removePredecessor(InitialInst->getParent(), true);
1270  ReplaceInstWithInst(InitialInst, I->clone());
1271  }
1272  return true;
1273  }
1274  if (auto *BR = dyn_cast<BranchInst>(I)) {
1275  if (BR->isUnconditional()) {
1276  BasicBlock *Succ = BR->getSuccessor(0);
1277  if (I == InitialInst)
1278  UnconditionalSucc = Succ;
1279  scanPHIsAndUpdateValueMap(I, Succ, ResolvedValues);
1280  I = GetFirstValidInstruction(Succ->getFirstNonPHIOrDbgOrLifetime());
1281  continue;
1282  }
1283 
1284  BasicBlock *BB = BR->getParent();
1285  // Handle the case the condition of the conditional branch is constant.
1286  // e.g.,
1287  //
1288  // br i1 false, label %cleanup, label %CoroEnd
1289  //
1290  // It is possible during the transformation. We could continue the
1291  // simplifying in this case.
1292  if (ConstantFoldTerminator(BB, /*DeleteDeadConditions=*/true)) {
1293  // Handle this branch in next iteration.
1294  I = BB->getTerminator();
1295  continue;
1296  }
1297  } else if (auto *CondCmp = dyn_cast<CmpInst>(I)) {
1298  // If the case number of suspended switch instruction is reduced to
1299  // 1, then it is simplified to CmpInst in llvm::ConstantFoldTerminator.
1300  auto *BR = dyn_cast<BranchInst>(
1301  GetFirstValidInstruction(CondCmp->getNextNode()));
1302  if (!BR || !BR->isConditional() || CondCmp != BR->getCondition())
1303  return false;
1304 
1305  // And the comparsion looks like : %cond = icmp eq i8 %V, constant.
1306  // So we try to resolve constant for the first operand only since the
1307  // second operand should be literal constant by design.
1308  ConstantInt *Cond0 = TryResolveConstant(CondCmp->getOperand(0));
1309  auto *Cond1 = dyn_cast<ConstantInt>(CondCmp->getOperand(1));
1310  if (!Cond0 || !Cond1)
1311  return false;
1312 
1313  // Both operands of the CmpInst are Constant. So that we could evaluate
1314  // it immediately to get the destination.
1315  auto *ConstResult =
1316  dyn_cast_or_null<ConstantInt>(ConstantFoldCompareInstOperands(
1317  CondCmp->getPredicate(), Cond0, Cond1, DL));
1318  if (!ConstResult)
1319  return false;
1320 
1321  CondCmp->replaceAllUsesWith(ConstResult);
1322  CondCmp->eraseFromParent();
1323 
1324  // Handle this branch in next iteration.
1325  I = BR;
1326  continue;
1327  } else if (auto *SI = dyn_cast<SwitchInst>(I)) {
1328  ConstantInt *Cond = TryResolveConstant(SI->getCondition());
1329  if (!Cond)
1330  return false;
1331 
1332  BasicBlock *BB = SI->findCaseValue(Cond)->getCaseSuccessor();
1333  scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
1334  I = GetFirstValidInstruction(BB->getFirstNonPHIOrDbgOrLifetime());
1335  continue;
1336  }
1337 
1338  return false;
1339  }
1340  return false;
1341 }
1342 
1343 // Check whether CI obeys the rules of musttail attribute.
1344 static bool shouldBeMustTail(const CallInst &CI, const Function &F) {
1345  if (CI.isInlineAsm())
1346  return false;
1347 
1348  // Match prototypes and calling conventions of resume function.
1349  FunctionType *CalleeTy = CI.getFunctionType();
1350  if (!CalleeTy->getReturnType()->isVoidTy() || (CalleeTy->getNumParams() != 1))
1351  return false;
1352 
1353  Type *CalleeParmTy = CalleeTy->getParamType(0);
1354  if (!CalleeParmTy->isPointerTy() ||
1355  (CalleeParmTy->getPointerAddressSpace() != 0))
1356  return false;
1357 
1358  if (CI.getCallingConv() != F.getCallingConv())
1359  return false;
1360 
1361  // CI should not has any ABI-impacting function attributes.
1362  static const Attribute::AttrKind ABIAttrs[] = {
1363  Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
1364  Attribute::Preallocated, Attribute::InReg, Attribute::Returned,
1365  Attribute::SwiftSelf, Attribute::SwiftError};
1367  for (auto AK : ABIAttrs)
1368  if (Attrs.hasParamAttr(0, AK))
1369  return false;
1370 
1371  return true;
1372 }
1373 
1374 // Add musttail to any resume instructions that is immediately followed by a
1375 // suspend (i.e. ret). We do this even in -O0 to support guaranteed tail call
1376 // for symmetrical coroutine control transfer (C++ Coroutines TS extension).
1377 // This transformation is done only in the resume part of the coroutine that has
1378 // identical signature and calling convention as the coro.resume call.
1380  bool changed = false;
1381 
1382  // Collect potential resume instructions.
1384  for (auto &I : instructions(F))
1385  if (auto *Call = dyn_cast<CallInst>(&I))
1386  if (shouldBeMustTail(*Call, F))
1387  Resumes.push_back(Call);
1388 
1389  // Set musttail on those that are followed by a ret instruction.
1390  for (CallInst *Call : Resumes)
1391  // Skip targets which don't support tail call on the specific case.
1392  if (TTI.supportsTailCallFor(Call) &&
1393  simplifyTerminatorLeadingToRet(Call->getNextNode())) {
1394  Call->setTailCallKind(CallInst::TCK_MustTail);
1395  changed = true;
1396  }
1397 
1398  if (changed)
1400 }
1401 
1402 // Coroutine has no suspend points. Remove heap allocation for the coroutine
1403 // frame if possible.
1405  auto *CoroBegin = Shape.CoroBegin;
1406  auto *CoroId = CoroBegin->getId();
1407  auto *AllocInst = CoroId->getCoroAlloc();
1408  switch (Shape.ABI) {
1409  case coro::ABI::Switch: {
1410  auto SwitchId = cast<CoroIdInst>(CoroId);
1411  coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr);
1412  if (AllocInst) {
1413  IRBuilder<> Builder(AllocInst);
1414  auto *Frame = Builder.CreateAlloca(Shape.FrameTy);
1415  Frame->setAlignment(Shape.FrameAlign);
1416  auto *VFrame = Builder.CreateBitCast(Frame, Builder.getInt8PtrTy());
1417  AllocInst->replaceAllUsesWith(Builder.getFalse());
1418  AllocInst->eraseFromParent();
1419  CoroBegin->replaceAllUsesWith(VFrame);
1420  } else {
1421  CoroBegin->replaceAllUsesWith(CoroBegin->getMem());
1422  }
1423 
1424  break;
1425  }
1426  case coro::ABI::Async:
1427  case coro::ABI::Retcon:
1428  case coro::ABI::RetconOnce:
1429  CoroBegin->replaceAllUsesWith(UndefValue::get(CoroBegin->getType()));
1430  break;
1431  }
1432 
1433  CoroBegin->eraseFromParent();
1434 }
1435 
1436 // SimplifySuspendPoint needs to check that there is no calls between
1437 // coro_save and coro_suspend, since any of the calls may potentially resume
1438 // the coroutine and if that is the case we cannot eliminate the suspend point.
1440  for (Instruction *I = From; I != To; I = I->getNextNode()) {
1441  // Assume that no intrinsic can resume the coroutine.
1442  if (isa<IntrinsicInst>(I))
1443  continue;
1444 
1445  if (isa<CallBase>(I))
1446  return true;
1447  }
1448  return false;
1449 }
1450 
1451 static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) {
1454 
1455  Set.insert(SaveBB);
1456  Worklist.push_back(ResDesBB);
1457 
1458  // Accumulate all blocks between SaveBB and ResDesBB. Because CoroSaveIntr
1459  // returns a token consumed by suspend instruction, all blocks in between
1460  // will have to eventually hit SaveBB when going backwards from ResDesBB.
1461  while (!Worklist.empty()) {
1462  auto *BB = Worklist.pop_back_val();
1463  Set.insert(BB);
1464  for (auto *Pred : predecessors(BB))
1465  if (!Set.contains(Pred))
1466  Worklist.push_back(Pred);
1467  }
1468 
1469  // SaveBB and ResDesBB are checked separately in hasCallsBetween.
1470  Set.erase(SaveBB);
1471  Set.erase(ResDesBB);
1472 
1473  for (auto *BB : Set)
1474  if (hasCallsInBlockBetween(BB->getFirstNonPHI(), nullptr))
1475  return true;
1476 
1477  return false;
1478 }
1479 
1480 static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy) {
1481  auto *SaveBB = Save->getParent();
1482  auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();
1483 
1484  if (SaveBB == ResumeOrDestroyBB)
1485  return hasCallsInBlockBetween(Save->getNextNode(), ResumeOrDestroy);
1486 
1487  // Any calls from Save to the end of the block?
1488  if (hasCallsInBlockBetween(Save->getNextNode(), nullptr))
1489  return true;
1490 
1491  // Any calls from begging of the block up to ResumeOrDestroy?
1492  if (hasCallsInBlockBetween(ResumeOrDestroyBB->getFirstNonPHI(),
1493  ResumeOrDestroy))
1494  return true;
1495 
1496  // Any calls in all of the blocks between SaveBB and ResumeOrDestroyBB?
1497  if (hasCallsInBlocksBetween(SaveBB, ResumeOrDestroyBB))
1498  return true;
1499 
1500  return false;
1501 }
1502 
1503 // If a SuspendIntrin is preceded by Resume or Destroy, we can eliminate the
1504 // suspend point and replace it with nornal control flow.
1506  CoroBeginInst *CoroBegin) {
1507  Instruction *Prev = Suspend->getPrevNode();
1508  if (!Prev) {
1509  auto *Pred = Suspend->getParent()->getSinglePredecessor();
1510  if (!Pred)
1511  return false;
1512  Prev = Pred->getTerminator();
1513  }
1514 
1515  CallBase *CB = dyn_cast<CallBase>(Prev);
1516  if (!CB)
1517  return false;
1518 
1519  auto *Callee = CB->getCalledOperand()->stripPointerCasts();
1520 
1521  // See if the callsite is for resumption or destruction of the coroutine.
1522  auto *SubFn = dyn_cast<CoroSubFnInst>(Callee);
1523  if (!SubFn)
1524  return false;
1525 
1526  // Does not refer to the current coroutine, we cannot do anything with it.
1527  if (SubFn->getFrame() != CoroBegin)
1528  return false;
1529 
1530  // See if the transformation is safe. Specifically, see if there are any
1531  // calls in between Save and CallInstr. They can potenitally resume the
1532  // coroutine rendering this optimization unsafe.
1533  auto *Save = Suspend->getCoroSave();
1534  if (hasCallsBetween(Save, CB))
1535  return false;
1536 
1537  // Replace llvm.coro.suspend with the value that results in resumption over
1538  // the resume or cleanup path.
1539  Suspend->replaceAllUsesWith(SubFn->getRawIndex());
1540  Suspend->eraseFromParent();
1541  Save->eraseFromParent();
1542 
1543  // No longer need a call to coro.resume or coro.destroy.
1544  if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
1545  BranchInst::Create(Invoke->getNormalDest(), Invoke);
1546  }
1547 
1548  // Grab the CalledValue from CB before erasing the CallInstr.
1549  auto *CalledValue = CB->getCalledOperand();
1550  CB->eraseFromParent();
1551 
1552  // If no more users remove it. Usually it is a bitcast of SubFn.
1553  if (CalledValue != SubFn && CalledValue->user_empty())
1554  if (auto *I = dyn_cast<Instruction>(CalledValue))
1555  I->eraseFromParent();
1556 
1557  // Now we are good to remove SubFn.
1558  if (SubFn->user_empty())
1559  SubFn->eraseFromParent();
1560 
1561  return true;
1562 }
1563 
1564 // Remove suspend points that are simplified.
1565 static void simplifySuspendPoints(coro::Shape &Shape) {
1566  // Currently, the only simplification we do is switch-lowering-specific.
1567  if (Shape.ABI != coro::ABI::Switch)
1568  return;
1569 
1570  auto &S = Shape.CoroSuspends;
1571  size_t I = 0, N = S.size();
1572  if (N == 0)
1573  return;
1574 
1575  size_t ChangedFinalIndex = std::numeric_limits<size_t>::max();
1576  while (true) {
1577  auto SI = cast<CoroSuspendInst>(S[I]);
1578  // Leave final.suspend to handleFinalSuspend since it is undefined behavior
1579  // to resume a coroutine suspended at the final suspend point.
1580  if (!SI->isFinal() && simplifySuspendPoint(SI, Shape.CoroBegin)) {
1581  if (--N == I)
1582  break;
1583 
1584  std::swap(S[I], S[N]);
1585 
1586  if (cast<CoroSuspendInst>(S[I])->isFinal()) {
1588  ChangedFinalIndex = I;
1589  }
1590 
1591  continue;
1592  }
1593  if (++I == N)
1594  break;
1595  }
1596  S.resize(N);
1597 
1598  // Maintain final.suspend in case final suspend was swapped.
1599  // Due to we requrie the final suspend to be the last element of CoroSuspends.
1600  if (ChangedFinalIndex < N) {
1601  assert(cast<CoroSuspendInst>(S[ChangedFinalIndex])->isFinal());
1602  std::swap(S[ChangedFinalIndex], S.back());
1603  }
1604 }
1605 
1609  assert(Shape.ABI == coro::ABI::Switch);
1610 
1611  createResumeEntryBlock(F, Shape);
1612  auto ResumeClone = createClone(F, ".resume", Shape,
1613  CoroCloner::Kind::SwitchResume);
1614  auto DestroyClone = createClone(F, ".destroy", Shape,
1615  CoroCloner::Kind::SwitchUnwind);
1616  auto CleanupClone = createClone(F, ".cleanup", Shape,
1617  CoroCloner::Kind::SwitchCleanup);
1618 
1619  postSplitCleanup(*ResumeClone);
1620  postSplitCleanup(*DestroyClone);
1621  postSplitCleanup(*CleanupClone);
1622 
1623  // Adding musttail call to support symmetric transfer.
1624  // Skip targets which don't support tail call.
1625  //
1626  // FIXME: Could we support symmetric transfer effectively without musttail
1627  // call?
1628  if (TTI.supportsTailCalls())
1629  addMustTailToCoroResumes(*ResumeClone, TTI);
1630 
1631  // Store addresses resume/destroy/cleanup functions in the coroutine frame.
1632  updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
1633 
1634  assert(Clones.empty());
1635  Clones.push_back(ResumeClone);
1636  Clones.push_back(DestroyClone);
1637  Clones.push_back(CleanupClone);
1638 
1639  // Create a constant array referring to resume/destroy/clone functions pointed
1640  // by the last argument of @llvm.coro.info, so that CoroElide pass can
1641  // determined correct function to call.
1642  setCoroInfo(F, Shape, Clones);
1643 }
1644 
1646  Value *Continuation) {
1647  auto *ResumeIntrinsic = Suspend->getResumeFunction();
1648  auto &Context = Suspend->getParent()->getParent()->getContext();
1649  auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1650 
1651  IRBuilder<> Builder(ResumeIntrinsic);
1652  auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy);
1653  ResumeIntrinsic->replaceAllUsesWith(Val);
1654  ResumeIntrinsic->eraseFromParent();
1656  UndefValue::get(Int8PtrTy));
1657 }
1658 
1659 /// Coerce the arguments in \p FnArgs according to \p FnTy in \p CallArgs.
1661  ArrayRef<Value *> FnArgs,
1662  SmallVectorImpl<Value *> &CallArgs) {
1663  size_t ArgIdx = 0;
1664  for (auto *paramTy : FnTy->params()) {
1665  assert(ArgIdx < FnArgs.size());
1666  if (paramTy != FnArgs[ArgIdx]->getType())
1667  CallArgs.push_back(
1668  Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy));
1669  else
1670  CallArgs.push_back(FnArgs[ArgIdx]);
1671  ++ArgIdx;
1672  }
1673 }
1674 
1677  IRBuilder<> &Builder) {
1678  auto *FnTy = MustTailCallFn->getFunctionType();
1679  // Coerce the arguments, llvm optimizations seem to ignore the types in
1680  // vaarg functions and throws away casts in optimized mode.
1681  SmallVector<Value *, 8> CallArgs;
1682  coerceArguments(Builder, FnTy, Arguments, CallArgs);
1683 
1684  auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);
1685  TailCall->setTailCallKind(CallInst::TCK_MustTail);
1686  TailCall->setDebugLoc(Loc);
1687  TailCall->setCallingConv(MustTailCallFn->getCallingConv());
1688  return TailCall;
1689 }
1690 
1692  SmallVectorImpl<Function *> &Clones) {
1693  assert(Shape.ABI == coro::ABI::Async);
1694  assert(Clones.empty());
1695  // Reset various things that the optimizer might have decided it
1696  // "knows" about the coroutine function due to not seeing a return.
1697  F.removeFnAttr(Attribute::NoReturn);
1698  F.removeRetAttr(Attribute::NoAlias);
1699  F.removeRetAttr(Attribute::NonNull);
1700 
1701  auto &Context = F.getContext();
1702  auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1703 
1704  auto *Id = cast<CoroIdAsyncInst>(Shape.CoroBegin->getId());
1706 
1707  auto *FramePtr = Id->getStorage();
1708  FramePtr = Builder.CreateBitOrPointerCast(FramePtr, Int8PtrTy);
1709  FramePtr = Builder.CreateConstInBoundsGEP1_32(
1711  "async.ctx.frameptr");
1712 
1713  // Map all uses of llvm.coro.begin to the allocated frame pointer.
1714  {
1715  // Make sure we don't invalidate Shape.FramePtr.
1716  TrackingVH<Value> Handle(Shape.FramePtr);
1717  Shape.CoroBegin->replaceAllUsesWith(FramePtr);
1718  Shape.FramePtr = Handle.getValPtr();
1719  }
1720 
1721  // Create all the functions in order after the main function.
1722  auto NextF = std::next(F.getIterator());
1723 
1724  // Create a continuation function for each of the suspend points.
1725  Clones.reserve(Shape.CoroSuspends.size());
1726  for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1727  auto *Suspend = cast<CoroSuspendAsyncInst>(Shape.CoroSuspends[Idx]);
1728 
1729  // Create the clone declaration.
1730  auto ResumeNameSuffix = ".resume.";
1731  auto ProjectionFunctionName =
1732  Suspend->getAsyncContextProjectionFunction()->getName();
1733  bool UseSwiftMangling = false;
1734  if (ProjectionFunctionName.equals("__swift_async_resume_project_context")) {
1735  ResumeNameSuffix = "TQ";
1736  UseSwiftMangling = true;
1737  } else if (ProjectionFunctionName.equals(
1738  "__swift_async_resume_get_context")) {
1739  ResumeNameSuffix = "TY";
1740  UseSwiftMangling = true;
1741  }
1742  auto *Continuation = createCloneDeclaration(
1743  F, Shape,
1744  UseSwiftMangling ? ResumeNameSuffix + Twine(Idx) + "_"
1745  : ResumeNameSuffix + Twine(Idx),
1746  NextF, Suspend);
1747  Clones.push_back(Continuation);
1748 
1749  // Insert a branch to a new return block immediately before the suspend
1750  // point.
1751  auto *SuspendBB = Suspend->getParent();
1752  auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1753  auto *Branch = cast<BranchInst>(SuspendBB->getTerminator());
1754 
1755  // Place it before the first suspend.
1756  auto *ReturnBB =
1757  BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
1758  Branch->setSuccessor(0, ReturnBB);
1759 
1760  IRBuilder<> Builder(ReturnBB);
1761 
1762  // Insert the call to the tail call function and inline it.
1763  auto *Fn = Suspend->getMustTailCallFunction();
1764  SmallVector<Value *, 8> Args(Suspend->args());
1765  auto FnArgs = ArrayRef<Value *>(Args).drop_front(
1767  auto *TailCall =
1768  coro::createMustTailCall(Suspend->getDebugLoc(), Fn, FnArgs, Builder);
1769  Builder.CreateRetVoid();
1770  InlineFunctionInfo FnInfo;
1771  auto InlineRes = InlineFunction(*TailCall, FnInfo);
1772  assert(InlineRes.isSuccess() && "Expected inlining to succeed");
1773  (void)InlineRes;
1774 
1775  // Replace the lvm.coro.async.resume intrisic call.
1776  replaceAsyncResumeFunction(Suspend, Continuation);
1777  }
1778 
1779  assert(Clones.size() == Shape.CoroSuspends.size());
1780  for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1781  auto *Suspend = Shape.CoroSuspends[Idx];
1782  auto *Clone = Clones[Idx];
1783 
1784  CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend).create();
1785  }
1786 }
1787 
1789  SmallVectorImpl<Function *> &Clones) {
1790  assert(Shape.ABI == coro::ABI::Retcon ||
1791  Shape.ABI == coro::ABI::RetconOnce);
1792  assert(Clones.empty());
1793 
1794  // Reset various things that the optimizer might have decided it
1795  // "knows" about the coroutine function due to not seeing a return.
1796  F.removeFnAttr(Attribute::NoReturn);
1797  F.removeRetAttr(Attribute::NoAlias);
1798  F.removeRetAttr(Attribute::NonNull);
1799 
1800  // Allocate the frame.
1801  auto *Id = cast<AnyCoroIdRetconInst>(Shape.CoroBegin->getId());
1802  Value *RawFramePtr;
1804  RawFramePtr = Id->getStorage();
1805  } else {
1807 
1808  // Determine the size of the frame.
1809  const DataLayout &DL = F.getParent()->getDataLayout();
1810  auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1811 
1812  // Allocate. We don't need to update the call graph node because we're
1813  // going to recompute it from scratch after splitting.
1814  // FIXME: pass the required alignment
1815  RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr);
1816  RawFramePtr =
1817  Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType());
1818 
1819  // Stash the allocated frame pointer in the continuation storage.
1820  auto Dest = Builder.CreateBitCast(Id->getStorage(),
1821  RawFramePtr->getType()->getPointerTo());
1822  Builder.CreateStore(RawFramePtr, Dest);
1823  }
1824 
1825  // Map all uses of llvm.coro.begin to the allocated frame pointer.
1826  {
1827  // Make sure we don't invalidate Shape.FramePtr.
1828  TrackingVH<Value> Handle(Shape.FramePtr);
1829  Shape.CoroBegin->replaceAllUsesWith(RawFramePtr);
1830  Shape.FramePtr = Handle.getValPtr();
1831  }
1832 
1833  // Create a unique return block.
1834  BasicBlock *ReturnBB = nullptr;
1835  SmallVector<PHINode *, 4> ReturnPHIs;
1836 
1837  // Create all the functions in order after the main function.
1838  auto NextF = std::next(F.getIterator());
1839 
1840  // Create a continuation function for each of the suspend points.
1841  Clones.reserve(Shape.CoroSuspends.size());
1842  for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1843  auto Suspend = cast<CoroSuspendRetconInst>(Shape.CoroSuspends[i]);
1844 
1845  // Create the clone declaration.
1846  auto Continuation =
1847  createCloneDeclaration(F, Shape, ".resume." + Twine(i), NextF, nullptr);
1848  Clones.push_back(Continuation);
1849 
1850  // Insert a branch to the unified return block immediately before
1851  // the suspend point.
1852  auto SuspendBB = Suspend->getParent();
1853  auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1854  auto Branch = cast<BranchInst>(SuspendBB->getTerminator());
1855 
1856  // Create the unified return block.
1857  if (!ReturnBB) {
1858  // Place it before the first suspend.
1859  ReturnBB = BasicBlock::Create(F.getContext(), "coro.return", &F,
1860  NewSuspendBB);
1861  Shape.RetconLowering.ReturnBlock = ReturnBB;
1862 
1863  IRBuilder<> Builder(ReturnBB);
1864 
1865  // Create PHIs for all the return values.
1866  assert(ReturnPHIs.empty());
1867 
1868  // First, the continuation.
1869  ReturnPHIs.push_back(Builder.CreatePHI(Continuation->getType(),
1870  Shape.CoroSuspends.size()));
1871 
1872  // Next, all the directly-yielded values.
1873  for (auto *ResultTy : Shape.getRetconResultTypes())
1874  ReturnPHIs.push_back(Builder.CreatePHI(ResultTy,
1875  Shape.CoroSuspends.size()));
1876 
1877  // Build the return value.
1878  auto RetTy = F.getReturnType();
1879 
1880  // Cast the continuation value if necessary.
1881  // We can't rely on the types matching up because that type would
1882  // have to be infinite.
1883  auto CastedContinuationTy =
1884  (ReturnPHIs.size() == 1 ? RetTy : RetTy->getStructElementType(0));
1885  auto *CastedContinuation =
1886  Builder.CreateBitCast(ReturnPHIs[0], CastedContinuationTy);
1887 
1888  Value *RetV;
1889  if (ReturnPHIs.size() == 1) {
1890  RetV = CastedContinuation;
1891  } else {
1892  RetV = UndefValue::get(RetTy);
1893  RetV = Builder.CreateInsertValue(RetV, CastedContinuation, 0);
1894  for (size_t I = 1, E = ReturnPHIs.size(); I != E; ++I)
1895  RetV = Builder.CreateInsertValue(RetV, ReturnPHIs[I], I);
1896  }
1897 
1898  Builder.CreateRet(RetV);
1899  }
1900 
1901  // Branch to the return block.
1902  Branch->setSuccessor(0, ReturnBB);
1903  ReturnPHIs[0]->addIncoming(Continuation, SuspendBB);
1904  size_t NextPHIIndex = 1;
1905  for (auto &VUse : Suspend->value_operands())
1906  ReturnPHIs[NextPHIIndex++]->addIncoming(&*VUse, SuspendBB);
1907  assert(NextPHIIndex == ReturnPHIs.size());
1908  }
1909 
1910  assert(Clones.size() == Shape.CoroSuspends.size());
1911  for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1912  auto Suspend = Shape.CoroSuspends[i];
1913  auto Clone = Clones[i];
1914 
1915  CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend).create();
1916  }
1917 }
1918 
1919 namespace {
1920  class PrettyStackTraceFunction : public PrettyStackTraceEntry {
1921  Function &F;
1922  public:
1923  PrettyStackTraceFunction(Function &F) : F(F) {}
1924  void print(raw_ostream &OS) const override {
1925  OS << "While splitting coroutine ";
1926  F.printAsOperand(OS, /*print type*/ false, F.getParent());
1927  OS << "\n";
1928  }
1929  };
1930 }
1931 
1935  bool OptimizeFrame) {
1936  PrettyStackTraceFunction prettyStackTrace(F);
1937 
1938  // The suspend-crossing algorithm in buildCoroutineFrame get tripped
1939  // up by uses in unreachable blocks, so remove them as a first pass.
1941 
1942  coro::Shape Shape(F, OptimizeFrame);
1943  if (!Shape.CoroBegin)
1944  return Shape;
1945 
1946  simplifySuspendPoints(Shape);
1947  buildCoroutineFrame(F, Shape);
1949 
1950  // If there are no suspend points, no split required, just remove
1951  // the allocation and deallocation blocks, they are not needed.
1952  if (Shape.CoroSuspends.empty()) {
1953  handleNoSuspendCoroutine(Shape);
1954  } else {
1955  switch (Shape.ABI) {
1956  case coro::ABI::Switch:
1957  splitSwitchCoroutine(F, Shape, Clones, TTI);
1958  break;
1959  case coro::ABI::Async:
1960  splitAsyncCoroutine(F, Shape, Clones);
1961  break;
1962  case coro::ABI::Retcon:
1963  case coro::ABI::RetconOnce:
1964  splitRetconCoroutine(F, Shape, Clones);
1965  break;
1966  }
1967  }
1968 
1969  // Replace all the swifterror operations in the original function.
1970  // This invalidates SwiftErrorOps in the Shape.
1971  replaceSwiftErrorOps(F, Shape, nullptr);
1972 
1973  // Finally, salvage the llvm.dbg.{declare,addr} in our original function that
1974  // point into the coroutine frame. We only do this for the current function
1975  // since the Cloner salvaged debug info for us in the new coroutine funclets.
1978  for (auto &BB : F) {
1979  for (auto &I : BB) {
1980  if (auto *DDI = dyn_cast<DbgDeclareInst>(&I)) {
1981  Worklist.push_back(DDI);
1982  continue;
1983  }
1984  if (auto *DDI = dyn_cast<DbgAddrIntrinsic>(&I)) {
1985  Worklist.push_back(DDI);
1986  continue;
1987  }
1988  }
1989  }
1990  for (auto *DDI : Worklist)
1991  coro::salvageDebugInfo(DbgPtrAllocaCache, DDI, Shape.OptimizeFrame);
1992 
1993  return Shape;
1994 }
1995 
1996 /// Remove calls to llvm.coro.end in the original function.
1997 static void removeCoroEnds(const coro::Shape &Shape) {
1998  for (auto *End : Shape.CoroEnds) {
1999  replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, nullptr);
2000  }
2001 }
2002 
2004  LazyCallGraph::Node &N, const coro::Shape &Shape,
2008  if (!Shape.CoroBegin)
2009  return;
2010 
2011  if (Shape.ABI != coro::ABI::Switch)
2012  removeCoroEnds(Shape);
2013  else {
2014  for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) {
2015  auto &Context = End->getContext();
2016  End->replaceAllUsesWith(ConstantInt::getFalse(Context));
2017  End->eraseFromParent();
2018  }
2019  }
2020 
2021  if (!Clones.empty()) {
2022  switch (Shape.ABI) {
2023  case coro::ABI::Switch:
2024  // Each clone in the Switch lowering is independent of the other clones.
2025  // Let the LazyCallGraph know about each one separately.
2026  for (Function *Clone : Clones)
2027  CG.addSplitFunction(N.getFunction(), *Clone);
2028  break;
2029  case coro::ABI::Async:
2030  case coro::ABI::Retcon:
2031  case coro::ABI::RetconOnce:
2032  // Each clone in the Async/Retcon lowering references of the other clones.
2033  // Let the LazyCallGraph know about all of them at once.
2034  if (!Clones.empty())
2035  CG.addSplitRefRecursiveFunctions(N.getFunction(), Clones);
2036  break;
2037  }
2038 
2039  // Let the CGSCC infra handle the changes to the original function.
2041  }
2042 
2043  // Do some cleanup and let the CGSCC infra see if we've cleaned up any edges
2044  // to the split functions.
2045  postSplitCleanup(N.getFunction());
2047 }
2048 
2049 /// Replace a call to llvm.coro.prepare.retcon.
2050 static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG,
2051  LazyCallGraph::SCC &C) {
2052  auto CastFn = Prepare->getArgOperand(0); // as an i8*
2053  auto Fn = CastFn->stripPointerCasts(); // as its original type
2054 
2055  // Attempt to peephole this pattern:
2056  // %0 = bitcast [[TYPE]] @some_function to i8*
2057  // %1 = call @llvm.coro.prepare.retcon(i8* %0)
2058  // %2 = bitcast %1 to [[TYPE]]
2059  // ==>
2060  // %2 = @some_function
2061  for (Use &U : llvm::make_early_inc_range(Prepare->uses())) {
2062  // Look for bitcasts back to the original function type.
2063  auto *Cast = dyn_cast<BitCastInst>(U.getUser());
2064  if (!Cast || Cast->getType() != Fn->getType())
2065  continue;
2066 
2067  // Replace and remove the cast.
2068  Cast->replaceAllUsesWith(Fn);
2069  Cast->eraseFromParent();
2070  }
2071 
2072  // Replace any remaining uses with the function as an i8*.
2073  // This can never directly be a callee, so we don't need to update CG.
2074  Prepare->replaceAllUsesWith(CastFn);
2075  Prepare->eraseFromParent();
2076 
2077  // Kill dead bitcasts.
2078  while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
2079  if (!Cast->use_empty())
2080  break;
2081  CastFn = Cast->getOperand(0);
2082  Cast->eraseFromParent();
2083  }
2084 }
2085 
2086 static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG,
2087  LazyCallGraph::SCC &C) {
2088  bool Changed = false;
2089  for (Use &P : llvm::make_early_inc_range(PrepareFn->uses())) {
2090  // Intrinsics can only be used in calls.
2091  auto *Prepare = cast<CallInst>(P.getUser());
2092  replacePrepare(Prepare, CG, C);
2093  Changed = true;
2094  }
2095 
2096  return Changed;
2097 }
2098 
2099 static void addPrepareFunction(const Module &M,
2101  StringRef Name) {
2102  auto *PrepareFn = M.getFunction(Name);
2103  if (PrepareFn && !PrepareFn->use_empty())
2104  Fns.push_back(PrepareFn);
2105 }
2106 
2109  LazyCallGraph &CG, CGSCCUpdateResult &UR) {
2110  // NB: One invariant of a valid LazyCallGraph::SCC is that it must contain a
2111  // non-zero number of nodes, so we assume that here and grab the first
2112  // node's function's module.
2113  Module &M = *C.begin()->getFunction().getParent();
2114  auto &FAM =
2115  AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
2116 
2117  // Check for uses of llvm.coro.prepare.retcon/async.
2118  SmallVector<Function *, 2> PrepareFns;
2119  addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
2120  addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
2121 
2122  // Find coroutines for processing.
2124  for (LazyCallGraph::Node &N : C)
2125  if (N.getFunction().isPresplitCoroutine())
2126  Coroutines.push_back(&N);
2127 
2128  if (Coroutines.empty() && PrepareFns.empty())
2129  return PreservedAnalyses::all();
2130 
2131  if (Coroutines.empty()) {
2132  for (auto *PrepareFn : PrepareFns) {
2133  replaceAllPrepares(PrepareFn, CG, C);
2134  }
2135  }
2136 
2137  // Split all the coroutines.
2138  for (LazyCallGraph::Node *N : Coroutines) {
2139  Function &F = N->getFunction();
2140  LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()
2141  << "\n");
2142  F.setSplittedCoroutine();
2143 
2145  const coro::Shape Shape = splitCoroutine(
2147  updateCallGraphAfterCoroutineSplit(*N, Shape, Clones, C, CG, AM, UR, FAM);
2148 
2149  if (!Shape.CoroSuspends.empty()) {
2150  // Run the CGSCC pipeline on the original and newly split functions.
2151  UR.CWorklist.insert(&C);
2152  for (Function *Clone : Clones)
2153  UR.CWorklist.insert(CG.lookupSCC(CG.get(*Clone)));
2154  }
2155  }
2156 
2157  if (!PrepareFns.empty()) {
2158  for (auto *PrepareFn : PrepareFns) {
2159  replaceAllPrepares(PrepareFn, CG, C);
2160  }
2161  }
2162 
2163  return PreservedAnalyses::none();
2164 }
llvm::coro::Shape::CoroSizes
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:76
postSplitCleanup
static void postSplitCleanup(Function &F)
Definition: CoroSplit.cpp:1201
llvm::coro::Shape::getInsertPtAfterFramePtr
Instruction * getInsertPtAfterFramePtr() const
Definition: CoroInternal.h:240
i
i
Definition: README.txt:29
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
llvm::coro::createMustTailCall
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, ArrayRef< Value * > Arguments, IRBuilder<> &)
Definition: CoroSplit.cpp:1675
addPrepareFunction
static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)
Definition: CoroSplit.cpp:2099
Attrs
Function Attrs
Definition: README_ALTIVEC.txt:215
llvm::TargetIRAnalysis
Analysis pass providing the TargetTransformInfo.
Definition: TargetTransformInfo.h:2586
llvm::coro::Shape::AsyncLoweringStorage::FrameOffset
uint64_t FrameOffset
Definition: CoroInternal.h:133
createCloneDeclaration
static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)
Definition: CoroSplit.cpp:507
shouldBeMustTail
static bool shouldBeMustTail(const CallInst &CI, const Function &F)
Definition: CoroSplit.cpp:1344
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::DILocalScope::getSubprogram
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
Definition: DebugInfoMetadata.cpp:961
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
updateCoroFrame
static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn, Function *DestroyFn, Function *CleanupFn)
Definition: CoroSplit.cpp:1175
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:69
llvm::ReturnInst
Return a value (possibly void), from a function.
Definition: Instructions.h:3052
llvm::SmallPtrSetImpl::erase
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:379
ValueMapper.h
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::AnyCoroSuspendInst
Definition: CoroInstr.h:477
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:291
llvm::coro::Shape::SwitchLowering
SwitchLoweringStorage SwitchLowering
Definition: CoroInternal.h:141
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:104
IntrinsicInst.h
llvm::Type::isPointerTy
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:237
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
llvm::Module::iterator
FunctionListType::iterator iterator
The Function iterators.
Definition: Module.h:90
Scalar.h
llvm::coro::Shape::FrameTy
StructType * FrameTy
Definition: CoroInternal.h:98
InstIterator.h
llvm::Function
Definition: Function.h:60
getFunction
static Function * getFunction(Constant *C)
Definition: Evaluator.cpp:236
StringRef.h
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1306
llvm::TrackingVH::getValPtr
ValueTy * getValPtr() const
Definition: ValueHandle.h:335
addFramePointerAttrs
static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment)
Definition: CoroSplit.cpp:851
hasCallsInBlockBetween
static bool hasCallsInBlockBetween(Instruction *From, Instruction *To)
Definition: CoroSplit.cpp:1439
llvm::ilist_node_with_parent::getNextNode
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:289
llvm::InlineFunction
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
Definition: InlineFunction.cpp:2046
llvm::SmallVector< Value *, 8 >
llvm::CallBase::isInlineAsm
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1465
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:173
llvm::Type::getPointerAddressSpace
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:729
llvm::coro::ABI::Retcon
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::SmallDenseMap
Definition: DenseMap.h:880
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:361
Local.h
llvm::CallGraph
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:72
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
FAM
FunctionAnalysisManager FAM
Definition: PassBuilderBindings.cpp:59
llvm::verifyFunction
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
Definition: Verifier.cpp:6319
hasCallsBetween
static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)
Definition: CoroSplit.cpp:1480
llvm::coro::ABI::Switch
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
llvm::PreservedAnalyses::none
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:321
scanPHIsAndUpdateValueMap
static void scanPHIsAndUpdateValueMap(Instruction *Prev, BasicBlock *NewBlock, DenseMap< Value *, Value * > &ResolvedValues)
Definition: CoroSplit.cpp:1216
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::coro::Shape::ABI
coro::ABI ABI
Definition: CoroInternal.h:96
DenseMap.h
llvm::removeUnreachableBlocks
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:2581
Module.h
llvm::CoroSuspendAsyncInst::getResumeFunction
CoroAsyncResumeInst * getResumeFunction() const
Definition: CoroInstr.h:546
llvm::AttributeList
Definition: Attributes.h:430
llvm::CoroBeginInst::getId
AnyCoroIdInst * getId() const
Definition: CoroInstr.h:424
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1475
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1255
replaceCoroEndAsync
static bool replaceCoroEndAsync(AnyCoroEndInst *End)
Replace an llvm.coro.end.async.
Definition: CoroSplit.cpp:183
llvm::BasicBlock::splitBasicBlock
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:402
llvm::AnyCoroIdInst::getCoroAlloc
CoroAllocInst * getCoroAlloc()
Definition: CoroInstr.h:84
llvm::LLVMContext::OB_funclet
@ OB_funclet
Definition: LLVMContext.h:90
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::CloneFunctionChangeType::LocalChangesOnly
@ LocalChangesOnly
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:891
handleNoSuspendCoroutine
static void handleNoSuspendCoroutine(coro::Shape &Shape)
Definition: CoroSplit.cpp:1404
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:139
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:285
llvm::coro::Shape::SwitchLoweringStorage::ResumeSwitch
SwitchInst * ResumeSwitch
Definition: CoroInternal.h:108
replaceCoroEnd
static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Definition: CoroSplit.cpp:343
llvm::coro::buildCoroutineFrame
void buildCoroutineFrame(Function &F, Shape &Shape)
Definition: CoroFrame.cpp:2696
llvm::CallInst::TCK_MustTail
@ TCK_MustTail
Definition: Instructions.h:1654
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:237
ConstantFolding.h
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::ConstantExpr::getPointerCast
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition: Constants.cpp:2014
llvm::coro::Shape::SwitchLoweringStorage::HasUnwindCoroEnd
bool HasUnwindCoroEnd
Definition: CoroInternal.h:115
Uses
SmallPtrSet< MachineInstr *, 2 > Uses
Definition: ARMLowOverheadLoops.cpp:590
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::coro::Shape::emitDealloc
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:476
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::CoroAlignInst
This represents the llvm.coro.align instruction.
Definition: CoroInstr.h:603
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:187
PriorityWorklist.h
llvm::PrettyStackTraceEntry
PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...
Definition: PrettyStackTrace.h:52
updateAsyncFuncPointerContextSize
static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)
Definition: CoroSplit.cpp:1099
addAsyncContextAttrs
static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:862
Instruction.h
llvm::CoroIdInst::setInfo
void setInfo(Constant *C)
Definition: CoroInstr.h:180
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::coro::Shape::RetconLoweringStorage::IsFrameInlineInStorage
bool IsFrameInlineInStorage
Definition: CoroInternal.h:123
llvm::TargetTransformInfo::supportsTailCallFor
bool supportsTailCallFor(const CallBase *CB) const
If target supports tail call on CB.
Definition: TargetTransformInfo.cpp:535
GlobalValue.h
llvm::CoroSuspendAsyncInst::MustTailCallFuncArg
@ MustTailCallFuncArg
Definition: CoroInstr.h:531
llvm::LazyCallGraph::SCC
An SCC of the call graph.
Definition: LazyCallGraph.h:420
llvm::coro::Shape::AsyncLoweringStorage::AsyncFuncPointer
GlobalVariable * AsyncFuncPointer
Definition: CoroInternal.h:135
Constants.h
removeCoroEnds
static void removeCoroEnds(const coro::Shape &Shape)
Remove calls to llvm.coro.end in the original function.
Definition: CoroSplit.cpp:1997
splitAsyncCoroutine
static void splitAsyncCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1691
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::User
Definition: User.h:44
createClone
static Function * createClone(Function &F, const Twine &Suffix, coro::Shape &Shape, CoroCloner::Kind FKind)
Definition: CoroSplit.cpp:1092
addSwiftSelfAttrs
static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:869
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::ARM_PROC::A
@ A
Definition: ARMBaseInfo.h:34
Twine.h
InstrTypes.h
llvm::coro::Shape::getRetconResultTypes
ArrayRef< Type * > getRetconResultTypes() const
Definition: CoroInternal.h:198
SI
@ SI
Definition: SIInstrInfo.cpp:7966
llvm::CoroSuspendInst
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:493
llvm::ReplaceInstWithInst
void ReplaceInstWithInst(BasicBlock::InstListType &BIL, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Definition: BasicBlockUtils.cpp:559
splitRetconCoroutine
static void splitRetconCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1788
llvm::Value::uses
iterator_range< use_iterator > uses()
Definition: Value.h:376
replaceFrameSizeAndAlignment
static void replaceFrameSizeAndAlignment(coro::Shape &Shape)
Definition: CoroSplit.cpp:1114
llvm::coro::Shape::CoroSuspends
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:78
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::isPotentiallyReachable
bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)
Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...
Definition: CFG.cpp:231
llvm::coro::Shape::CoroEnds
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:75
llvm::coro::replaceCoroFree
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:130
llvm::Instruction
Definition: Instruction.h:42
llvm::coro::Shape::SwitchLoweringStorage::HasFinalSuspend
bool HasFinalSuspend
Definition: CoroInternal.h:114
InlineInfo
@ InlineInfo
Definition: FunctionInfo.cpp:25
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::salvageDebugInfo
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition: Utils.cpp:1361
llvm::BasicBlock::phis
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:364
llvm::predecessors
auto predecessors(MachineBasicBlock *BB)
Definition: MachineSSAContext.h:30
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:53
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1713
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::FunctionType::params
ArrayRef< Type * > params() const
Definition: DerivedTypes.h:130
SmallPtrSet.h
llvm::LazyCallGraph::lookupSCC
SCC * lookupSCC(Node &N) const
Lookup a function's SCC in the graph.
Definition: LazyCallGraph.h:980
LazyCallGraph.h
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::CoroSplitPass::OptimizeFrame
bool OptimizeFrame
Definition: CoroSplit.h:32
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1456
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
llvm::AttrBuilder::addAlignmentAttr
AttrBuilder & addAlignmentAttr(MaybeAlign Align)
This turns an alignment into the form used internally in Attribute.
Definition: Attributes.cpp:1747
Type.h
llvm::coro::Shape::FrameAlign
Align FrameAlign
Definition: CoroInternal.h:99
CFG.h
llvm::LazyCallGraph::get
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
Definition: LazyCallGraph.h:995
llvm::coro::Shape::getIndexType
IntegerType * getIndexType() const
Definition: CoroInternal.h:167
llvm::coro::Shape
Definition: CoroInternal.h:73
llvm::coro::Shape::AsyncLowering
AsyncLoweringStorage AsyncLowering
Definition: CoroInternal.h:143
llvm::AttrBuilder::addAttribute
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
Definition: Attributes.cpp:1701
CoroInternal.h
BasicBlock.h
hasCallsInBlocksBetween
static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)
Definition: CoroSplit.cpp:1451
llvm::DbgVariableIntrinsic
This is the common base class for debug info intrinsics for variables.
Definition: IntrinsicInst.h:180
llvm::coro::Shape::SwiftErrorOps
SmallVector< CallInst *, 2 > SwiftErrorOps
Definition: CoroInternal.h:79
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
setCoroInfo
static void setCoroInfo(Function &F, coro::Shape &Shape, ArrayRef< Function * > Fns)
Definition: CoroSplit.cpp:1151
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition: GlobalVariable.h:135
VI
@ VI
Definition: SIInstrInfo.cpp:7967
llvm::ArrayRef::drop_front
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:203
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:81
simplifySuspendPoints
static void simplifySuspendPoints(coro::Shape &Shape)
Definition: CoroSplit.cpp:1565
llvm::CoroSplitPass::run
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: CoroSplit.cpp:2107
uint64_t
llvm::updateCGAndAnalysisManagerForFunctionPass
LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a function pass.
Definition: CGSCCPassManager.cpp:1209
llvm::Function::getCallingConv
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:238
llvm::LazyCallGraph::addSplitFunction
void addSplitFunction(Function &OriginalFunction, Function &NewFunction)
Add a new function split/outlined from an existing function.
Definition: LazyCallGraph.cpp:1609
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
CoroSplit.h
llvm::ConstantPointerNull::get
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1699
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:53
llvm::BranchInst::Create
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3190
llvm::DenseMap
Definition: DenseMap.h:714
coerceArguments
static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)
Coerce the arguments in FnArgs according to FnTy in CallArgs.
Definition: CoroSplit.cpp:1660
addMustTailToCoroResumes
static void addMustTailToCoroResumes(Function &F, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1379
markCoroutineAsDone
static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)
Definition: CoroSplit.cpp:291
llvm::CoroSuspendAsyncInst
This represents the llvm.coro.suspend.async instruction.
Definition: CoroInstr.h:525
PrettyStackTrace.h
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::AttrBuilder
Definition: Attributes.h:1038
Cloning.h
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
llvm::CGSCCUpdateResult::CWorklist
SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist
Worklist of the SCCs queued for processing.
Definition: CGSCCPassManager.h:257
llvm::Attribute::AttrKind
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
Definition: Attributes.h:85
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:716
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:632
llvm::ConstantFoldCompareInstOperands
Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
Definition: ConstantFolding.cpp:1220
llvm::TrackingVH< Value >
llvm::MDString::get
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:498
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::find
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
splitCoroutine
static coro::Shape splitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI, bool OptimizeFrame)
Definition: CoroSplit.cpp:1932
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:137
llvm::ilist_node_with_parent::getPrevNode
NodeTy * getPrevNode()
Definition: ilist_node.h:275
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
replaceUnwindCoroEnd
static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace an unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:305
llvm::coro::ABI::Async
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
llvm::coro::Shape::OptimizeFrame
bool OptimizeFrame
This would only be true if optimization are enabled.
Definition: CoroInternal.h:105
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::ArrayType::get
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:638
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
llvm::LazyCallGraph::Node
A node in the call graph.
Definition: LazyCallGraph.h:316
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::User::setOperand
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
getType
static M68kRelType getType(unsigned Kind, MCSymbolRefExpr::VariantKind &Modifier, bool &IsPCRel)
Definition: M68kELFObjectWriter.cpp:48
llvm::print
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr)
Definition: GCNRegPressure.cpp:138
llvm::coro::Shape::AsyncLoweringStorage::ContextSize
uint64_t ContextSize
Definition: CoroInternal.h:134
replaceAsyncResumeFunction
static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)
Definition: CoroSplit.cpp:1645
CFG.h
replaceFallthroughCoroEnd
static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace a non-unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:226
simplifyTerminatorLeadingToRet
static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst)
Definition: CoroSplit.cpp:1233
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::isInstructionTriviallyDead
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
Definition: Local.cpp:396
llvm::CoroSuspendAsyncInst::ResumeFunctionArg
@ ResumeFunctionArg
Definition: CoroInstr.h:529
DataLayout.h
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:138
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Dwarf.h
createResumeEntryBlock
static void createResumeEntryBlock(Function &F, coro::Shape &Shape)
Definition: CoroSplit.cpp:358
replaceSwiftErrorOps
static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)
Definition: CoroSplit.cpp:627
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
updateCallGraphAfterCoroutineSplit
static void updateCallGraphAfterCoroutineSplit(LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Definition: CoroSplit.cpp:2003
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:97
uint32_t
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::coro::Shape::CoroBegin
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:74
llvm::StructType::getTypeAtIndex
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
Definition: Type.cpp:603
llvm::coro::Shape::getResumeFunctionType
FunctionType * getResumeFunctionType() const
Definition: CoroInternal.h:182
llvm::AnyCoroEndInst
Definition: CoroInstr.h:614
llvm::CoroAllocInst
This represents the llvm.coro.alloc instruction.
Definition: CoroInstr.h:70
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
llvm::ValueMap< const Value *, WeakTrackingVH >
llvm::MipsISD::TailCall
@ TailCall
Definition: MipsISelLowering.h:65
llvm::coro::Shape::SwitchLoweringStorage::ResumeEntryBlock
BasicBlock * ResumeEntryBlock
Definition: CoroInternal.h:110
llvm::coro::Shape::RetconLowering
RetconLoweringStorage RetconLowering
Definition: CoroInternal.h:142
llvm::Value::stripPointerCasts
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:685
llvm::ConstantInt::getFalse
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:834
Argument.h
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:187
llvm::coro::salvageDebugInfo
void salvageDebugInfo(SmallDenseMap< llvm::Value *, llvm::AllocaInst *, 4 > &DbgPtrAllocaCache, DbgVariableIntrinsic *DVI, bool OptimizeFrame)
Recover a dbg.declare prepared by the frontend and emit an alloca holding a pointer to the coroutine ...
Definition: CoroFrame.cpp:2606
CallingConv.h
Attributes.h
llvm::Function::addParamAttr
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
adds the attribute to the list of attributes for the given arg.
Definition: Function.cpp:567
llvm::coro::Shape::CoroAligns
SmallVector< CoroAlignInst *, 2 > CoroAligns
Definition: CoroInternal.h:77
llvm::coro::ABI::RetconOnce
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
llvm::CGSCCUpdateResult
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Definition: CGSCCPassManager.h:232
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::ConstantFoldTerminator
bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions=false, const TargetLibraryInfo *TLI=nullptr, DomTreeUpdater *DTU=nullptr)
If a terminator instruction is predicated on a constant value, convert it into an unconditional branc...
Definition: Local.cpp:126
Verifier.h
llvm::TargetTransformInfo::supportsTailCalls
bool supportsTailCalls() const
If the target supports tail calls.
Definition: TargetTransformInfo.cpp:531
llvm::ConstantInt::getTrue
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:827
llvm::LazyCallGraph::addSplitRefRecursiveFunctions
void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)
Add new ref-recursive functions split/outlined from an existing function.
Definition: LazyCallGraph.cpp:1688
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::end
iterator end()
Definition: DenseMap.h:84
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
llvm::ISD::BR
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:982
llvm::PHINode::Create
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Definition: Instructions.h:2741
llvm::ArrayRef::begin
iterator begin() const
Definition: ArrayRef.h:152
llvm::Align::value
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
GlobalVariable.h
llvm::MCID::Branch
@ Branch
Definition: MCInstrDesc.h:158
llvm::ConstantTokenNone::get
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition: Constants.cpp:1421
Casting.h
llvm::coro::Shape::getIndex
ConstantInt * getIndex(uint64_t Value) const
Definition: CoroInternal.h:172
Function.h
getFunctionTypeFromAsyncSuspend
static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)
Definition: CoroSplit.cpp:499
Arguments
AMDGPU Lower Kernel Arguments
Definition: AMDGPULowerKernelArguments.cpp:242
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:774
llvm::InlineFunctionInfo
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:203
llvm::coro::Shape::emitAlloc
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:453
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:175
llvm::ConstantArray::get
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1241
llvm::coro::Shape::SwitchFieldIndex::Destroy
@ Destroy
Definition: CoroInternal.h:85
llvm::coro::Shape::getSwitchCoroId
CoroIdInst * getSwitchCoroId() const
Definition: CoroInternal.h:146
splitSwitchCoroutine
static void splitSwitchCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1606
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition: InstrTypes.h:1390
replaceAllPrepares
static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2086
llvm::updateCGAndAnalysisManagerForCGSCCPass
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
Definition: CGSCCPassManager.cpp:1216
llvm::coro::Shape::getSwitchIndexField
unsigned getSwitchIndexField() const
Definition: CoroInternal.h:162
llvm::BasicBlock::getFirstNonPHIOrDbgOrLifetime
const Instruction * getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode, a debug intrinsic,...
Definition: BasicBlock.cpp:230
llvm::AttrBuilder::addDereferenceableAttr
AttrBuilder & addDereferenceableAttr(uint64_t Bytes)
This turns the number of dereferenceable bytes into the form used internally in Attribute.
Definition: Attributes.cpp:1764
llvm::GlobalValue::ExternalLinkage
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
CallGraph.h
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:222
llvm::changeToUnreachable
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2216
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Instructions.h
llvm::CoroSuspendInst::getCoroSave
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:497
llvm::CoroSizeInst
This represents the llvm.coro.size instruction.
Definition: CoroInstr.h:591
llvm::coro::Shape::RetconLoweringStorage::ReturnBlock
BasicBlock * ReturnBlock
Definition: CoroInternal.h:122
SmallVector.h
Dominators.h
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1342
llvm::FreezeInst
This class represents a freeze function that returns random concrete value if an operand is either a ...
Definition: Instructions.h:5438
N
#define N
maybeFreeRetconStorage
static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)
Definition: CoroSplit.cpp:169
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::SmallVectorImpl::pop_back_val
T pop_back_val()
Definition: SmallVector.h:677
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:164
TargetTransformInfo.h
llvm::PHINode
Definition: Instructions.h:2699
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:119
llvm::CoroBeginInst
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:420
llvm::BasicBlock::removePredecessor
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Definition: BasicBlock.cpp:342
CoroInstr.h
llvm::DISubprogram
Subprogram description.
Definition: DebugInfoMetadata.h:1841
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::coro::Shape::FramePtr
Value * FramePtr
Definition: CoroInternal.h:101
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1175
llvm::GlobalValue::getType
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:290
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:399
DerivedTypes.h
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
replacePrepare
static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Replace a call to llvm.coro.prepare.retcon.
Definition: CoroSplit.cpp:2050
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1474
llvm::CloneFunctionInto
void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.
Definition: CloneFunction.cpp:86
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
LLVMContext.h
llvm::coro::Shape::SwitchFieldIndex::Resume
@ Resume
Definition: CoroInternal.h:84
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:381
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
simplifySuspendPoint
static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)
Definition: CoroSplit.cpp:1505
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:394
CallGraphUpdater.h
llvm::LazyCallGraph
A lazily constructed view of the call graph of a module.
Definition: LazyCallGraph.h:111
raw_ostream.h
llvm::AMDGPU::VGPRIndexMode::Id
Id
Definition: SIDefines.h:241
llvm::SmallPtrSetImpl::contains
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:389
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition: SmallVector.h:667
llvm::FunctionAnalysisManagerCGSCCProxy
A proxy from a FunctionAnalysisManager to an SCC.
Definition: CGSCCPassManager.h:392
BasicBlockUtils.h
Value.h
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition: DerivedTypes.h:124
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
Debug.h
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
llvm::UWTableKind::Async
@ Async
"Asynchronous" unwind tables (instr precise)
llvm::ArrayRef::end
iterator end() const
Definition: ArrayRef.h:153
llvm::GlobalVariable::setInitializer
void setInitializer(Constant *InitVal)
setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...
Definition: Globals.cpp:463
llvm::FunctionType
Class to represent function types.
Definition: DerivedTypes.h:103
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365