LLVM 17.0.0git
CoroSplit.cpp
Go to the documentation of this file.
1//===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass builds the coroutine frame and outlines resume and destroy parts
9// of the coroutine into separate functions.
10//
11// We present a coroutine to an LLVM as an ordinary function with suspension
12// points marked up with intrinsics. We let the optimizer party on the coroutine
13// as a single function for as long as possible. Shortly before the coroutine is
14// eligible to be inlined into its callers, we split up the coroutine into parts
15// corresponding to an initial, resume and destroy invocations of the coroutine,
16// add them to the current SCC and restart the IPO pipeline to optimize the
17// coroutine subfunctions we extracted before proceeding to the caller of the
18// coroutine.
19//===----------------------------------------------------------------------===//
20
22#include "CoroInstr.h"
23#include "CoroInternal.h"
24#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/ADT/Twine.h"
30#include "llvm/Analysis/CFG.h"
37#include "llvm/IR/Argument.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/BasicBlock.h"
40#include "llvm/IR/CFG.h"
41#include "llvm/IR/CallingConv.h"
42#include "llvm/IR/Constants.h"
43#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/Dominators.h"
46#include "llvm/IR/Function.h"
47#include "llvm/IR/GlobalValue.h"
49#include "llvm/IR/IRBuilder.h"
51#include "llvm/IR/InstrTypes.h"
52#include "llvm/IR/Instruction.h"
55#include "llvm/IR/LLVMContext.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/Type.h"
58#include "llvm/IR/Value.h"
59#include "llvm/IR/Verifier.h"
61#include "llvm/Support/Debug.h"
70#include <cassert>
71#include <cstddef>
72#include <cstdint>
73#include <initializer_list>
74#include <iterator>
75
76using namespace llvm;
77
78#define DEBUG_TYPE "coro-split"
79
80namespace {
81
82/// A little helper class for building
83class CoroCloner {
84public:
85 enum class Kind {
86 /// The shared resume function for a switch lowering.
87 SwitchResume,
88
89 /// The shared unwind function for a switch lowering.
90 SwitchUnwind,
91
92 /// The shared cleanup function for a switch lowering.
93 SwitchCleanup,
94
95 /// An individual continuation function.
96 Continuation,
97
98 /// An async resume function.
99 Async,
100 };
101
102private:
103 Function &OrigF;
104 Function *NewF;
105 const Twine &Suffix;
106 coro::Shape &Shape;
107 Kind FKind;
110 Value *NewFramePtr = nullptr;
111
112 /// The active suspend instruction; meaningful only for continuation and async
113 /// ABIs.
114 AnyCoroSuspendInst *ActiveSuspend = nullptr;
115
116public:
117 /// Create a cloner for a switch lowering.
118 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
119 Kind FKind)
120 : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape),
121 FKind(FKind), Builder(OrigF.getContext()) {
122 assert(Shape.ABI == coro::ABI::Switch);
123 }
124
125 /// Create a cloner for a continuation lowering.
126 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
127 Function *NewF, AnyCoroSuspendInst *ActiveSuspend)
128 : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape),
129 FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation),
130 Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend) {
131 assert(Shape.ABI == coro::ABI::Retcon ||
132 Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async);
133 assert(NewF && "need existing function for continuation");
134 assert(ActiveSuspend && "need active suspend point for continuation");
135 }
136
137 Function *getFunction() const {
138 assert(NewF != nullptr && "declaration not yet set");
139 return NewF;
140 }
141
142 void create();
143
144private:
145 bool isSwitchDestroyFunction() {
146 switch (FKind) {
147 case Kind::Async:
148 case Kind::Continuation:
149 case Kind::SwitchResume:
150 return false;
151 case Kind::SwitchUnwind:
152 case Kind::SwitchCleanup:
153 return true;
154 }
155 llvm_unreachable("Unknown CoroCloner::Kind enum");
156 }
157
158 void replaceEntryBlock();
159 Value *deriveNewFramePointer();
160 void replaceRetconOrAsyncSuspendUses();
161 void replaceCoroSuspends();
162 void replaceCoroEnds();
164 void salvageDebugInfo();
165 void handleFinalSuspend();
166};
167
168} // end anonymous namespace
169
171 const coro::Shape &Shape, Value *FramePtr,
172 CallGraph *CG) {
173 assert(Shape.ABI == coro::ABI::Retcon ||
174 Shape.ABI == coro::ABI::RetconOnce);
176 return;
177
178 Shape.emitDealloc(Builder, FramePtr, CG);
179}
180
181/// Replace an llvm.coro.end.async.
182/// Will inline the must tail call function call if there is one.
183/// \returns true if cleanup of the coro.end block is needed, false otherwise.
185 IRBuilder<> Builder(End);
186
187 auto *EndAsync = dyn_cast<CoroAsyncEndInst>(End);
188 if (!EndAsync) {
189 Builder.CreateRetVoid();
190 return true /*needs cleanup of coro.end block*/;
191 }
192
193 auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();
194 if (!MustTailCallFunc) {
195 Builder.CreateRetVoid();
196 return true /*needs cleanup of coro.end block*/;
197 }
198
199 // Move the must tail call from the predecessor block into the end block.
200 auto *CoroEndBlock = End->getParent();
201 auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();
202 assert(MustTailCallFuncBlock && "Must have a single predecessor block");
203 auto It = MustTailCallFuncBlock->getTerminator()->getIterator();
204 auto *MustTailCall = cast<CallInst>(&*std::prev(It));
205 CoroEndBlock->splice(End->getIterator(), MustTailCallFuncBlock,
206 MustTailCall->getIterator());
207
208 // Insert the return instruction.
209 Builder.SetInsertPoint(End);
210 Builder.CreateRetVoid();
211 InlineFunctionInfo FnInfo;
212
213 // Remove the rest of the block, by splitting it into an unreachable block.
214 auto *BB = End->getParent();
215 BB->splitBasicBlock(End);
216 BB->getTerminator()->eraseFromParent();
217
218 auto InlineRes = InlineFunction(*MustTailCall, FnInfo);
219 assert(InlineRes.isSuccess() && "Expected inlining to succeed");
220 (void)InlineRes;
221
222 // We have cleaned up the coro.end block above.
223 return false;
224}
225
226/// Replace a non-unwind call to llvm.coro.end.
228 const coro::Shape &Shape, Value *FramePtr,
229 bool InResume, CallGraph *CG) {
230 // Start inserting right before the coro.end.
231 IRBuilder<> Builder(End);
232
233 // Create the return instruction.
234 switch (Shape.ABI) {
235 // The cloned functions in switch-lowering always return void.
236 case coro::ABI::Switch:
237 // coro.end doesn't immediately end the coroutine in the main function
238 // in this lowering, because we need to deallocate the coroutine.
239 if (!InResume)
240 return;
241 Builder.CreateRetVoid();
242 break;
243
244 // In async lowering this returns.
245 case coro::ABI::Async: {
246 bool CoroEndBlockNeedsCleanup = replaceCoroEndAsync(End);
247 if (!CoroEndBlockNeedsCleanup)
248 return;
249 break;
250 }
251
252 // In unique continuation lowering, the continuations always return void.
253 // But we may have implicitly allocated storage.
254 case coro::ABI::RetconOnce:
256 Builder.CreateRetVoid();
257 break;
258
259 // In non-unique continuation lowering, we signal completion by returning
260 // a null continuation.
261 case coro::ABI::Retcon: {
263 auto RetTy = Shape.getResumeFunctionType()->getReturnType();
264 auto RetStructTy = dyn_cast<StructType>(RetTy);
265 PointerType *ContinuationTy =
266 cast<PointerType>(RetStructTy ? RetStructTy->getElementType(0) : RetTy);
267
268 Value *ReturnValue = ConstantPointerNull::get(ContinuationTy);
269 if (RetStructTy) {
270 ReturnValue = Builder.CreateInsertValue(UndefValue::get(RetStructTy),
271 ReturnValue, 0);
272 }
273 Builder.CreateRet(ReturnValue);
274 break;
275 }
276 }
277
278 // Remove the rest of the block, by splitting it into an unreachable block.
279 auto *BB = End->getParent();
280 BB->splitBasicBlock(End);
281 BB->getTerminator()->eraseFromParent();
282}
283
284// Mark a coroutine as done, which implies that the coroutine is finished and
285// never get resumed.
286//
287// In resume-switched ABI, the done state is represented by storing zero in
288// ResumeFnAddr.
289//
290// NOTE: We couldn't omit the argument `FramePtr`. It is necessary because the
291// pointer to the frame in splitted function is not stored in `Shape`.
292static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape,
293 Value *FramePtr) {
294 assert(
295 Shape.ABI == coro::ABI::Switch &&
296 "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");
297 auto *GepIndex = Builder.CreateStructGEP(
299 "ResumeFn.addr");
300 auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
302 Builder.CreateStore(NullPtr, GepIndex);
303}
304
305/// Replace an unwind call to llvm.coro.end.
306static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
307 Value *FramePtr, bool InResume,
308 CallGraph *CG) {
309 IRBuilder<> Builder(End);
310
311 switch (Shape.ABI) {
312 // In switch-lowering, this does nothing in the main function.
313 case coro::ABI::Switch: {
314 // In C++'s specification, the coroutine should be marked as done
315 // if promise.unhandled_exception() throws. The frontend will
316 // call coro.end(true) along this path.
317 //
318 // FIXME: We should refactor this once there is other language
319 // which uses Switch-Resumed style other than C++.
321 if (!InResume)
322 return;
323 break;
324 }
325 // In async lowering this does nothing.
326 case coro::ABI::Async:
327 break;
328 // In continuation-lowering, this frees the continuation storage.
329 case coro::ABI::Retcon:
330 case coro::ABI::RetconOnce:
332 break;
333 }
334
335 // If coro.end has an associated bundle, add cleanupret instruction.
336 if (auto Bundle = End->getOperandBundle(LLVMContext::OB_funclet)) {
337 auto *FromPad = cast<CleanupPadInst>(Bundle->Inputs[0]);
338 auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);
339 End->getParent()->splitBasicBlock(End);
340 CleanupRet->getParent()->getTerminator()->eraseFromParent();
341 }
342}
343
344static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
345 Value *FramePtr, bool InResume, CallGraph *CG) {
346 if (End->isUnwind())
347 replaceUnwindCoroEnd(End, Shape, FramePtr, InResume, CG);
348 else
349 replaceFallthroughCoroEnd(End, Shape, FramePtr, InResume, CG);
350
351 auto &Context = End->getContext();
352 End->replaceAllUsesWith(InResume ? ConstantInt::getTrue(Context)
354 End->eraseFromParent();
355}
356
357// Create an entry block for a resume function with a switch that will jump to
358// suspend points.
360 assert(Shape.ABI == coro::ABI::Switch);
361 LLVMContext &C = F.getContext();
362
363 // resume.entry:
364 // %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0,
365 // i32 2
366 // % index = load i32, i32* %index.addr
367 // switch i32 %index, label %unreachable [
368 // i32 0, label %resume.0
369 // i32 1, label %resume.1
370 // ...
371 // ]
372
373 auto *NewEntry = BasicBlock::Create(C, "resume.entry", &F);
374 auto *UnreachBB = BasicBlock::Create(C, "unreachable", &F);
375
376 IRBuilder<> Builder(NewEntry);
377 auto *FramePtr = Shape.FramePtr;
378 auto *FrameTy = Shape.FrameTy;
379 auto *GepIndex = Builder.CreateStructGEP(
380 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
381 auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
382 auto *Switch =
383 Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
384 Shape.SwitchLowering.ResumeSwitch = Switch;
385
386 size_t SuspendIndex = 0;
387 for (auto *AnyS : Shape.CoroSuspends) {
388 auto *S = cast<CoroSuspendInst>(AnyS);
389 ConstantInt *IndexVal = Shape.getIndex(SuspendIndex);
390
391 // Replace CoroSave with a store to Index:
392 // %index.addr = getelementptr %f.frame... (index field number)
393 // store i32 %IndexVal, i32* %index.addr1
394 auto *Save = S->getCoroSave();
395 Builder.SetInsertPoint(Save);
396 if (S->isFinal()) {
397 // The coroutine should be marked done if it reaches the final suspend
398 // point.
400 }
401
402 // If the coroutine don't have unwind coro end, we could omit the store to
403 // the final suspend point since we could infer the coroutine is suspended
404 // at the final suspend point by the nullness of ResumeFnAddr.
405 // However, we can't skip it if the coroutine have unwind coro end. Since
406 // the coroutine reaches unwind coro end is considered suspended at the
407 // final suspend point (the ResumeFnAddr is null) but in fact the coroutine
408 // didn't complete yet. We need the IndexVal for the final suspend point
409 // to make the states clear.
410 if (!S->isFinal() || Shape.SwitchLowering.HasUnwindCoroEnd) {
411 auto *GepIndex = Builder.CreateStructGEP(
412 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
413 Builder.CreateStore(IndexVal, GepIndex);
414 }
415
416 Save->replaceAllUsesWith(ConstantTokenNone::get(C));
417 Save->eraseFromParent();
418
419 // Split block before and after coro.suspend and add a jump from an entry
420 // switch:
421 //
422 // whateverBB:
423 // whatever
424 // %0 = call i8 @llvm.coro.suspend(token none, i1 false)
425 // switch i8 %0, label %suspend[i8 0, label %resume
426 // i8 1, label %cleanup]
427 // becomes:
428 //
429 // whateverBB:
430 // whatever
431 // br label %resume.0.landing
432 //
433 // resume.0: ; <--- jump from the switch in the resume.entry
434 // %0 = tail call i8 @llvm.coro.suspend(token none, i1 false)
435 // br label %resume.0.landing
436 //
437 // resume.0.landing:
438 // %1 = phi i8[-1, %whateverBB], [%0, %resume.0]
439 // switch i8 % 1, label %suspend [i8 0, label %resume
440 // i8 1, label %cleanup]
441
442 auto *SuspendBB = S->getParent();
443 auto *ResumeBB =
444 SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));
445 auto *LandingBB = ResumeBB->splitBasicBlock(
446 S->getNextNode(), ResumeBB->getName() + Twine(".landing"));
447 Switch->addCase(IndexVal, ResumeBB);
448
449 cast<BranchInst>(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);
450 auto *PN = PHINode::Create(Builder.getInt8Ty(), 2, "", &LandingBB->front());
451 S->replaceAllUsesWith(PN);
452 PN->addIncoming(Builder.getInt8(-1), SuspendBB);
453 PN->addIncoming(S, ResumeBB);
454
455 ++SuspendIndex;
456 }
457
458 Builder.SetInsertPoint(UnreachBB);
459 Builder.CreateUnreachable();
460
461 Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
462}
463
464// In the resume function, we remove the last case (when coro::Shape is built,
465// the final suspend point (if present) is always the last element of
466// CoroSuspends array) since it is an undefined behavior to resume a coroutine
467// suspended at the final suspend point.
468// In the destroy function, if it isn't possible that the ResumeFnAddr is NULL
469// and the coroutine doesn't suspend at the final suspend point actually (this
470// is possible since the coroutine is considered suspended at the final suspend
471// point if promise.unhandled_exception() exits via an exception), we can
472// remove the last case.
473void CoroCloner::handleFinalSuspend() {
474 assert(Shape.ABI == coro::ABI::Switch &&
475 Shape.SwitchLowering.HasFinalSuspend);
476
477 if (isSwitchDestroyFunction() && Shape.SwitchLowering.HasUnwindCoroEnd)
478 return;
479
480 auto *Switch = cast<SwitchInst>(VMap[Shape.SwitchLowering.ResumeSwitch]);
481 auto FinalCaseIt = std::prev(Switch->case_end());
482 BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();
483 Switch->removeCase(FinalCaseIt);
484 if (isSwitchDestroyFunction()) {
485 BasicBlock *OldSwitchBB = Switch->getParent();
486 auto *NewSwitchBB = OldSwitchBB->splitBasicBlock(Switch, "Switch");
487 Builder.SetInsertPoint(OldSwitchBB->getTerminator());
488 auto *GepIndex = Builder.CreateStructGEP(Shape.FrameTy, NewFramePtr,
490 "ResumeFn.addr");
491 auto *Load = Builder.CreateLoad(Shape.getSwitchResumePointerType(),
492 GepIndex);
493 auto *Cond = Builder.CreateIsNull(Load);
494 Builder.CreateCondBr(Cond, ResumeBB, NewSwitchBB);
495 OldSwitchBB->getTerminator()->eraseFromParent();
496 }
497}
498
499static FunctionType *
501 auto *AsyncSuspend = cast<CoroSuspendAsyncInst>(Suspend);
502 auto *StructTy = cast<StructType>(AsyncSuspend->getType());
503 auto &Context = Suspend->getParent()->getParent()->getContext();
504 auto *VoidTy = Type::getVoidTy(Context);
505 return FunctionType::get(VoidTy, StructTy->elements(), false);
506}
507
509 const Twine &Suffix,
510 Module::iterator InsertBefore,
511 AnyCoroSuspendInst *ActiveSuspend) {
512 Module *M = OrigF.getParent();
513 auto *FnTy = (Shape.ABI != coro::ABI::Async)
514 ? Shape.getResumeFunctionType()
515 : getFunctionTypeFromAsyncSuspend(ActiveSuspend);
516
517 Function *NewF =
518 Function::Create(FnTy, GlobalValue::LinkageTypes::InternalLinkage,
519 OrigF.getName() + Suffix);
520
521 M->getFunctionList().insert(InsertBefore, NewF);
522
523 return NewF;
524}
525
526/// Replace uses of the active llvm.coro.suspend.retcon/async call with the
527/// arguments to the continuation function.
528///
529/// This assumes that the builder has a meaningful insertion point.
530void CoroCloner::replaceRetconOrAsyncSuspendUses() {
531 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||
532 Shape.ABI == coro::ABI::Async);
533
534 auto NewS = VMap[ActiveSuspend];
535 if (NewS->use_empty()) return;
536
537 // Copy out all the continuation arguments after the buffer pointer into
538 // an easily-indexed data structure for convenience.
540 // The async ABI includes all arguments -- including the first argument.
541 bool IsAsyncABI = Shape.ABI == coro::ABI::Async;
542 for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),
543 E = NewF->arg_end();
544 I != E; ++I)
545 Args.push_back(&*I);
546
547 // If the suspend returns a single scalar value, we can just do a simple
548 // replacement.
549 if (!isa<StructType>(NewS->getType())) {
550 assert(Args.size() == 1);
551 NewS->replaceAllUsesWith(Args.front());
552 return;
553 }
554
555 // Try to peephole extracts of an aggregate return.
556 for (Use &U : llvm::make_early_inc_range(NewS->uses())) {
557 auto *EVI = dyn_cast<ExtractValueInst>(U.getUser());
558 if (!EVI || EVI->getNumIndices() != 1)
559 continue;
560
561 EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);
562 EVI->eraseFromParent();
563 }
564
565 // If we have no remaining uses, we're done.
566 if (NewS->use_empty()) return;
567
568 // Otherwise, we need to create an aggregate.
569 Value *Agg = UndefValue::get(NewS->getType());
570 for (size_t I = 0, E = Args.size(); I != E; ++I)
571 Agg = Builder.CreateInsertValue(Agg, Args[I], I);
572
573 NewS->replaceAllUsesWith(Agg);
574}
575
576void CoroCloner::replaceCoroSuspends() {
577 Value *SuspendResult;
578
579 switch (Shape.ABI) {
580 // In switch lowering, replace coro.suspend with the appropriate value
581 // for the type of function we're extracting.
582 // Replacing coro.suspend with (0) will result in control flow proceeding to
583 // a resume label associated with a suspend point, replacing it with (1) will
584 // result in control flow proceeding to a cleanup label associated with this
585 // suspend point.
586 case coro::ABI::Switch:
587 SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);
588 break;
589
590 // In async lowering there are no uses of the result.
591 case coro::ABI::Async:
592 return;
593
594 // In returned-continuation lowering, the arguments from earlier
595 // continuations are theoretically arbitrary, and they should have been
596 // spilled.
597 case coro::ABI::RetconOnce:
598 case coro::ABI::Retcon:
599 return;
600 }
601
602 for (AnyCoroSuspendInst *CS : Shape.CoroSuspends) {
603 // The active suspend was handled earlier.
604 if (CS == ActiveSuspend) continue;
605
606 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[CS]);
607 MappedCS->replaceAllUsesWith(SuspendResult);
608 MappedCS->eraseFromParent();
609 }
610}
611
612void CoroCloner::replaceCoroEnds() {
613 for (AnyCoroEndInst *CE : Shape.CoroEnds) {
614 // We use a null call graph because there's no call graph node for
615 // the cloned function yet. We'll just be rebuilding that later.
616 auto *NewCE = cast<AnyCoroEndInst>(VMap[CE]);
617 replaceCoroEnd(NewCE, Shape, NewFramePtr, /*in resume*/ true, nullptr);
618 }
619}
620
622 ValueToValueMapTy *VMap) {
623 if (Shape.ABI == coro::ABI::Async && Shape.CoroSuspends.empty())
624 return;
625 Value *CachedSlot = nullptr;
626 auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {
627 if (CachedSlot) {
628 assert(cast<PointerType>(CachedSlot->getType())
629 ->isOpaqueOrPointeeTypeMatches(ValueTy) &&
630 "multiple swifterror slots in function with different types");
631 return CachedSlot;
632 }
633
634 // Check if the function has a swifterror argument.
635 for (auto &Arg : F.args()) {
636 if (Arg.isSwiftError()) {
637 CachedSlot = &Arg;
638 assert(cast<PointerType>(Arg.getType())
639 ->isOpaqueOrPointeeTypeMatches(ValueTy) &&
640 "swifterror argument does not have expected type");
641 return &Arg;
642 }
643 }
644
645 // Create a swifterror alloca.
646 IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
647 auto Alloca = Builder.CreateAlloca(ValueTy);
648 Alloca->setSwiftError(true);
649
650 CachedSlot = Alloca;
651 return Alloca;
652 };
653
654 for (CallInst *Op : Shape.SwiftErrorOps) {
655 auto MappedOp = VMap ? cast<CallInst>((*VMap)[Op]) : Op;
656 IRBuilder<> Builder(MappedOp);
657
658 // If there are no arguments, this is a 'get' operation.
659 Value *MappedResult;
660 if (Op->arg_empty()) {
661 auto ValueTy = Op->getType();
662 auto Slot = getSwiftErrorSlot(ValueTy);
663 MappedResult = Builder.CreateLoad(ValueTy, Slot);
664 } else {
665 assert(Op->arg_size() == 1);
666 auto Value = MappedOp->getArgOperand(0);
667 auto ValueTy = Value->getType();
668 auto Slot = getSwiftErrorSlot(ValueTy);
669 Builder.CreateStore(Value, Slot);
670 MappedResult = Slot;
671 }
672
673 MappedOp->replaceAllUsesWith(MappedResult);
674 MappedOp->eraseFromParent();
675 }
676
677 // If we're updating the original function, we've invalidated SwiftErrorOps.
678 if (VMap == nullptr) {
679 Shape.SwiftErrorOps.clear();
680 }
681}
682
683void CoroCloner::replaceSwiftErrorOps() {
684 ::replaceSwiftErrorOps(*NewF, Shape, &VMap);
685}
686
687void CoroCloner::salvageDebugInfo() {
690 for (auto &BB : *NewF)
691 for (auto &I : BB)
692 if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
693 Worklist.push_back(DVI);
694 for (DbgVariableIntrinsic *DVI : Worklist)
695 coro::salvageDebugInfo(DbgPtrAllocaCache, DVI, Shape.OptimizeFrame);
696
697 // Remove all salvaged dbg.declare intrinsics that became
698 // either unreachable or stale due to the CoroSplit transformation.
699 DominatorTree DomTree(*NewF);
700 auto IsUnreachableBlock = [&](BasicBlock *BB) {
701 return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
702 &DomTree);
703 };
704 for (DbgVariableIntrinsic *DVI : Worklist) {
705 if (IsUnreachableBlock(DVI->getParent()))
706 DVI->eraseFromParent();
707 else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
708 // Count all non-debuginfo uses in reachable blocks.
709 unsigned Uses = 0;
710 for (auto *User : DVI->getVariableLocationOp(0)->users())
711 if (auto *I = dyn_cast<Instruction>(User))
712 if (!isa<AllocaInst>(I) && !IsUnreachableBlock(I->getParent()))
713 ++Uses;
714 if (!Uses)
715 DVI->eraseFromParent();
716 }
717 }
718}
719
720void CoroCloner::replaceEntryBlock() {
721 // In the original function, the AllocaSpillBlock is a block immediately
722 // following the allocation of the frame object which defines GEPs for
723 // all the allocas that have been moved into the frame, and it ends by
724 // branching to the original beginning of the coroutine. Make this
725 // the entry block of the cloned function.
726 auto *Entry = cast<BasicBlock>(VMap[Shape.AllocaSpillBlock]);
727 auto *OldEntry = &NewF->getEntryBlock();
728 Entry->setName("entry" + Suffix);
729 Entry->moveBefore(OldEntry);
730 Entry->getTerminator()->eraseFromParent();
731
732 // Clear all predecessors of the new entry block. There should be
733 // exactly one predecessor, which we created when splitting out
734 // AllocaSpillBlock to begin with.
735 assert(Entry->hasOneUse());
736 auto BranchToEntry = cast<BranchInst>(Entry->user_back());
737 assert(BranchToEntry->isUnconditional());
738 Builder.SetInsertPoint(BranchToEntry);
739 Builder.CreateUnreachable();
740 BranchToEntry->eraseFromParent();
741
742 // Branch from the entry to the appropriate place.
743 Builder.SetInsertPoint(Entry);
744 switch (Shape.ABI) {
745 case coro::ABI::Switch: {
746 // In switch-lowering, we built a resume-entry block in the original
747 // function. Make the entry block branch to this.
748 auto *SwitchBB =
749 cast<BasicBlock>(VMap[Shape.SwitchLowering.ResumeEntryBlock]);
750 Builder.CreateBr(SwitchBB);
751 break;
752 }
753 case coro::ABI::Async:
754 case coro::ABI::Retcon:
755 case coro::ABI::RetconOnce: {
756 // In continuation ABIs, we want to branch to immediately after the
757 // active suspend point. Earlier phases will have put the suspend in its
758 // own basic block, so just thread our jump directly to its successor.
759 assert((Shape.ABI == coro::ABI::Async &&
760 isa<CoroSuspendAsyncInst>(ActiveSuspend)) ||
761 ((Shape.ABI == coro::ABI::Retcon ||
762 Shape.ABI == coro::ABI::RetconOnce) &&
763 isa<CoroSuspendRetconInst>(ActiveSuspend)));
764 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[ActiveSuspend]);
765 auto Branch = cast<BranchInst>(MappedCS->getNextNode());
766 assert(Branch->isUnconditional());
767 Builder.CreateBr(Branch->getSuccessor(0));
768 break;
769 }
770 }
771
772 // Any static alloca that's still being used but not reachable from the new
773 // entry needs to be moved to the new entry.
774 Function *F = OldEntry->getParent();
775 DominatorTree DT{*F};
777 auto *Alloca = dyn_cast<AllocaInst>(&I);
778 if (!Alloca || I.use_empty())
779 continue;
780 if (DT.isReachableFromEntry(I.getParent()) ||
781 !isa<ConstantInt>(Alloca->getArraySize()))
782 continue;
783 I.moveBefore(*Entry, Entry->getFirstInsertionPt());
784 }
785}
786
787/// Derive the value of the new frame pointer.
788Value *CoroCloner::deriveNewFramePointer() {
789 // Builder should be inserting to the front of the new entry block.
790
791 switch (Shape.ABI) {
792 // In switch-lowering, the argument is the frame pointer.
793 case coro::ABI::Switch:
794 return &*NewF->arg_begin();
795 // In async-lowering, one of the arguments is an async context as determined
796 // by the `llvm.coro.id.async` intrinsic. We can retrieve the async context of
797 // the resume function from the async context projection function associated
798 // with the active suspend. The frame is located as a tail to the async
799 // context header.
800 case coro::ABI::Async: {
801 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
802 auto ContextIdx = ActiveAsyncSuspend->getStorageArgumentIndex() & 0xff;
803 auto *CalleeContext = NewF->getArg(ContextIdx);
804 auto *FramePtrTy = Shape.FrameTy->getPointerTo();
805 auto *ProjectionFunc =
806 ActiveAsyncSuspend->getAsyncContextProjectionFunction();
807 auto DbgLoc =
808 cast<CoroSuspendAsyncInst>(VMap[ActiveSuspend])->getDebugLoc();
809 // Calling i8* (i8*)
810 auto *CallerContext = Builder.CreateCall(ProjectionFunc->getFunctionType(),
811 ProjectionFunc, CalleeContext);
812 CallerContext->setCallingConv(ProjectionFunc->getCallingConv());
813 CallerContext->setDebugLoc(DbgLoc);
814 // The frame is located after the async_context header.
815 auto &Context = Builder.getContext();
816 auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(
817 Type::getInt8Ty(Context), CallerContext,
818 Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr");
819 // Inline the projection function.
821 auto InlineRes = InlineFunction(*CallerContext, InlineInfo);
822 assert(InlineRes.isSuccess());
823 (void)InlineRes;
824 return Builder.CreateBitCast(FramePtrAddr, FramePtrTy);
825 }
826 // In continuation-lowering, the argument is the opaque storage.
827 case coro::ABI::Retcon:
828 case coro::ABI::RetconOnce: {
829 Argument *NewStorage = &*NewF->arg_begin();
830 auto FramePtrTy = Shape.FrameTy->getPointerTo();
831
832 // If the storage is inline, just bitcast to the storage to the frame type.
833 if (Shape.RetconLowering.IsFrameInlineInStorage)
834 return Builder.CreateBitCast(NewStorage, FramePtrTy);
835
836 // Otherwise, load the real frame from the opaque storage.
837 auto FramePtrPtr =
838 Builder.CreateBitCast(NewStorage, FramePtrTy->getPointerTo());
839 return Builder.CreateLoad(FramePtrTy, FramePtrPtr);
840 }
841 }
842 llvm_unreachable("bad ABI");
843}
844
845static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context,
846 unsigned ParamIndex, uint64_t Size,
847 Align Alignment, bool NoAlias) {
848 AttrBuilder ParamAttrs(Context);
849 ParamAttrs.addAttribute(Attribute::NonNull);
850 ParamAttrs.addAttribute(Attribute::NoUndef);
851
852 if (NoAlias)
853 ParamAttrs.addAttribute(Attribute::NoAlias);
854
855 ParamAttrs.addAlignmentAttr(Alignment);
856 ParamAttrs.addDereferenceableAttr(Size);
857 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
858}
859
860static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context,
861 unsigned ParamIndex) {
862 AttrBuilder ParamAttrs(Context);
863 ParamAttrs.addAttribute(Attribute::SwiftAsync);
864 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
865}
866
867static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context,
868 unsigned ParamIndex) {
869 AttrBuilder ParamAttrs(Context);
870 ParamAttrs.addAttribute(Attribute::SwiftSelf);
871 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
872}
873
874/// Clone the body of the original function into a resume function of
875/// some sort.
876void CoroCloner::create() {
877 // Create the new function if we don't already have one.
878 if (!NewF) {
879 NewF = createCloneDeclaration(OrigF, Shape, Suffix,
880 OrigF.getParent()->end(), ActiveSuspend);
881 }
882
883 // Replace all args with dummy instructions. If an argument is the old frame
884 // pointer, the dummy will be replaced by the new frame pointer once it is
885 // computed below. Uses of all other arguments should have already been
886 // rewritten by buildCoroutineFrame() to use loads/stores on the coroutine
887 // frame.
889 for (Argument &A : OrigF.args()) {
890 DummyArgs.push_back(new FreezeInst(UndefValue::get(A.getType())));
891 VMap[&A] = DummyArgs.back();
892 }
893
895
896 // Ignore attempts to change certain attributes of the function.
897 // TODO: maybe there should be a way to suppress this during cloning?
898 auto savedVisibility = NewF->getVisibility();
899 auto savedUnnamedAddr = NewF->getUnnamedAddr();
900 auto savedDLLStorageClass = NewF->getDLLStorageClass();
901
902 // NewF's linkage (which CloneFunctionInto does *not* change) might not
903 // be compatible with the visibility of OrigF (which it *does* change),
904 // so protect against that.
905 auto savedLinkage = NewF->getLinkage();
906 NewF->setLinkage(llvm::GlobalValue::ExternalLinkage);
907
908 CloneFunctionInto(NewF, &OrigF, VMap,
909 CloneFunctionChangeType::LocalChangesOnly, Returns);
910
911 auto &Context = NewF->getContext();
912
913 // For async functions / continuations, adjust the scope line of the
914 // clone to the line number of the suspend point. However, only
915 // adjust the scope line when the files are the same. This ensures
916 // line number and file name belong together. The scope line is
917 // associated with all pre-prologue instructions. This avoids a jump
918 // in the linetable from the function declaration to the suspend point.
919 if (DISubprogram *SP = NewF->getSubprogram()) {
920 assert(SP != OrigF.getSubprogram() && SP->isDistinct());
921 if (ActiveSuspend)
922 if (auto DL = ActiveSuspend->getDebugLoc())
923 if (SP->getFile() == DL->getFile())
924 SP->setScopeLine(DL->getLine());
925 // Update the linkage name to reflect the modified symbol name. It
926 // is necessary to update the linkage name in Swift, since the
927 // mangling changes for resume functions. It might also be the
928 // right thing to do in C++, but due to a limitation in LLVM's
929 // AsmPrinter we can only do this if the function doesn't have an
930 // abstract specification, since the DWARF backend expects the
931 // abstract specification to contain the linkage name and asserts
932 // that they are identical.
933 if (!SP->getDeclaration() && SP->getUnit() &&
934 SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift)
935 SP->replaceLinkageName(MDString::get(Context, NewF->getName()));
936 }
937
938 NewF->setLinkage(savedLinkage);
939 NewF->setVisibility(savedVisibility);
940 NewF->setUnnamedAddr(savedUnnamedAddr);
941 NewF->setDLLStorageClass(savedDLLStorageClass);
942 // The function sanitizer metadata needs to match the signature of the
943 // function it is being attached to. However this does not hold for split
944 // functions here. Thus remove the metadata for split functions.
945 if (Shape.ABI == coro::ABI::Switch &&
946 NewF->hasMetadata(LLVMContext::MD_func_sanitize))
947 NewF->eraseMetadata(LLVMContext::MD_func_sanitize);
948
949 // Replace the attributes of the new function:
950 auto OrigAttrs = NewF->getAttributes();
951 auto NewAttrs = AttributeList();
952
953 switch (Shape.ABI) {
954 case coro::ABI::Switch:
955 // Bootstrap attributes by copying function attributes from the
956 // original function. This should include optimization settings and so on.
957 NewAttrs = NewAttrs.addFnAttributes(
958 Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));
959
960 addFramePointerAttrs(NewAttrs, Context, 0, Shape.FrameSize,
961 Shape.FrameAlign, /*NoAlias=*/false);
962 break;
963 case coro::ABI::Async: {
964 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
965 if (OrigF.hasParamAttribute(Shape.AsyncLowering.ContextArgNo,
966 Attribute::SwiftAsync)) {
967 uint32_t ArgAttributeIndices =
968 ActiveAsyncSuspend->getStorageArgumentIndex();
969 auto ContextArgIndex = ArgAttributeIndices & 0xff;
970 addAsyncContextAttrs(NewAttrs, Context, ContextArgIndex);
971
972 // `swiftasync` must preceed `swiftself` so 0 is not a valid index for
973 // `swiftself`.
974 auto SwiftSelfIndex = ArgAttributeIndices >> 8;
975 if (SwiftSelfIndex)
976 addSwiftSelfAttrs(NewAttrs, Context, SwiftSelfIndex);
977 }
978
979 // Transfer the original function's attributes.
980 auto FnAttrs = OrigF.getAttributes().getFnAttrs();
981 NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, FnAttrs));
982 break;
983 }
984 case coro::ABI::Retcon:
985 case coro::ABI::RetconOnce:
986 // If we have a continuation prototype, just use its attributes,
987 // full-stop.
988 NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes();
989
990 /// FIXME: Is it really good to add the NoAlias attribute?
991 addFramePointerAttrs(NewAttrs, Context, 0,
992 Shape.getRetconCoroId()->getStorageSize(),
993 Shape.getRetconCoroId()->getStorageAlignment(),
994 /*NoAlias=*/true);
995
996 break;
997 }
998
999 switch (Shape.ABI) {
1000 // In these ABIs, the cloned functions always return 'void', and the
1001 // existing return sites are meaningless. Note that for unique
1002 // continuations, this includes the returns associated with suspends;
1003 // this is fine because we can't suspend twice.
1004 case coro::ABI::Switch:
1005 case coro::ABI::RetconOnce:
1006 // Remove old returns.
1007 for (ReturnInst *Return : Returns)
1008 changeToUnreachable(Return);
1009 break;
1010
1011 // With multi-suspend continuations, we'll already have eliminated the
1012 // original returns and inserted returns before all the suspend points,
1013 // so we want to leave any returns in place.
1014 case coro::ABI::Retcon:
1015 break;
1016 // Async lowering will insert musttail call functions at all suspend points
1017 // followed by a return.
1018 // Don't change returns to unreachable because that will trip up the verifier.
1019 // These returns should be unreachable from the clone.
1020 case coro::ABI::Async:
1021 break;
1022 }
1023
1024 NewF->setAttributes(NewAttrs);
1025 NewF->setCallingConv(Shape.getResumeFunctionCC());
1026
1027 // Set up the new entry block.
1028 replaceEntryBlock();
1029
1030 Builder.SetInsertPoint(&NewF->getEntryBlock().front());
1031 NewFramePtr = deriveNewFramePointer();
1032
1033 // Remap frame pointer.
1034 Value *OldFramePtr = VMap[Shape.FramePtr];
1035 NewFramePtr->takeName(OldFramePtr);
1036 OldFramePtr->replaceAllUsesWith(NewFramePtr);
1037
1038 // Remap vFrame pointer.
1039 auto *NewVFrame = Builder.CreateBitCast(
1040 NewFramePtr, Type::getInt8PtrTy(Builder.getContext()), "vFrame");
1041 Value *OldVFrame = cast<Value>(VMap[Shape.CoroBegin]);
1042 if (OldVFrame != NewVFrame)
1043 OldVFrame->replaceAllUsesWith(NewVFrame);
1044
1045 // All uses of the arguments should have been resolved by this point,
1046 // so we can safely remove the dummy values.
1047 for (Instruction *DummyArg : DummyArgs) {
1048 DummyArg->replaceAllUsesWith(UndefValue::get(DummyArg->getType()));
1049 DummyArg->deleteValue();
1050 }
1051
1052 switch (Shape.ABI) {
1053 case coro::ABI::Switch:
1054 // Rewrite final suspend handling as it is not done via switch (allows to
1055 // remove final case from the switch, since it is undefined behavior to
1056 // resume the coroutine suspended at the final suspend point.
1057 if (Shape.SwitchLowering.HasFinalSuspend)
1058 handleFinalSuspend();
1059 break;
1060 case coro::ABI::Async:
1061 case coro::ABI::Retcon:
1062 case coro::ABI::RetconOnce:
1063 // Replace uses of the active suspend with the corresponding
1064 // continuation-function arguments.
1065 assert(ActiveSuspend != nullptr &&
1066 "no active suspend when lowering a continuation-style coroutine");
1067 replaceRetconOrAsyncSuspendUses();
1068 break;
1069 }
1070
1071 // Handle suspends.
1072 replaceCoroSuspends();
1073
1074 // Handle swifterror.
1076
1077 // Remove coro.end intrinsics.
1078 replaceCoroEnds();
1079
1080 // Salvage debug info that points into the coroutine frame.
1082
1083 // Eliminate coro.free from the clones, replacing it with 'null' in cleanup,
1084 // to suppress deallocation code.
1085 if (Shape.ABI == coro::ABI::Switch)
1086 coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]),
1087 /*Elide=*/ FKind == CoroCloner::Kind::SwitchCleanup);
1088}
1089
1090// Create a resume clone by cloning the body of the original function, setting
1091// new entry block and replacing coro.suspend an appropriate value to force
1092// resume or cleanup pass for every suspend point.
1093static Function *createClone(Function &F, const Twine &Suffix,
1094 coro::Shape &Shape, CoroCloner::Kind FKind) {
1095 CoroCloner Cloner(F, Suffix, Shape, FKind);
1096 Cloner.create();
1097 return Cloner.getFunction();
1098}
1099
1101 assert(Shape.ABI == coro::ABI::Async);
1102
1103 auto *FuncPtrStruct = cast<ConstantStruct>(
1105 auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);
1106 auto *OrigContextSize = FuncPtrStruct->getOperand(1);
1107 auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),
1109 auto *NewFuncPtrStruct = ConstantStruct::get(
1110 FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);
1111
1112 Shape.AsyncLowering.AsyncFuncPointer->setInitializer(NewFuncPtrStruct);
1113}
1114
1116 if (Shape.ABI == coro::ABI::Async)
1118
1119 for (CoroAlignInst *CA : Shape.CoroAligns) {
1121 ConstantInt::get(CA->getType(), Shape.FrameAlign.value()));
1122 CA->eraseFromParent();
1123 }
1124
1125 if (Shape.CoroSizes.empty())
1126 return;
1127
1128 // In the same function all coro.sizes should have the same result type.
1129 auto *SizeIntrin = Shape.CoroSizes.back();
1130 Module *M = SizeIntrin->getModule();
1131 const DataLayout &DL = M->getDataLayout();
1132 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1133 auto *SizeConstant = ConstantInt::get(SizeIntrin->getType(), Size);
1134
1135 for (CoroSizeInst *CS : Shape.CoroSizes) {
1136 CS->replaceAllUsesWith(SizeConstant);
1137 CS->eraseFromParent();
1138 }
1139}
1140
1141// Create a global constant array containing pointers to functions provided and
1142// set Info parameter of CoroBegin to point at this constant. Example:
1143//
1144// @f.resumers = internal constant [2 x void(%f.frame*)*]
1145// [void(%f.frame*)* @f.resume, void(%f.frame*)* @f.destroy]
1146// define void @f() {
1147// ...
1148// call i8* @llvm.coro.begin(i8* null, i32 0, i8* null,
1149// i8* bitcast([2 x void(%f.frame*)*] * @f.resumers to i8*))
1150//
1151// Assumes that all the functions have the same signature.
1152static void setCoroInfo(Function &F, coro::Shape &Shape,
1154 // This only works under the switch-lowering ABI because coro elision
1155 // only works on the switch-lowering ABI.
1156 assert(Shape.ABI == coro::ABI::Switch);
1157
1158 SmallVector<Constant *, 4> Args(Fns.begin(), Fns.end());
1159 assert(!Args.empty());
1160 Function *Part = *Fns.begin();
1161 Module *M = Part->getParent();
1162 auto *ArrTy = ArrayType::get(Part->getType(), Args.size());
1163
1164 auto *ConstVal = ConstantArray::get(ArrTy, Args);
1165 auto *GV = new GlobalVariable(*M, ConstVal->getType(), /*isConstant=*/true,
1166 GlobalVariable::PrivateLinkage, ConstVal,
1167 F.getName() + Twine(".resumers"));
1168
1169 // Update coro.begin instruction to refer to this constant.
1170 LLVMContext &C = F.getContext();
1172 Shape.getSwitchCoroId()->setInfo(BC);
1173}
1174
1175// Store addresses of Resume/Destroy/Cleanup functions in the coroutine frame.
1176static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
1177 Function *DestroyFn, Function *CleanupFn) {
1178 assert(Shape.ABI == coro::ABI::Switch);
1179
1181
1182 auto *ResumeAddr = Builder.CreateStructGEP(
1184 "resume.addr");
1185 Builder.CreateStore(ResumeFn, ResumeAddr);
1186
1187 Value *DestroyOrCleanupFn = DestroyFn;
1188
1189 CoroIdInst *CoroId = Shape.getSwitchCoroId();
1190 if (CoroAllocInst *CA = CoroId->getCoroAlloc()) {
1191 // If there is a CoroAlloc and it returns false (meaning we elide the
1192 // allocation, use CleanupFn instead of DestroyFn).
1193 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);
1194 }
1195
1196 auto *DestroyAddr = Builder.CreateStructGEP(
1198 "destroy.addr");
1199 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);
1200}
1201
1204
1205#ifndef NDEBUG
1206 // For now, we do a mandatory verification step because we don't
1207 // entirely trust this pass. Note that we don't want to add a verifier
1208 // pass to FPM below because it will also verify all the global data.
1209 if (verifyFunction(F, &errs()))
1210 report_fatal_error("Broken function");
1211#endif
1212}
1213
1214// Assuming we arrived at the block NewBlock from Prev instruction, store
1215// PHI's incoming values in the ResolvedValues map.
1216static void
1218 DenseMap<Value *, Value *> &ResolvedValues) {
1219 auto *PrevBB = Prev->getParent();
1220 for (PHINode &PN : NewBlock->phis()) {
1221 auto V = PN.getIncomingValueForBlock(PrevBB);
1222 // See if we already resolved it.
1223 auto VI = ResolvedValues.find(V);
1224 if (VI != ResolvedValues.end())
1225 V = VI->second;
1226 // Remember the value.
1227 ResolvedValues[&PN] = V;
1228 }
1229}
1230
1231// Replace a sequence of branches leading to a ret, with a clone of a ret
1232// instruction. Suspend instruction represented by a switch, track the PHI
1233// values and select the correct case successor when possible.
1235 DenseMap<Value *, Value *> ResolvedValues;
1236 BasicBlock *UnconditionalSucc = nullptr;
1237 assert(InitialInst->getModule());
1238 const DataLayout &DL = InitialInst->getModule()->getDataLayout();
1239
1240 auto GetFirstValidInstruction = [](Instruction *I) {
1241 while (I) {
1242 // BitCastInst wouldn't generate actual code so that we could skip it.
1243 if (isa<BitCastInst>(I) || I->isDebugOrPseudoInst() ||
1244 I->isLifetimeStartOrEnd())
1245 I = I->getNextNode();
1246 else if (isInstructionTriviallyDead(I))
1247 // Duing we are in the middle of the transformation, we need to erase
1248 // the dead instruction manually.
1249 I = &*I->eraseFromParent();
1250 else
1251 break;
1252 }
1253 return I;
1254 };
1255
1256 auto TryResolveConstant = [&ResolvedValues](Value *V) {
1257 auto It = ResolvedValues.find(V);
1258 if (It != ResolvedValues.end())
1259 V = It->second;
1260 return dyn_cast<ConstantInt>(V);
1261 };
1262
1263 Instruction *I = InitialInst;
1264 while (I->isTerminator() || isa<CmpInst>(I)) {
1265 if (isa<ReturnInst>(I)) {
1266 if (I != InitialInst) {
1267 // If InitialInst is an unconditional branch,
1268 // remove PHI values that come from basic block of InitialInst
1269 if (UnconditionalSucc)
1270 UnconditionalSucc->removePredecessor(InitialInst->getParent(), true);
1271 ReplaceInstWithInst(InitialInst, I->clone());
1272 }
1273 return true;
1274 }
1275 if (auto *BR = dyn_cast<BranchInst>(I)) {
1276 if (BR->isUnconditional()) {
1277 BasicBlock *Succ = BR->getSuccessor(0);
1278 if (I == InitialInst)
1279 UnconditionalSucc = Succ;
1280 scanPHIsAndUpdateValueMap(I, Succ, ResolvedValues);
1281 I = GetFirstValidInstruction(Succ->getFirstNonPHIOrDbgOrLifetime());
1282 continue;
1283 }
1284
1285 BasicBlock *BB = BR->getParent();
1286 // Handle the case the condition of the conditional branch is constant.
1287 // e.g.,
1288 //
1289 // br i1 false, label %cleanup, label %CoroEnd
1290 //
1291 // It is possible during the transformation. We could continue the
1292 // simplifying in this case.
1293 if (ConstantFoldTerminator(BB, /*DeleteDeadConditions=*/true)) {
1294 // Handle this branch in next iteration.
1295 I = BB->getTerminator();
1296 continue;
1297 }
1298 } else if (auto *CondCmp = dyn_cast<CmpInst>(I)) {
1299 // If the case number of suspended switch instruction is reduced to
1300 // 1, then it is simplified to CmpInst in llvm::ConstantFoldTerminator.
1301 auto *BR = dyn_cast<BranchInst>(
1302 GetFirstValidInstruction(CondCmp->getNextNode()));
1303 if (!BR || !BR->isConditional() || CondCmp != BR->getCondition())
1304 return false;
1305
1306 // And the comparsion looks like : %cond = icmp eq i8 %V, constant.
1307 // So we try to resolve constant for the first operand only since the
1308 // second operand should be literal constant by design.
1309 ConstantInt *Cond0 = TryResolveConstant(CondCmp->getOperand(0));
1310 auto *Cond1 = dyn_cast<ConstantInt>(CondCmp->getOperand(1));
1311 if (!Cond0 || !Cond1)
1312 return false;
1313
1314 // Both operands of the CmpInst are Constant. So that we could evaluate
1315 // it immediately to get the destination.
1316 auto *ConstResult =
1317 dyn_cast_or_null<ConstantInt>(ConstantFoldCompareInstOperands(
1318 CondCmp->getPredicate(), Cond0, Cond1, DL));
1319 if (!ConstResult)
1320 return false;
1321
1322 CondCmp->replaceAllUsesWith(ConstResult);
1323 CondCmp->eraseFromParent();
1324
1325 // Handle this branch in next iteration.
1326 I = BR;
1327 continue;
1328 } else if (auto *SI = dyn_cast<SwitchInst>(I)) {
1329 ConstantInt *Cond = TryResolveConstant(SI->getCondition());
1330 if (!Cond)
1331 return false;
1332
1333 BasicBlock *BB = SI->findCaseValue(Cond)->getCaseSuccessor();
1334 scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
1335 I = GetFirstValidInstruction(BB->getFirstNonPHIOrDbgOrLifetime());
1336 continue;
1337 }
1338
1339 return false;
1340 }
1341 return false;
1342}
1343
1344// Check whether CI obeys the rules of musttail attribute.
1345static bool shouldBeMustTail(const CallInst &CI, const Function &F) {
1346 if (CI.isInlineAsm())
1347 return false;
1348
1349 // Match prototypes and calling conventions of resume function.
1350 FunctionType *CalleeTy = CI.getFunctionType();
1351 if (!CalleeTy->getReturnType()->isVoidTy() || (CalleeTy->getNumParams() != 1))
1352 return false;
1353
1354 Type *CalleeParmTy = CalleeTy->getParamType(0);
1355 if (!CalleeParmTy->isPointerTy() ||
1356 (CalleeParmTy->getPointerAddressSpace() != 0))
1357 return false;
1358
1359 if (CI.getCallingConv() != F.getCallingConv())
1360 return false;
1361
1362 // CI should not has any ABI-impacting function attributes.
1363 static const Attribute::AttrKind ABIAttrs[] = {
1364 Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
1365 Attribute::Preallocated, Attribute::InReg, Attribute::Returned,
1366 Attribute::SwiftSelf, Attribute::SwiftError};
1367 AttributeList Attrs = CI.getAttributes();
1368 for (auto AK : ABIAttrs)
1369 if (Attrs.hasParamAttr(0, AK))
1370 return false;
1371
1372 return true;
1373}
1374
1375// Add musttail to any resume instructions that is immediately followed by a
1376// suspend (i.e. ret). We do this even in -O0 to support guaranteed tail call
1377// for symmetrical coroutine control transfer (C++ Coroutines TS extension).
1378// This transformation is done only in the resume part of the coroutine that has
1379// identical signature and calling convention as the coro.resume call.
1381 bool changed = false;
1382
1383 // Collect potential resume instructions.
1385 for (auto &I : instructions(F))
1386 if (auto *Call = dyn_cast<CallInst>(&I))
1387 if (shouldBeMustTail(*Call, F))
1388 Resumes.push_back(Call);
1389
1390 // Set musttail on those that are followed by a ret instruction.
1391 for (CallInst *Call : Resumes)
1392 // Skip targets which don't support tail call on the specific case.
1393 if (TTI.supportsTailCallFor(Call) &&
1394 simplifyTerminatorLeadingToRet(Call->getNextNode())) {
1395 Call->setTailCallKind(CallInst::TCK_MustTail);
1396 changed = true;
1397 }
1398
1399 if (changed)
1401}
1402
1403// Coroutine has no suspend points. Remove heap allocation for the coroutine
1404// frame if possible.
1406 auto *CoroBegin = Shape.CoroBegin;
1407 auto *CoroId = CoroBegin->getId();
1408 auto *AllocInst = CoroId->getCoroAlloc();
1409 switch (Shape.ABI) {
1410 case coro::ABI::Switch: {
1411 auto SwitchId = cast<CoroIdInst>(CoroId);
1412 coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr);
1413 if (AllocInst) {
1414 IRBuilder<> Builder(AllocInst);
1415 auto *Frame = Builder.CreateAlloca(Shape.FrameTy);
1416 Frame->setAlignment(Shape.FrameAlign);
1417 auto *VFrame = Builder.CreateBitCast(Frame, Builder.getInt8PtrTy());
1418 AllocInst->replaceAllUsesWith(Builder.getFalse());
1419 AllocInst->eraseFromParent();
1420 CoroBegin->replaceAllUsesWith(VFrame);
1421 } else {
1422 CoroBegin->replaceAllUsesWith(CoroBegin->getMem());
1423 }
1424
1425 break;
1426 }
1427 case coro::ABI::Async:
1428 case coro::ABI::Retcon:
1429 case coro::ABI::RetconOnce:
1430 CoroBegin->replaceAllUsesWith(UndefValue::get(CoroBegin->getType()));
1431 break;
1432 }
1433
1434 CoroBegin->eraseFromParent();
1435}
1436
1437// SimplifySuspendPoint needs to check that there is no calls between
1438// coro_save and coro_suspend, since any of the calls may potentially resume
1439// the coroutine and if that is the case we cannot eliminate the suspend point.
1441 for (Instruction *I = From; I != To; I = I->getNextNode()) {
1442 // Assume that no intrinsic can resume the coroutine.
1443 if (isa<IntrinsicInst>(I))
1444 continue;
1445
1446 if (isa<CallBase>(I))
1447 return true;
1448 }
1449 return false;
1450}
1451
1452static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) {
1455
1456 Set.insert(SaveBB);
1457 Worklist.push_back(ResDesBB);
1458
1459 // Accumulate all blocks between SaveBB and ResDesBB. Because CoroSaveIntr
1460 // returns a token consumed by suspend instruction, all blocks in between
1461 // will have to eventually hit SaveBB when going backwards from ResDesBB.
1462 while (!Worklist.empty()) {
1463 auto *BB = Worklist.pop_back_val();
1464 Set.insert(BB);
1465 for (auto *Pred : predecessors(BB))
1466 if (!Set.contains(Pred))
1467 Worklist.push_back(Pred);
1468 }
1469
1470 // SaveBB and ResDesBB are checked separately in hasCallsBetween.
1471 Set.erase(SaveBB);
1472 Set.erase(ResDesBB);
1473
1474 for (auto *BB : Set)
1475 if (hasCallsInBlockBetween(BB->getFirstNonPHI(), nullptr))
1476 return true;
1477
1478 return false;
1479}
1480
1481static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy) {
1482 auto *SaveBB = Save->getParent();
1483 auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();
1484
1485 if (SaveBB == ResumeOrDestroyBB)
1486 return hasCallsInBlockBetween(Save->getNextNode(), ResumeOrDestroy);
1487
1488 // Any calls from Save to the end of the block?
1489 if (hasCallsInBlockBetween(Save->getNextNode(), nullptr))
1490 return true;
1491
1492 // Any calls from begging of the block up to ResumeOrDestroy?
1493 if (hasCallsInBlockBetween(ResumeOrDestroyBB->getFirstNonPHI(),
1494 ResumeOrDestroy))
1495 return true;
1496
1497 // Any calls in all of the blocks between SaveBB and ResumeOrDestroyBB?
1498 if (hasCallsInBlocksBetween(SaveBB, ResumeOrDestroyBB))
1499 return true;
1500
1501 return false;
1502}
1503
1504// If a SuspendIntrin is preceded by Resume or Destroy, we can eliminate the
1505// suspend point and replace it with nornal control flow.
1507 CoroBeginInst *CoroBegin) {
1508 Instruction *Prev = Suspend->getPrevNode();
1509 if (!Prev) {
1510 auto *Pred = Suspend->getParent()->getSinglePredecessor();
1511 if (!Pred)
1512 return false;
1513 Prev = Pred->getTerminator();
1514 }
1515
1516 CallBase *CB = dyn_cast<CallBase>(Prev);
1517 if (!CB)
1518 return false;
1519
1520 auto *Callee = CB->getCalledOperand()->stripPointerCasts();
1521
1522 // See if the callsite is for resumption or destruction of the coroutine.
1523 auto *SubFn = dyn_cast<CoroSubFnInst>(Callee);
1524 if (!SubFn)
1525 return false;
1526
1527 // Does not refer to the current coroutine, we cannot do anything with it.
1528 if (SubFn->getFrame() != CoroBegin)
1529 return false;
1530
1531 // See if the transformation is safe. Specifically, see if there are any
1532 // calls in between Save and CallInstr. They can potenitally resume the
1533 // coroutine rendering this optimization unsafe.
1534 auto *Save = Suspend->getCoroSave();
1535 if (hasCallsBetween(Save, CB))
1536 return false;
1537
1538 // Replace llvm.coro.suspend with the value that results in resumption over
1539 // the resume or cleanup path.
1540 Suspend->replaceAllUsesWith(SubFn->getRawIndex());
1541 Suspend->eraseFromParent();
1542 Save->eraseFromParent();
1543
1544 // No longer need a call to coro.resume or coro.destroy.
1545 if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
1546 BranchInst::Create(Invoke->getNormalDest(), Invoke);
1547 }
1548
1549 // Grab the CalledValue from CB before erasing the CallInstr.
1550 auto *CalledValue = CB->getCalledOperand();
1551 CB->eraseFromParent();
1552
1553 // If no more users remove it. Usually it is a bitcast of SubFn.
1554 if (CalledValue != SubFn && CalledValue->user_empty())
1555 if (auto *I = dyn_cast<Instruction>(CalledValue))
1556 I->eraseFromParent();
1557
1558 // Now we are good to remove SubFn.
1559 if (SubFn->user_empty())
1560 SubFn->eraseFromParent();
1561
1562 return true;
1563}
1564
1565// Remove suspend points that are simplified.
1567 // Currently, the only simplification we do is switch-lowering-specific.
1568 if (Shape.ABI != coro::ABI::Switch)
1569 return;
1570
1571 auto &S = Shape.CoroSuspends;
1572 size_t I = 0, N = S.size();
1573 if (N == 0)
1574 return;
1575
1576 size_t ChangedFinalIndex = std::numeric_limits<size_t>::max();
1577 while (true) {
1578 auto SI = cast<CoroSuspendInst>(S[I]);
1579 // Leave final.suspend to handleFinalSuspend since it is undefined behavior
1580 // to resume a coroutine suspended at the final suspend point.
1581 if (!SI->isFinal() && simplifySuspendPoint(SI, Shape.CoroBegin)) {
1582 if (--N == I)
1583 break;
1584
1585 std::swap(S[I], S[N]);
1586
1587 if (cast<CoroSuspendInst>(S[I])->isFinal()) {
1589 ChangedFinalIndex = I;
1590 }
1591
1592 continue;
1593 }
1594 if (++I == N)
1595 break;
1596 }
1597 S.resize(N);
1598
1599 // Maintain final.suspend in case final suspend was swapped.
1600 // Due to we requrie the final suspend to be the last element of CoroSuspends.
1601 if (ChangedFinalIndex < N) {
1602 assert(cast<CoroSuspendInst>(S[ChangedFinalIndex])->isFinal());
1603 std::swap(S[ChangedFinalIndex], S.back());
1604 }
1605}
1606
1610 assert(Shape.ABI == coro::ABI::Switch);
1611
1612 createResumeEntryBlock(F, Shape);
1613 auto ResumeClone = createClone(F, ".resume", Shape,
1614 CoroCloner::Kind::SwitchResume);
1615 auto DestroyClone = createClone(F, ".destroy", Shape,
1616 CoroCloner::Kind::SwitchUnwind);
1617 auto CleanupClone = createClone(F, ".cleanup", Shape,
1618 CoroCloner::Kind::SwitchCleanup);
1619
1620 postSplitCleanup(*ResumeClone);
1621 postSplitCleanup(*DestroyClone);
1622 postSplitCleanup(*CleanupClone);
1623
1624 // Adding musttail call to support symmetric transfer.
1625 // Skip targets which don't support tail call.
1626 //
1627 // FIXME: Could we support symmetric transfer effectively without musttail
1628 // call?
1629 if (TTI.supportsTailCalls())
1630 addMustTailToCoroResumes(*ResumeClone, TTI);
1631
1632 // Store addresses resume/destroy/cleanup functions in the coroutine frame.
1633 updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
1634
1635 assert(Clones.empty());
1636 Clones.push_back(ResumeClone);
1637 Clones.push_back(DestroyClone);
1638 Clones.push_back(CleanupClone);
1639
1640 // Create a constant array referring to resume/destroy/clone functions pointed
1641 // by the last argument of @llvm.coro.info, so that CoroElide pass can
1642 // determined correct function to call.
1643 setCoroInfo(F, Shape, Clones);
1644}
1645
1647 Value *Continuation) {
1648 auto *ResumeIntrinsic = Suspend->getResumeFunction();
1649 auto &Context = Suspend->getParent()->getParent()->getContext();
1650 auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1651
1652 IRBuilder<> Builder(ResumeIntrinsic);
1653 auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy);
1654 ResumeIntrinsic->replaceAllUsesWith(Val);
1655 ResumeIntrinsic->eraseFromParent();
1657 UndefValue::get(Int8PtrTy));
1658}
1659
1660/// Coerce the arguments in \p FnArgs according to \p FnTy in \p CallArgs.
1661static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy,
1662 ArrayRef<Value *> FnArgs,
1663 SmallVectorImpl<Value *> &CallArgs) {
1664 size_t ArgIdx = 0;
1665 for (auto *paramTy : FnTy->params()) {
1666 assert(ArgIdx < FnArgs.size());
1667 if (paramTy != FnArgs[ArgIdx]->getType())
1668 CallArgs.push_back(
1669 Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy));
1670 else
1671 CallArgs.push_back(FnArgs[ArgIdx]);
1672 ++ArgIdx;
1673 }
1674}
1675
1678 IRBuilder<> &Builder) {
1679 auto *FnTy = MustTailCallFn->getFunctionType();
1680 // Coerce the arguments, llvm optimizations seem to ignore the types in
1681 // vaarg functions and throws away casts in optimized mode.
1682 SmallVector<Value *, 8> CallArgs;
1683 coerceArguments(Builder, FnTy, Arguments, CallArgs);
1684
1685 auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);
1686 TailCall->setTailCallKind(CallInst::TCK_MustTail);
1687 TailCall->setDebugLoc(Loc);
1688 TailCall->setCallingConv(MustTailCallFn->getCallingConv());
1689 return TailCall;
1690}
1691
1694 assert(Shape.ABI == coro::ABI::Async);
1695 assert(Clones.empty());
1696 // Reset various things that the optimizer might have decided it
1697 // "knows" about the coroutine function due to not seeing a return.
1698 F.removeFnAttr(Attribute::NoReturn);
1699 F.removeRetAttr(Attribute::NoAlias);
1700 F.removeRetAttr(Attribute::NonNull);
1701
1702 auto &Context = F.getContext();
1703 auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1704
1705 auto *Id = cast<CoroIdAsyncInst>(Shape.CoroBegin->getId());
1706 IRBuilder<> Builder(Id);
1707
1708 auto *FramePtr = Id->getStorage();
1709 FramePtr = Builder.CreateBitOrPointerCast(FramePtr, Int8PtrTy);
1710 FramePtr = Builder.CreateConstInBoundsGEP1_32(
1712 "async.ctx.frameptr");
1713
1714 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1715 {
1716 // Make sure we don't invalidate Shape.FramePtr.
1717 TrackingVH<Value> Handle(Shape.FramePtr);
1719 Shape.FramePtr = Handle.getValPtr();
1720 }
1721
1722 // Create all the functions in order after the main function.
1723 auto NextF = std::next(F.getIterator());
1724
1725 // Create a continuation function for each of the suspend points.
1726 Clones.reserve(Shape.CoroSuspends.size());
1727 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1728 auto *Suspend = cast<CoroSuspendAsyncInst>(Shape.CoroSuspends[Idx]);
1729
1730 // Create the clone declaration.
1731 auto ResumeNameSuffix = ".resume.";
1732 auto ProjectionFunctionName =
1733 Suspend->getAsyncContextProjectionFunction()->getName();
1734 bool UseSwiftMangling = false;
1735 if (ProjectionFunctionName.equals("__swift_async_resume_project_context")) {
1736 ResumeNameSuffix = "TQ";
1737 UseSwiftMangling = true;
1738 } else if (ProjectionFunctionName.equals(
1739 "__swift_async_resume_get_context")) {
1740 ResumeNameSuffix = "TY";
1741 UseSwiftMangling = true;
1742 }
1743 auto *Continuation = createCloneDeclaration(
1744 F, Shape,
1745 UseSwiftMangling ? ResumeNameSuffix + Twine(Idx) + "_"
1746 : ResumeNameSuffix + Twine(Idx),
1747 NextF, Suspend);
1748 Clones.push_back(Continuation);
1749
1750 // Insert a branch to a new return block immediately before the suspend
1751 // point.
1752 auto *SuspendBB = Suspend->getParent();
1753 auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1754 auto *Branch = cast<BranchInst>(SuspendBB->getTerminator());
1755
1756 // Place it before the first suspend.
1757 auto *ReturnBB =
1758 BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
1759 Branch->setSuccessor(0, ReturnBB);
1760
1761 IRBuilder<> Builder(ReturnBB);
1762
1763 // Insert the call to the tail call function and inline it.
1764 auto *Fn = Suspend->getMustTailCallFunction();
1765 SmallVector<Value *, 8> Args(Suspend->args());
1766 auto FnArgs = ArrayRef<Value *>(Args).drop_front(
1768 auto *TailCall =
1769 coro::createMustTailCall(Suspend->getDebugLoc(), Fn, FnArgs, Builder);
1770 Builder.CreateRetVoid();
1771 InlineFunctionInfo FnInfo;
1772 auto InlineRes = InlineFunction(*TailCall, FnInfo);
1773 assert(InlineRes.isSuccess() && "Expected inlining to succeed");
1774 (void)InlineRes;
1775
1776 // Replace the lvm.coro.async.resume intrisic call.
1777 replaceAsyncResumeFunction(Suspend, Continuation);
1778 }
1779
1780 assert(Clones.size() == Shape.CoroSuspends.size());
1781 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1782 auto *Suspend = Shape.CoroSuspends[Idx];
1783 auto *Clone = Clones[Idx];
1784
1785 CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend).create();
1786 }
1787}
1788
1791 assert(Shape.ABI == coro::ABI::Retcon ||
1792 Shape.ABI == coro::ABI::RetconOnce);
1793 assert(Clones.empty());
1794
1795 // Reset various things that the optimizer might have decided it
1796 // "knows" about the coroutine function due to not seeing a return.
1797 F.removeFnAttr(Attribute::NoReturn);
1798 F.removeRetAttr(Attribute::NoAlias);
1799 F.removeRetAttr(Attribute::NonNull);
1800
1801 // Allocate the frame.
1802 auto *Id = cast<AnyCoroIdRetconInst>(Shape.CoroBegin->getId());
1803 Value *RawFramePtr;
1805 RawFramePtr = Id->getStorage();
1806 } else {
1807 IRBuilder<> Builder(Id);
1808
1809 // Determine the size of the frame.
1810 const DataLayout &DL = F.getParent()->getDataLayout();
1811 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1812
1813 // Allocate. We don't need to update the call graph node because we're
1814 // going to recompute it from scratch after splitting.
1815 // FIXME: pass the required alignment
1816 RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr);
1817 RawFramePtr =
1818 Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType());
1819
1820 // Stash the allocated frame pointer in the continuation storage.
1821 auto Dest = Builder.CreateBitCast(Id->getStorage(),
1822 RawFramePtr->getType()->getPointerTo());
1823 Builder.CreateStore(RawFramePtr, Dest);
1824 }
1825
1826 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1827 {
1828 // Make sure we don't invalidate Shape.FramePtr.
1829 TrackingVH<Value> Handle(Shape.FramePtr);
1830 Shape.CoroBegin->replaceAllUsesWith(RawFramePtr);
1831 Shape.FramePtr = Handle.getValPtr();
1832 }
1833
1834 // Create a unique return block.
1835 BasicBlock *ReturnBB = nullptr;
1836 SmallVector<PHINode *, 4> ReturnPHIs;
1837
1838 // Create all the functions in order after the main function.
1839 auto NextF = std::next(F.getIterator());
1840
1841 // Create a continuation function for each of the suspend points.
1842 Clones.reserve(Shape.CoroSuspends.size());
1843 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1844 auto Suspend = cast<CoroSuspendRetconInst>(Shape.CoroSuspends[i]);
1845
1846 // Create the clone declaration.
1847 auto Continuation =
1848 createCloneDeclaration(F, Shape, ".resume." + Twine(i), NextF, nullptr);
1849 Clones.push_back(Continuation);
1850
1851 // Insert a branch to the unified return block immediately before
1852 // the suspend point.
1853 auto SuspendBB = Suspend->getParent();
1854 auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1855 auto Branch = cast<BranchInst>(SuspendBB->getTerminator());
1856
1857 // Create the unified return block.
1858 if (!ReturnBB) {
1859 // Place it before the first suspend.
1860 ReturnBB = BasicBlock::Create(F.getContext(), "coro.return", &F,
1861 NewSuspendBB);
1862 Shape.RetconLowering.ReturnBlock = ReturnBB;
1863
1864 IRBuilder<> Builder(ReturnBB);
1865
1866 // Create PHIs for all the return values.
1867 assert(ReturnPHIs.empty());
1868
1869 // First, the continuation.
1870 ReturnPHIs.push_back(Builder.CreatePHI(Continuation->getType(),
1871 Shape.CoroSuspends.size()));
1872
1873 // Next, all the directly-yielded values.
1874 for (auto *ResultTy : Shape.getRetconResultTypes())
1875 ReturnPHIs.push_back(Builder.CreatePHI(ResultTy,
1876 Shape.CoroSuspends.size()));
1877
1878 // Build the return value.
1879 auto RetTy = F.getReturnType();
1880
1881 // Cast the continuation value if necessary.
1882 // We can't rely on the types matching up because that type would
1883 // have to be infinite.
1884 auto CastedContinuationTy =
1885 (ReturnPHIs.size() == 1 ? RetTy : RetTy->getStructElementType(0));
1886 auto *CastedContinuation =
1887 Builder.CreateBitCast(ReturnPHIs[0], CastedContinuationTy);
1888
1889 Value *RetV;
1890 if (ReturnPHIs.size() == 1) {
1891 RetV = CastedContinuation;
1892 } else {
1893 RetV = UndefValue::get(RetTy);
1894 RetV = Builder.CreateInsertValue(RetV, CastedContinuation, 0);
1895 for (size_t I = 1, E = ReturnPHIs.size(); I != E; ++I)
1896 RetV = Builder.CreateInsertValue(RetV, ReturnPHIs[I], I);
1897 }
1898
1899 Builder.CreateRet(RetV);
1900 }
1901
1902 // Branch to the return block.
1903 Branch->setSuccessor(0, ReturnBB);
1904 ReturnPHIs[0]->addIncoming(Continuation, SuspendBB);
1905 size_t NextPHIIndex = 1;
1906 for (auto &VUse : Suspend->value_operands())
1907 ReturnPHIs[NextPHIIndex++]->addIncoming(&*VUse, SuspendBB);
1908 assert(NextPHIIndex == ReturnPHIs.size());
1909 }
1910
1911 assert(Clones.size() == Shape.CoroSuspends.size());
1912 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1913 auto Suspend = Shape.CoroSuspends[i];
1914 auto Clone = Clones[i];
1915
1916 CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend).create();
1917 }
1918}
1919
1920namespace {
1921 class PrettyStackTraceFunction : public PrettyStackTraceEntry {
1922 Function &F;
1923 public:
1924 PrettyStackTraceFunction(Function &F) : F(F) {}
1925 void print(raw_ostream &OS) const override {
1926 OS << "While splitting coroutine ";
1927 F.printAsOperand(OS, /*print type*/ false, F.getParent());
1928 OS << "\n";
1929 }
1930 };
1931}
1932
1933static coro::Shape
1935 TargetTransformInfo &TTI, bool OptimizeFrame,
1936 std::function<bool(Instruction &)> MaterializableCallback) {
1937 PrettyStackTraceFunction prettyStackTrace(F);
1938
1939 // The suspend-crossing algorithm in buildCoroutineFrame get tripped
1940 // up by uses in unreachable blocks, so remove them as a first pass.
1942
1943 coro::Shape Shape(F, OptimizeFrame);
1944 if (!Shape.CoroBegin)
1945 return Shape;
1946
1947 simplifySuspendPoints(Shape);
1948 buildCoroutineFrame(F, Shape, MaterializableCallback);
1950
1951 // If there are no suspend points, no split required, just remove
1952 // the allocation and deallocation blocks, they are not needed.
1953 if (Shape.CoroSuspends.empty()) {
1955 } else {
1956 switch (Shape.ABI) {
1957 case coro::ABI::Switch:
1958 splitSwitchCoroutine(F, Shape, Clones, TTI);
1959 break;
1960 case coro::ABI::Async:
1961 splitAsyncCoroutine(F, Shape, Clones);
1962 break;
1963 case coro::ABI::Retcon:
1964 case coro::ABI::RetconOnce:
1965 splitRetconCoroutine(F, Shape, Clones);
1966 break;
1967 }
1968 }
1969
1970 // Replace all the swifterror operations in the original function.
1971 // This invalidates SwiftErrorOps in the Shape.
1972 replaceSwiftErrorOps(F, Shape, nullptr);
1973
1974 // Finally, salvage the llvm.dbg.declare in our original function that point
1975 // into the coroutine frame. We only do this for the current function since
1976 // the Cloner salvaged debug info for us in the new coroutine funclets.
1979 for (auto &BB : F) {
1980 for (auto &I : BB) {
1981 if (auto *DDI = dyn_cast<DbgDeclareInst>(&I)) {
1982 Worklist.push_back(DDI);
1983 continue;
1984 }
1985 }
1986 }
1987 for (auto *DDI : Worklist)
1988 coro::salvageDebugInfo(DbgPtrAllocaCache, DDI, Shape.OptimizeFrame);
1989
1990 return Shape;
1991}
1992
1993/// Remove calls to llvm.coro.end in the original function.
1994static void removeCoroEnds(const coro::Shape &Shape) {
1995 for (auto *End : Shape.CoroEnds) {
1996 replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, nullptr);
1997 }
1998}
1999
2001 LazyCallGraph::Node &N, const coro::Shape &Shape,
2005 if (!Shape.CoroBegin)
2006 return;
2007
2008 if (Shape.ABI != coro::ABI::Switch)
2009 removeCoroEnds(Shape);
2010 else {
2011 for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) {
2012 auto &Context = End->getContext();
2013 End->replaceAllUsesWith(ConstantInt::getFalse(Context));
2014 End->eraseFromParent();
2015 }
2016 }
2017
2018 if (!Clones.empty()) {
2019 switch (Shape.ABI) {
2020 case coro::ABI::Switch:
2021 // Each clone in the Switch lowering is independent of the other clones.
2022 // Let the LazyCallGraph know about each one separately.
2023 for (Function *Clone : Clones)
2024 CG.addSplitFunction(N.getFunction(), *Clone);
2025 break;
2026 case coro::ABI::Async:
2027 case coro::ABI::Retcon:
2028 case coro::ABI::RetconOnce:
2029 // Each clone in the Async/Retcon lowering references of the other clones.
2030 // Let the LazyCallGraph know about all of them at once.
2031 if (!Clones.empty())
2032 CG.addSplitRefRecursiveFunctions(N.getFunction(), Clones);
2033 break;
2034 }
2035
2036 // Let the CGSCC infra handle the changes to the original function.
2038 }
2039
2040 // Do some cleanup and let the CGSCC infra see if we've cleaned up any edges
2041 // to the split functions.
2042 postSplitCleanup(N.getFunction());
2044}
2045
2046/// Replace a call to llvm.coro.prepare.retcon.
2047static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG,
2049 auto CastFn = Prepare->getArgOperand(0); // as an i8*
2050 auto Fn = CastFn->stripPointerCasts(); // as its original type
2051
2052 // Attempt to peephole this pattern:
2053 // %0 = bitcast [[TYPE]] @some_function to i8*
2054 // %1 = call @llvm.coro.prepare.retcon(i8* %0)
2055 // %2 = bitcast %1 to [[TYPE]]
2056 // ==>
2057 // %2 = @some_function
2058 for (Use &U : llvm::make_early_inc_range(Prepare->uses())) {
2059 // Look for bitcasts back to the original function type.
2060 auto *Cast = dyn_cast<BitCastInst>(U.getUser());
2061 if (!Cast || Cast->getType() != Fn->getType())
2062 continue;
2063
2064 // Replace and remove the cast.
2065 Cast->replaceAllUsesWith(Fn);
2066 Cast->eraseFromParent();
2067 }
2068
2069 // Replace any remaining uses with the function as an i8*.
2070 // This can never directly be a callee, so we don't need to update CG.
2071 Prepare->replaceAllUsesWith(CastFn);
2072 Prepare->eraseFromParent();
2073
2074 // Kill dead bitcasts.
2075 while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
2076 if (!Cast->use_empty())
2077 break;
2078 CastFn = Cast->getOperand(0);
2079 Cast->eraseFromParent();
2080 }
2081}
2082
2083static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG,
2085 bool Changed = false;
2086 for (Use &P : llvm::make_early_inc_range(PrepareFn->uses())) {
2087 // Intrinsics can only be used in calls.
2088 auto *Prepare = cast<CallInst>(P.getUser());
2089 replacePrepare(Prepare, CG, C);
2090 Changed = true;
2091 }
2092
2093 return Changed;
2094}
2095
2096static void addPrepareFunction(const Module &M,
2098 StringRef Name) {
2099 auto *PrepareFn = M.getFunction(Name);
2100 if (PrepareFn && !PrepareFn->use_empty())
2101 Fns.push_back(PrepareFn);
2102}
2103
2105 : MaterializableCallback(coro::defaultMaterializable),
2106 OptimizeFrame(OptimizeFrame) {}
2107
2111 // NB: One invariant of a valid LazyCallGraph::SCC is that it must contain a
2112 // non-zero number of nodes, so we assume that here and grab the first
2113 // node's function's module.
2114 Module &M = *C.begin()->getFunction().getParent();
2115 auto &FAM =
2116 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
2117
2118 // Check for uses of llvm.coro.prepare.retcon/async.
2119 SmallVector<Function *, 2> PrepareFns;
2120 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
2121 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
2122
2123 // Find coroutines for processing.
2125 for (LazyCallGraph::Node &N : C)
2126 if (N.getFunction().isPresplitCoroutine())
2127 Coroutines.push_back(&N);
2128
2129 if (Coroutines.empty() && PrepareFns.empty())
2130 return PreservedAnalyses::all();
2131
2132 if (Coroutines.empty()) {
2133 for (auto *PrepareFn : PrepareFns) {
2134 replaceAllPrepares(PrepareFn, CG, C);
2135 }
2136 }
2137
2138 // Split all the coroutines.
2139 for (LazyCallGraph::Node *N : Coroutines) {
2140 Function &F = N->getFunction();
2141 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()
2142 << "\n");
2143 F.setSplittedCoroutine();
2144
2147 const coro::Shape Shape =
2150 updateCallGraphAfterCoroutineSplit(*N, Shape, Clones, C, CG, AM, UR, FAM);
2151
2152 ORE.emit([&]() {
2153 return OptimizationRemark(DEBUG_TYPE, "CoroSplit", &F)
2154 << "Split '" << ore::NV("function", F.getName())
2155 << "' (frame_size=" << ore::NV("frame_size", Shape.FrameSize)
2156 << ", align=" << ore::NV("align", Shape.FrameAlign.value()) << ")";
2157 });
2158
2159 if (!Shape.CoroSuspends.empty()) {
2160 // Run the CGSCC pipeline on the original and newly split functions.
2161 UR.CWorklist.insert(&C);
2162 for (Function *Clone : Clones)
2163 UR.CWorklist.insert(CG.lookupSCC(CG.get(*Clone)));
2164 }
2165 }
2166
2167 if (!PrepareFns.empty()) {
2168 for (auto *PrepareFn : PrepareFns) {
2169 replaceAllPrepares(PrepareFn, CG, C);
2170 }
2171 }
2172
2173 return PreservedAnalyses::none();
2174}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
AMDGPU Lower Kernel Arguments
SmallPtrSet< MachineInstr *, 2 > Uses
assume Assume Builder
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to manipulate a call graph, regardless if it is a "old style" Call...
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Remove calls to llvm coro end in the original static function void removeCoroEnds(const coro::Shape &Shape)
Definition: CoroSplit.cpp:1994
static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:867
static void splitSwitchCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1607
static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)
Definition: CoroSplit.cpp:1481
static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)
Definition: CoroSplit.cpp:621
static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:860
static void addMustTailToCoroResumes(Function &F, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1380
static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)
Definition: CoroSplit.cpp:170
static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)
Definition: CoroSplit.cpp:1452
static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)
Definition: CoroSplit.cpp:508
static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)
Definition: CoroSplit.cpp:500
static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)
Definition: CoroSplit.cpp:2096
static void updateCallGraphAfterCoroutineSplit(LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Definition: CoroSplit.cpp:2000
static void simplifySuspendPoints(coro::Shape &Shape)
Definition: CoroSplit.cpp:1566
static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)
Definition: CoroSplit.cpp:845
static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2083
static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace a non-unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:227
static void replaceFrameSizeAndAlignment(coro::Shape &Shape)
Definition: CoroSplit.cpp:1115
static bool replaceCoroEndAsync(AnyCoroEndInst *End)
Replace an llvm.coro.end.async.
Definition: CoroSplit.cpp:184
Replace a call to llvm coro prepare static retcon void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2047
static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace an unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:306
static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)
Definition: CoroSplit.cpp:1506
static bool hasCallsInBlockBetween(Instruction *From, Instruction *To)
Definition: CoroSplit.cpp:1440
static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)
Definition: CoroSplit.cpp:292
static void splitAsyncCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1692
static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)
Definition: CoroSplit.cpp:1100
static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Definition: CoroSplit.cpp:344
static void setCoroInfo(Function &F, coro::Shape &Shape, ArrayRef< Function * > Fns)
Definition: CoroSplit.cpp:1152
static void handleNoSuspendCoroutine(coro::Shape &Shape)
Definition: CoroSplit.cpp:1405
static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn, Function *DestroyFn, Function *CleanupFn)
Definition: CoroSplit.cpp:1176
static void createResumeEntryBlock(Function &F, coro::Shape &Shape)
Definition: CoroSplit.cpp:359
static coro::Shape splitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI, bool OptimizeFrame, std::function< bool(Instruction &)> MaterializableCallback)
Definition: CoroSplit.cpp:1934
static void postSplitCleanup(Function &F)
Definition: CoroSplit.cpp:1202
static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst)
Definition: CoroSplit.cpp:1234
static void splitRetconCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1789
static void scanPHIsAndUpdateValueMap(Instruction *Prev, BasicBlock *NewBlock, DenseMap< Value *, Value * > &ResolvedValues)
Definition: CoroSplit.cpp:1217
Coerce the arguments in p FnArgs according to p FnTy in p static CallArgs void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)
Definition: CoroSplit.cpp:1661
static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)
Definition: CoroSplit.cpp:1646
static bool shouldBeMustTail(const CallInst &CI, const Function &F)
Definition: CoroSplit.cpp:1345
static Function * createClone(Function &F, const Twine &Suffix, coro::Shape &Shape, CoroCloner::Kind FKind)
Definition: CoroSplit.cpp:1093
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
This file contains constants used for implementing Dwarf debug support.
std::string Name
uint64_t Size
static Function * getFunction(Constant *C)
Definition: Evaluator.cpp:236
@ InlineInfo
#define DEBUG_TYPE
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Implements a lazy call graph analysis and related passes for the new pass manager.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Module.h This file contains the declarations for the Module class.
print must be executed print the must be executed context for all instructions
LLVMContext & Context
#define P(N)
FunctionAnalysisManager FAM
This file provides a priority worklist.
@ SI
@ VI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This pass exposes codegen information to IR-level passes.
static const unsigned FramePtr
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
CoroAllocInst * getCoroAlloc()
Definition: CoroInstr.h:84
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:202
iterator end() const
Definition: ArrayRef.h:152
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
iterator begin() const
Definition: ArrayRef.h:151
AttrBuilder & addAlignmentAttr(MaybeAlign Align)
This turns an alignment into the form used internally in Attribute.
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
AttrBuilder & addDereferenceableAttr(uint64_t Bytes)
This turns the number of dereferenceable bytes into the form used internally in Attribute.
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
Definition: Attributes.h:87
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:372
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:105
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:401
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:284
const Instruction * getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode, a debug intrinsic,...
Definition: BasicBlock.cpp:229
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:127
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Definition: BasicBlock.cpp:341
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1186
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1476
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1467
Value * getCalledOperand() const
Definition: InstrTypes.h:1401
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1353
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1266
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1486
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:72
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1250
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition: Constants.cpp:2040
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:833
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:840
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1706
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1315
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition: Constants.cpp:1430
This represents the llvm.coro.align instruction.
Definition: CoroInstr.h:603
This represents the llvm.coro.alloc instruction.
Definition: CoroInstr.h:70
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:420
AnyCoroIdInst * getId() const
Definition: CoroInstr.h:424
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
void setInfo(Constant *C)
Definition: CoroInstr.h:180
This represents the llvm.coro.size instruction.
Definition: CoroInstr.h:591
This represents the llvm.coro.suspend.async instruction.
Definition: CoroInstr.h:525
CoroAsyncResumeInst * getResumeFunction() const
Definition: CoroInstr.h:546
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:493
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:497
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
Subprogram description.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
This is the common base class for debug info intrinsics for variables.
A debug info location.
Definition: DebugLoc.h:33
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
iterator end()
Definition: DenseMap.h:84
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
This class represents a freeze function that returns random concrete value if an operand is either a ...
A proxy from a FunctionAnalysisManager to an SCC.
Type * getReturnType() const
Definition: DerivedTypes.h:124
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:136
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:174
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:237
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:319
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:290
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void setInitializer(Constant *InitVal)
setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...
Definition: Globals.cpp:472
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2558
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:203
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:70
const BasicBlock * getParent() const
Definition: Instruction.h:90
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:82
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
void addSplitFunction(Function &OriginalFunction, Function &NewFunction)
Add a new function split/outlined from an existing function.
void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)
Add new ref-recursive functions split/outlined from an existing function.
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
SCC * lookupSCC(Node &N) const
Lookup a function's SCC in the graph.
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:497
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
FunctionListType::iterator iterator
The Function iterators.
Definition: Module.h:90
Diagnostic information for applied optimization remarks.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...
virtual void print(raw_ostream &OS) const =0
print - Emit information about this stack frame to OS.
Return a value (possibly void), from a function.
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:379
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:389
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void reserve(size_type N)
Definition: SmallVector.h:667
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
Definition: Type.cpp:623
Analysis pass providing the TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
bool supportsTailCallFor(const CallBase *CB) const
If target supports tail call on CB.
bool supportsTailCalls() const
If the target supports tail calls.
Value handle that tracks a Value across RAUW.
Definition: ValueHandle.h:331
ValueTy * getValPtr() const
Definition: ValueHandle.h:335
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:258
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1739
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
iterator_range< user_iterator > users()
Definition: Value.h:421
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:685
iterator_range< use_iterator > uses()
Definition: Value.h:376
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:381
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:289
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
void salvageDebugInfo(SmallDenseMap< llvm::Value *, llvm::AllocaInst *, 4 > &DbgPtrAllocaCache, DbgVariableIntrinsic *DVI, bool OptimizeFrame)
Recover a dbg.declare prepared by the frontend and emit an alloca holding a pointer to the coroutine ...
Definition: CoroFrame.cpp:2799
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:130
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, ArrayRef< Value * > Arguments, IRBuilder<> &)
Definition: CoroSplit.cpp:1676
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions=false, const TargetLibraryInfo *TLI=nullptr, DomTreeUpdater *DTU=nullptr)
If a terminator instruction is predicated on a constant value, convert it into an unconditional branc...
Definition: Local.cpp:126
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
Definition: Verifier.cpp:6465
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition: Utils.cpp:1367
LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a function pass.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:732
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
Definition: Local.cpp:398
@ Async
"Asynchronous" unwind tables (instr precise)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2243
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.
auto predecessors(const MachineBasicBlock *BB)
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:2607
bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)
Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...
Definition: CFG.cpp:231
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist
Worklist of the SCCs queued for processing.
const std::function< bool(Instruction &)> MaterializableCallback
Definition: CoroSplit.h:25
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: CoroSplit.cpp:2108
CoroSplitPass(bool OptimizeFrame=false)
Definition: CoroSplit.cpp:2104
AsyncLoweringStorage AsyncLowering
Definition: CoroInternal.h:143
FunctionType * getResumeFunctionType() const
Definition: CoroInternal.h:182
IntegerType * getIndexType() const
Definition: CoroInternal.h:167
StructType * FrameTy
Definition: CoroInternal.h:98
CoroIdInst * getSwitchCoroId() const
Definition: CoroInternal.h:146
Instruction * getInsertPtAfterFramePtr() const
Definition: CoroInternal.h:240
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:76
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:78
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:453
SmallVector< CallInst *, 2 > SwiftErrorOps
Definition: CoroInternal.h:79
ConstantInt * getIndex(uint64_t Value) const
Definition: CoroInternal.h:172
bool OptimizeFrame
This would only be true if optimization are enabled.
Definition: CoroInternal.h:105
SwitchLoweringStorage SwitchLowering
Definition: CoroInternal.h:141
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:74
ArrayRef< Type * > getRetconResultTypes() const
Definition: CoroInternal.h:198
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:476
RetconLoweringStorage RetconLowering
Definition: CoroInternal.h:142
SmallVector< CoroAlignInst *, 2 > CoroAligns
Definition: CoroInternal.h:77
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:75
unsigned getSwitchIndexField() const
Definition: CoroInternal.h:162