CoroSplit.cpp File Reference

#include "llvm/Transforms/Coroutines/CoroSplit.h"
#include "CoroCloner.h"
#include "CoroInternal.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/PriorityWorklist.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Coroutines/MaterializationUtils.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/CallGraphUpdater.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <iterator>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "coro-split"

Functions
static void	lowerAwaitSuspend (IRBuilder<> &Builder, CoroAwaitSuspendInst *CB, coro::Shape &Shape)
static void	lowerAwaitSuspends (Function &F, coro::Shape &Shape)
static void	maybeFreeRetconStorage (IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)
static Value *	createSwitchDestroyPtr (const coro::Shape &Shape, IRBuilder<> &Builder, Value *FramePtr)
	Create a pointer to the switch destroy function field in the coroutine frame.
static void	replaceSwitchResumeCoroFree (const coro::Shape &Shape, Function &Resume, Function &Cleanup)
	Make resume-clone coro.free conditional on whether the frame is elided.
static bool	replaceCoroEndAsync (AnyCoroEndInst *End)
	Replace an llvm.coro.end.async.
static void	replaceFallthroughCoroEnd (AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InRamp, CallGraph *CG)
	Replace a non-unwind call to llvm.coro.end.
static Value *	createSwitchIndexPtr (const coro::Shape &Shape, IRBuilder<> &Builder, Value *FramePtr)
	Create a pointer to the switch index field in the coroutine frame.
static void	markCoroutineAsDone (IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)
static void	replaceUnwindCoroEnd (AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InRamp, CallGraph *CG)
	Replace an unwind call to llvm.coro.end.
static void	replaceCoroEnd (AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InRamp, CallGraph *CG)
static FunctionType *	getFunctionTypeFromAsyncSuspend (AnyCoroSuspendInst *Suspend)
static Function *	createCloneDeclaration (Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)
static void	replaceSwiftErrorOps (Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)
static SmallVector< DbgVariableRecord * >	collectDbgVariableRecords (Function &F)
	Returns all debug records in F.
static void	updateScopeLine (Instruction *ActiveSuspend, DISubprogram &SPToUpdate)
	Adjust the scope line of the funclet to the first line number after the suspend point.
static void	addFramePointerAttrs (AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)
static void	addAsyncContextAttrs (AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
static void	addSwiftSelfAttrs (AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
static void	updateAsyncFuncPointerContextSize (coro::Shape &Shape)
static void	replaceFrameSizeAndAlignment (coro::Shape &Shape)
static void	postSplitCleanup (Function &F)
static void	handleNoSuspendCoroutine (coro::Shape &Shape)
static bool	hasCallsInBlockBetween (iterator_range< BasicBlock::iterator > R)
static bool	hasCallsInBlocksBetween (BasicBlock *SaveBB, BasicBlock *ResDesBB)
static bool	hasCallsBetween (Instruction *Save, Instruction *ResumeOrDestroy)
static bool	simplifySuspendPoint (CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)
static void	simplifySuspendPoints (coro::Shape &Shape)
static void	replaceAsyncResumeFunction (CoroSuspendAsyncInst *Suspend, Value *Continuation)
static void	coerceArguments (IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)
	Coerce the arguments in FnArgs according to FnTy in CallArgs.
static void	removeCoroEndsFromRampFunction (const coro::Shape &Shape)
	Remove calls to llvm.coro.end in the original function.
static void	removeCoroIsInRampFromRampFunction (const coro::Shape &Shape)
static bool	hasSafeElideCaller (Function &F)
static void	doSplitCoroutine (Function &F, SmallVectorImpl< Function * > &Clones, coro::BaseABI &ABI, TargetTransformInfo &TTI, bool OptimizeFrame)
static LazyCallGraph::SCC &	updateCallGraphAfterCoroutineSplit (LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
static void	replacePrepare (CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)
	Replace a call to llvm.coro.prepare.retcon.
static bool	replaceAllPrepares (Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)
static void	addPrepareFunction (const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)
static std::unique_ptr< coro::BaseABI >	CreateNewABI (Function &F, coro::Shape &S, std::function< bool(Instruction &)> IsMatCallback, const SmallVector< CoroSplitPass::BaseABITy > GenCustomABIs)

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "coro-split"

Definition at line 83 of file CoroSplit.cpp.

Function Documentation

addAsyncContextAttrs()

void addAsyncContextAttrs ( AttributeList & Attrs, LLVMContext & Context, unsigned ParamIndex )

static

Definition at line 925 of file CoroSplit.cpp.

Referenced by llvm::coro::BaseCloner::create().

addFramePointerAttrs()

void addFramePointerAttrs ( AttributeList & Attrs, LLVMContext & Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias )

static

Definition at line 910 of file CoroSplit.cpp.

References Size.

Referenced by llvm::coro::BaseCloner::create().

addPrepareFunction()

void addPrepareFunction ( const Module & M, SmallVectorImpl< Function * > & Fns, StringRef Name )

static

Definition at line 2242 of file CoroSplit.cpp.

References llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::CoroSplitPass::run().

addSwiftSelfAttrs()

void addSwiftSelfAttrs ( AttributeList & Attrs, LLVMContext & Context, unsigned ParamIndex )

static

Definition at line 932 of file CoroSplit.cpp.

Referenced by llvm::coro::BaseCloner::create().

coerceArguments()

void coerceArguments ( IRBuilder<> & Builder, FunctionType * FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > & CallArgs )

static

Coerce the arguments in FnArgs according to FnTy in CallArgs.

Definition at line 1774 of file CoroSplit.cpp.

References assert(), getType(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ArrayRef< T >::size().

Referenced by llvm::coro::createMustTailCall().

collectDbgVariableRecords()

SmallVector< DbgVariableRecord * > collectDbgVariableRecords ( Function & F )

static

Returns all debug records in F.

Definition at line 688 of file CoroSplit.cpp.

References F, llvm::filterDbgVars(), I, instructions, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by doSplitCoroutine(), and llvm::coro::BaseCloner::salvageDebugInfo().

createCloneDeclaration()

Function * createCloneDeclaration ( Function & OrigF, coro::Shape & Shape, const Twine & Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst * ActiveSuspend )

static

Definition at line 505 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::Function::Create(), llvm::GlobalValue::getAddressSpace(), getFunctionTypeFromAsyncSuspend(), llvm::Value::getName(), llvm::GlobalValue::getParent(), llvm::coro::Shape::getResumeFunctionType(), and llvm::GlobalValue::InternalLinkage.

Referenced by llvm::coro::SwitchCloner::create(), llvm::coro::AnyRetconABI::splitCoroutine(), and llvm::coro::AsyncABI::splitCoroutine().

CreateNewABI()

std::unique_ptr< coro::BaseABI > CreateNewABI ( Function & F, coro::Shape & S, std::function< bool(Instruction &)> IsMatCallback, const SmallVector< CoroSplitPass::BaseABITy > GenCustomABIs )

static

Definition at line 2251 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::coro::Shape::CoroBegin, F, llvm::CoroBeginInst::getCustomABI(), llvm::CoroBeginInst::hasCustomABI(), llvm_unreachable, llvm::coro::Retcon, llvm::coro::RetconOnce, llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::coro::Switch.

Referenced by llvm::CoroSplitPass::CoroSplitPass(), llvm::CoroSplitPass::CoroSplitPass(), llvm::CoroSplitPass::CoroSplitPass(), and llvm::CoroSplitPass::CoroSplitPass().

createSwitchDestroyPtr()

Value * createSwitchDestroyPtr ( const coro::Shape & Shape, IRBuilder<> & Builder, Value * FramePtr )

static

Create a pointer to the switch destroy function field in the coroutine frame.

Definition at line 172 of file CoroSplit.cpp.

References llvm::coro::Shape::SwitchLoweringStorage::DestroyOffset, FramePtr, llvm::Type::getInt64Ty(), llvm::Offset, and llvm::coro::Shape::SwitchLowering.

Referenced by replaceSwitchResumeCoroFree().

createSwitchIndexPtr()

Value * createSwitchIndexPtr ( const coro::Shape & Shape, IRBuilder<> & Builder, Value * FramePtr )

static

Create a pointer to the switch index field in the coroutine frame.

Definition at line 358 of file CoroSplit.cpp.

References FramePtr, llvm::Type::getInt64Ty(), llvm::coro::Shape::SwitchLoweringStorage::IndexOffset, llvm::Offset, and llvm::coro::Shape::SwitchLowering.

Referenced by markCoroutineAsDone().

doSplitCoroutine()

void doSplitCoroutine ( Function & F, SmallVectorImpl< Function * > & Clones, coro::BaseABI & ABI, TargetTransformInfo & TTI, bool OptimizeFrame )

static

Definition at line 2090 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), collectDbgVariableRecords(), llvm::coro::Shape::CoroBegin, llvm::coro::Shape::CoroSuspends, F, handleNoSuspendCoroutine(), llvm::coro::Shape::SwitchLoweringStorage::HasCoroElideNoAllocVariant, hasSafeElideCaller(), lowerAwaitSuspends(), llvm::coro::normalizeCoroutine(), removeCoroEndsFromRampFunction(), removeCoroIsInRampFromRampFunction(), replaceFrameSizeAndAlignment(), replaceSwiftErrorOps(), llvm::coro::salvageDebugInfo(), llvm::coro::Shape::Shape(), simplifySuspendPoints(), llvm::coro::Switch, and llvm::coro::Shape::SwitchLowering.

Referenced by llvm::CoroSplitPass::run().

getFunctionTypeFromAsyncSuspend()

FunctionType * getFunctionTypeFromAsyncSuspend ( AnyCoroSuspendInst * Suspend )

static

Definition at line 497 of file CoroSplit.cpp.

References llvm::cast(), llvm::FunctionType::get(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), and llvm::Type::getVoidTy().

Referenced by createCloneDeclaration().

handleNoSuspendCoroutine()

void handleNoSuspendCoroutine ( coro::Shape & Shape )

static

Definition at line 1228 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::coro::Shape::CoroBegin, llvm::coro::elideCoroFree(), llvm::Instruction::eraseFromParent(), llvm::coro::Shape::FrameAlign, llvm::coro::Shape::FrameSize, llvm::ArrayType::get(), llvm::PoisonValue::get(), llvm::AnyCoroIdInst::getCoroAlloc(), llvm::Type::getInt8Ty(), llvm::coro::Shape::getSwitchCoroId(), llvm::coro::Retcon, llvm::coro::RetconOnce, and llvm::coro::Switch.

Referenced by doSplitCoroutine().

hasCallsBetween()

bool hasCallsBetween ( Instruction * Save, Instruction * ResumeOrDestroy )

static

Definition at line 1306 of file CoroSplit.cpp.

References llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), hasCallsInBlockBetween(), and hasCallsInBlocksBetween().

Referenced by simplifySuspendPoint().

hasCallsInBlockBetween()

bool hasCallsInBlockBetween ( iterator_range< BasicBlock::iterator > R )

static

Definition at line 1265 of file CoroSplit.cpp.

References I, and llvm::isa().

Referenced by hasCallsBetween(), and hasCallsInBlocksBetween().

hasCallsInBlocksBetween()

bool hasCallsInBlocksBetween ( BasicBlock * SaveBB, BasicBlock * ResDesBB )

static

Definition at line 1277 of file CoroSplit.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::empty(), hasCallsInBlockBetween(), llvm::SmallVectorImpl< T >::pop_back_val(), llvm::predecessors(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by hasCallsBetween().

hasSafeElideCaller()

bool hasSafeElideCaller ( Function & F )

static

Definition at line 2072 of file CoroSplit.cpp.

References llvm::dyn_cast(), and F.

Referenced by doSplitCoroutine().

lowerAwaitSuspend()

void lowerAwaitSuspend ( IRBuilder<> & Builder, CoroAwaitSuspendInst * CB, coro::Shape & Shape )

static

Definition at line 89 of file CoroSplit.cpp.

References Call, llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), FramePtr, llvm::FunctionType::get(), llvm::CallBase::getAttributes(), llvm::CoroAwaitSuspendInst::getAwaiter(), llvm::CallBase::getCalledFunction(), llvm::Value::getContext(), llvm::CoroAwaitSuspendInst::getFrame(), llvm::Function::getIntrinsicID(), llvm::PointerType::getUnqual(), llvm::Type::getVoidTy(), llvm::CoroAwaitSuspendInst::getWrapperFunction(), llvm_unreachable, llvm::coro::LowererBase::makeSubFnCall(), llvm::Value::replaceAllUsesWith(), llvm::CoroSubFnInst::ResumeIndex, llvm::coro::Shape::SymmetricTransfers, and Wrapper.

Referenced by lowerAwaitSuspends().

lowerAwaitSuspends()

void lowerAwaitSuspends ( Function & F, coro::Shape & Shape )

static

Definition at line 154 of file CoroSplit.cpp.

References llvm::coro::Shape::CoroAwaitSuspends, F, and lowerAwaitSuspend().

Referenced by doSplitCoroutine().

markCoroutineAsDone()

void markCoroutineAsDone ( IRBuilder<> & Builder, const coro::Shape & Shape, Value * FramePtr )

static

Definition at line 373 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), llvm::cast(), llvm::coro::Shape::CoroSuspends, createSwitchIndexPtr(), FramePtr, llvm::ConstantPointerNull::get(), llvm::coro::Shape::getIndex(), llvm::coro::Shape::getSwitchResumePointerType(), llvm::coro::Shape::SwitchLoweringStorage::HasFinalSuspend, llvm::coro::Shape::SwitchLoweringStorage::HasUnwindCoroEnd, llvm::coro::Switch, and llvm::coro::Shape::SwitchLowering.

Referenced by replaceUnwindCoroEnd().

maybeFreeRetconStorage()

void maybeFreeRetconStorage ( IRBuilder<> & Builder, const coro::Shape & Shape, Value * FramePtr, CallGraph * CG )

static

Definition at line 160 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), llvm::coro::Shape::emitDealloc(), FramePtr, llvm::coro::Shape::RetconLoweringStorage::IsFrameInlineInStorage, llvm::coro::Retcon, llvm::coro::Shape::RetconLowering, and llvm::coro::RetconOnce.

Referenced by replaceFallthroughCoroEnd(), and replaceUnwindCoroEnd().

postSplitCleanup()

void postSplitCleanup ( Function & F )

static

Definition at line 1214 of file CoroSplit.cpp.

References llvm::errs(), F, llvm::removeUnreachableBlocks(), llvm::report_fatal_error(), and llvm::verifyFunction().

Referenced by updateCallGraphAfterCoroutineSplit().

removeCoroEndsFromRampFunction()

void removeCoroEndsFromRampFunction ( const coro::Shape & Shape )

static

Remove calls to llvm.coro.end in the original function.

Definition at line 2053 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::coro::Shape::CoroEnds, llvm::Instruction::eraseFromParent(), llvm::coro::Shape::FramePtr, replaceCoroEnd(), and llvm::coro::Switch.

Referenced by doSplitCoroutine().

removeCoroIsInRampFromRampFunction()

void removeCoroIsInRampFromRampFunction ( const coro::Shape & Shape )

static

Definition at line 2064 of file CoroSplit.cpp.

References llvm::coro::Shape::CoroIsInRampInsts, llvm::ConstantInt::getTrue(), and II.

Referenced by doSplitCoroutine().

replaceAllPrepares()

bool replaceAllPrepares ( Function * PrepareFn, LazyCallGraph & CG, LazyCallGraph::SCC & C )

static

Definition at line 2229 of file CoroSplit.cpp.

References C(), llvm::cast(), Changed, llvm::make_early_inc_range(), P, replacePrepare(), and llvm::Value::uses().

Referenced by llvm::CoroSplitPass::run().

replaceAsyncResumeFunction()

void replaceAsyncResumeFunction ( CoroSuspendAsyncInst * Suspend, Value * Continuation )

static

Definition at line 1759 of file CoroSplit.cpp.

References llvm::coro::Continuation, llvm::PoisonValue::get(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::CoroSuspendAsyncInst::getResumeFunction(), llvm::PointerType::getUnqual(), llvm::CoroSuspendAsyncInst::ResumeFunctionArg, and llvm::User::setOperand().

Referenced by llvm::coro::AsyncABI::splitCoroutine().

replaceCoroEnd()

void replaceCoroEnd ( AnyCoroEndInst * End, const coro::Shape & Shape, Value * FramePtr, bool InRamp, CallGraph * CG )

static

Definition at line 439 of file CoroSplit.cpp.

References llvm::Instruction::eraseFromParent(), FramePtr, llvm::AnyCoroEndInst::isUnwind(), replaceFallthroughCoroEnd(), and replaceUnwindCoroEnd().

Referenced by removeCoroEndsFromRampFunction(), and llvm::coro::BaseCloner::replaceCoroEnds().

replaceCoroEndAsync()

bool replaceCoroEndAsync ( AnyCoroEndInst * End )

static

Replace an llvm.coro.end.async.

Will inline the must tail call function call if there is one.

Returns

true if cleanup of the coro.end block is needed, false otherwise.

Definition at line 223 of file CoroSplit.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), and llvm::InlineFunction().

Referenced by replaceFallthroughCoroEnd().

replaceFallthroughCoroEnd()

void replaceFallthroughCoroEnd ( AnyCoroEndInst * End, const coro::Shape & Shape, Value * FramePtr, bool InRamp, CallGraph * CG )

static

Replace a non-unwind call to llvm.coro.end.

Definition at line 266 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), llvm::coro::Async, llvm::cast(), llvm::dyn_cast(), FramePtr, llvm::ConstantPointerNull::get(), llvm::ConstantTokenNone::get(), llvm::PoisonValue::get(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::coro::Shape::getResumeFunctionType(), llvm::FunctionType::getReturnType(), maybeFreeRetconStorage(), replaceCoroEndAsync(), llvm::coro::Retcon, llvm::coro::RetconOnce, and llvm::coro::Switch.

Referenced by replaceCoroEnd().

replaceFrameSizeAndAlignment()

void replaceFrameSizeAndAlignment ( coro::Shape & Shape )

static

Definition at line 1190 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::coro::Shape::CoroAligns, llvm::coro::Shape::CoroSizes, llvm::sampleprof::CS, llvm::Instruction::eraseFromParent(), llvm::coro::Shape::FrameAlign, llvm::coro::Shape::FrameSize, llvm::TypeSize::getFixed(), llvm::Value::getType(), llvm::Value::replaceAllUsesWith(), updateAsyncFuncPointerContextSize(), and llvm::Align::value().

Referenced by doSplitCoroutine().

replacePrepare()

void replacePrepare ( CallInst * Prepare, LazyCallGraph & CG, LazyCallGraph::SCC & C )

static

Replace a call to llvm.coro.prepare.retcon.

Definition at line 2193 of file CoroSplit.cpp.

References C(), llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), llvm::CallBase::getArgOperand(), llvm::make_early_inc_range(), llvm::Value::replaceAllUsesWith(), llvm::Value::stripPointerCasts(), and llvm::Value::uses().

Referenced by replaceAllPrepares().

replaceSwiftErrorOps()

void replaceSwiftErrorOps ( Function & F, coro::Shape & Shape, ValueToValueMapTy * VMap )

static

Definition at line 630 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), llvm::coro::Async, llvm::cast(), llvm::coro::Shape::CoroSuspends, F, llvm::Value::getType(), llvm::Value::replaceAllUsesWith(), and llvm::coro::Shape::SwiftErrorOps.

Referenced by llvm::coro::BaseCloner::create(), doSplitCoroutine(), and llvm::coro::BaseCloner::replaceSwiftErrorOps().

replaceSwitchResumeCoroFree()

void replaceSwitchResumeCoroFree ( const coro::Shape & Shape, Function & Resume, Function & Cleanup )

static

Make resume-clone coro.free conditional on whether the frame is elided.

The destroy slot holds the cleanup clone for an elided frame and the destroy clone for a heap frame. Load it before user code can reentrantly destroy the enclosing caller frame, then use the cached comparison to suppress only the deallocation. The resume clone has already performed the shared coroutine cleanup, so calling either clone here would run that cleanup twice.

Definition at line 186 of file CoroSplit.cpp.

References llvm::applyProfMetadataIfEnabled(), llvm::cast(), Cleanup, llvm::IRBuilderBase::CreateICmpEQ(), llvm::IRBuilderBase::CreateLoad(), llvm::IRBuilderBase::CreatePointerCast(), createSwitchDestroyPtr(), DEBUG_TYPE, llvm::dyn_cast(), FramePtr, llvm::ConstantPointerNull::get(), llvm::Function::getArg(), llvm::Function::getEntryBlock(), llvm::Instruction::getFunction(), llvm::coro::Shape::getSwitchResumePointerType(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::Null, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::Value::replaceAllUsesWith(), and llvm::setExplicitlyUnknownBranchWeightsIfProfiled().

replaceUnwindCoroEnd()

void replaceUnwindCoroEnd ( AnyCoroEndInst * End, const coro::Shape & Shape, Value * FramePtr, bool InRamp, CallGraph * CG )

static

Replace an unwind call to llvm.coro.end.

Definition at line 402 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::coro::Async, llvm::cast(), FramePtr, llvm::CallBase::getOperandBundle(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), markCoroutineAsDone(), maybeFreeRetconStorage(), llvm::LLVMContext::OB_funclet, llvm::coro::Retcon, llvm::coro::RetconOnce, and llvm::coro::Switch.

Referenced by replaceCoroEnd().

simplifySuspendPoint()

bool simplifySuspendPoint ( CoroSuspendInst * Suspend, CoroBeginInst * CoroBegin )

static

Definition at line 1333 of file CoroSplit.cpp.

References llvm::UncondBrInst::Create(), llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), llvm::CallBase::getCalledOperand(), llvm::CoroSuspendInst::getCoroSave(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getPrevNode(), hasCallsBetween(), I, llvm::Value::replaceAllUsesWith(), and llvm::Value::stripPointerCasts().

Referenced by simplifySuspendPoints().

simplifySuspendPoints()

void simplifySuspendPoints ( coro::Shape & Shape )

static

Definition at line 1393 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), llvm::cast(), llvm::coro::Shape::CoroBegin, llvm::coro::Shape::CoroSuspends, llvm::coro::Shape::SwitchLoweringStorage::HasFinalSuspend, I, N, simplifySuspendPoint(), std::swap(), llvm::coro::Switch, and llvm::coro::Shape::SwitchLowering.

Referenced by doSplitCoroutine().

updateAsyncFuncPointerContextSize()

void updateAsyncFuncPointerContextSize ( coro::Shape & Shape )

static

Definition at line 1175 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, assert(), llvm::coro::Async, llvm::coro::Shape::AsyncLoweringStorage::AsyncFuncPointer, llvm::coro::Shape::AsyncLowering, llvm::cast(), llvm::coro::Shape::AsyncLoweringStorage::ContextSize, llvm::ConstantStruct::get(), llvm::GlobalVariable::getInitializer(), and llvm::GlobalVariable::setInitializer().

Referenced by replaceFrameSizeAndAlignment().

updateCallGraphAfterCoroutineSplit()

LazyCallGraph::SCC & updateCallGraphAfterCoroutineSplit ( LazyCallGraph::Node & N, const coro::Shape & Shape, const SmallVectorImpl< Function * > & Clones, LazyCallGraph::SCC & C, LazyCallGraph & CG, CGSCCAnalysisManager & AM, CGSCCUpdateResult & UR, FunctionAnalysisManager & FAM )

static

Definition at line 2142 of file CoroSplit.cpp.

References llvm::coro::Shape::ABI, llvm::LazyCallGraph::addSplitFunction(), llvm::LazyCallGraph::addSplitRefRecursiveFunctions(), assert(), llvm::coro::Async, C(), llvm::drop_begin(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), FAM, llvm::coro::Shape::SwitchLoweringStorage::HasCoroElideNoAllocVariant, N, postSplitCleanup(), llvm::coro::Retcon, llvm::coro::RetconOnce, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::coro::Switch, llvm::coro::Shape::SwitchLowering, llvm::updateCGAndAnalysisManagerForCGSCCPass(), and llvm::updateCGAndAnalysisManagerForFunctionPass().

Referenced by llvm::CoroSplitPass::run().

updateScopeLine()

void updateScopeLine ( Instruction * ActiveSuspend, DISubprogram & SPToUpdate )

static

Adjust the scope line of the funclet to the first line number after the suspend point.

This avoids a jump in the line table from the function declaration (where prologue instructions are attributed to) to the suspend point. Only adjust the scope line when the files are the same. If no candidate line number is found, fallback to the line of ActiveSuspend.

Definition at line 863 of file CoroSplit.cpp.

References DL, llvm::dyn_cast_or_null(), llvm::BasicBlock::end(), llvm::Instruction::getDebugLoc(), llvm::DIScope::getFile(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getNextNode(), llvm::skipDebugIntrinsics(), and llvm::Successor.

Referenced by llvm::coro::BaseCloner::create().