LLVM  16.0.0git
CoroEarly.cpp
Go to the documentation of this file.
1 //===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 #include "CoroInternal.h"
11 #include "llvm/IR/DIBuilder.h"
12 #include "llvm/IR/Function.h"
13 #include "llvm/IR/IRBuilder.h"
14 #include "llvm/IR/InstIterator.h"
15 #include "llvm/IR/Module.h"
16 
17 using namespace llvm;
18 
19 #define DEBUG_TYPE "coro-early"
20 
21 namespace {
22 // Created on demand if the coro-early pass has work to do.
23 class Lowerer : public coro::LowererBase {
25  PointerType *const AnyResumeFnPtrTy;
26  Constant *NoopCoro = nullptr;
27 
28  void lowerResumeOrDestroy(CallBase &CB, CoroSubFnInst::ResumeKind);
29  void lowerCoroPromise(CoroPromiseInst *Intrin);
30  void lowerCoroDone(IntrinsicInst *II);
31  void lowerCoroNoop(IntrinsicInst *II);
32 
33 public:
34  Lowerer(Module &M)
35  : LowererBase(M), Builder(Context),
36  AnyResumeFnPtrTy(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
37  /*isVarArg=*/false)
38  ->getPointerTo()) {}
39  void lowerEarlyIntrinsics(Function &F);
40 };
41 }
42 
43 // Replace a direct call to coro.resume or coro.destroy with an indirect call to
44 // an address returned by coro.subfn.addr intrinsic. This is done so that
45 // CGPassManager recognizes devirtualization when CoroElide pass replaces a call
46 // to coro.subfn.addr with an appropriate function address.
47 void Lowerer::lowerResumeOrDestroy(CallBase &CB,
49  Value *ResumeAddr = makeSubFnCall(CB.getArgOperand(0), Index, &CB);
50  CB.setCalledOperand(ResumeAddr);
52 }
53 
54 // Coroutine promise field is always at the fixed offset from the beginning of
55 // the coroutine frame. i8* coro.promise(i8*, i1 from) intrinsic adds an offset
56 // to a passed pointer to move from coroutine frame to coroutine promise and
57 // vice versa. Since we don't know exactly which coroutine frame it is, we build
58 // a coroutine frame mock up starting with two function pointers, followed by a
59 // properly aligned coroutine promise field.
60 // TODO: Handle the case when coroutine promise alloca has align override.
61 void Lowerer::lowerCoroPromise(CoroPromiseInst *Intrin) {
62  Value *Operand = Intrin->getArgOperand(0);
63  Align Alignment = Intrin->getAlignment();
64  Type *Int8Ty = Builder.getInt8Ty();
65 
66  auto *SampleStruct =
67  StructType::get(Context, {AnyResumeFnPtrTy, AnyResumeFnPtrTy, Int8Ty});
68  const DataLayout &DL = TheModule.getDataLayout();
69  int64_t Offset = alignTo(
70  DL.getStructLayout(SampleStruct)->getElementOffset(2), Alignment);
71  if (Intrin->isFromPromise())
72  Offset = -Offset;
73 
74  Builder.SetInsertPoint(Intrin);
75  Value *Replacement =
76  Builder.CreateConstInBoundsGEP1_32(Int8Ty, Operand, Offset);
77 
78  Intrin->replaceAllUsesWith(Replacement);
79  Intrin->eraseFromParent();
80 }
81 
82 // When a coroutine reaches final suspend point, it zeros out ResumeFnAddr in
83 // the coroutine frame (it is UB to resume from a final suspend point).
84 // The llvm.coro.done intrinsic is used to check whether a coroutine is
85 // suspended at the final suspend point or not.
86 void Lowerer::lowerCoroDone(IntrinsicInst *II) {
87  Value *Operand = II->getArgOperand(0);
88 
89  // ResumeFnAddr is the first pointer sized element of the coroutine frame.
90  static_assert(coro::Shape::SwitchFieldIndex::Resume == 0,
91  "resume function not at offset zero");
92  auto *FrameTy = Int8Ptr;
93  PointerType *FramePtrTy = FrameTy->getPointerTo();
94 
95  Builder.SetInsertPoint(II);
96  auto *BCI = Builder.CreateBitCast(Operand, FramePtrTy);
97  auto *Load = Builder.CreateLoad(FrameTy, BCI);
98  auto *Cond = Builder.CreateICmpEQ(Load, NullPtr);
99 
101  II->eraseFromParent();
102 }
103 
105  Module &M = *NoopFn->getParent();
106  if (M.debug_compile_units().empty())
107  return;
108 
109  DICompileUnit *CU = *M.debug_compile_units_begin();
110  DIBuilder DB(M, /*AllowUnresolved*/ false, CU);
111  std::array<Metadata *, 2> Params{nullptr, nullptr};
112  auto *SubroutineType =
114  StringRef Name = NoopFn->getName();
115  auto *SP = DB.createFunction(
116  CU, /*Name=*/Name, /*LinkageName=*/Name, /*File=*/ CU->getFile(),
117  /*LineNo=*/0, SubroutineType, /*ScopeLine=*/0, DINode::FlagArtificial,
118  DISubprogram::SPFlagDefinition);
119  NoopFn->setSubprogram(SP);
120  DB.finalize();
121 }
122 
123 void Lowerer::lowerCoroNoop(IntrinsicInst *II) {
124  if (!NoopCoro) {
125  LLVMContext &C = Builder.getContext();
126  Module &M = *II->getModule();
127 
128  // Create a noop.frame struct type.
129  StructType *FrameTy = StructType::create(C, "NoopCoro.Frame");
130  auto *FramePtrTy = FrameTy->getPointerTo();
131  auto *FnTy = FunctionType::get(Type::getVoidTy(C), FramePtrTy,
132  /*isVarArg=*/false);
133  auto *FnPtrTy = FnTy->getPointerTo();
134  FrameTy->setBody({FnPtrTy, FnPtrTy});
135 
136  // Create a Noop function that does nothing.
137  Function *NoopFn =
138  Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
139  "__NoopCoro_ResumeDestroy", &M);
142  auto *Entry = BasicBlock::Create(C, "entry", NoopFn);
143  ReturnInst::Create(C, Entry);
144 
145  // Create a constant struct for the frame.
146  Constant* Values[] = {NoopFn, NoopFn};
147  Constant* NoopCoroConst = ConstantStruct::get(FrameTy, Values);
148  NoopCoro = new GlobalVariable(M, NoopCoroConst->getType(), /*isConstant=*/true,
149  GlobalVariable::PrivateLinkage, NoopCoroConst,
150  "NoopCoro.Frame.Const");
151  }
152 
153  Builder.SetInsertPoint(II);
154  auto *NoopCoroVoidPtr = Builder.CreateBitCast(NoopCoro, Int8Ptr);
155  II->replaceAllUsesWith(NoopCoroVoidPtr);
156  II->eraseFromParent();
157 }
158 
159 // Prior to CoroSplit, calls to coro.begin needs to be marked as NoDuplicate,
160 // as CoroSplit assumes there is exactly one coro.begin. After CoroSplit,
161 // NoDuplicate attribute will be removed from coro.begin otherwise, it will
162 // interfere with inlining.
163 static void setCannotDuplicate(CoroIdInst *CoroId) {
164  for (User *U : CoroId->users())
165  if (auto *CB = dyn_cast<CoroBeginInst>(U))
166  CB->setCannotDuplicate();
167 }
168 
169 void Lowerer::lowerEarlyIntrinsics(Function &F) {
170  CoroIdInst *CoroId = nullptr;
172  bool HasCoroSuspend = false;
174  auto *CB = dyn_cast<CallBase>(&I);
175  if (!CB)
176  continue;
177 
178  switch (CB->getIntrinsicID()) {
179  default:
180  continue;
181  case Intrinsic::coro_free:
182  CoroFrees.push_back(cast<CoroFreeInst>(&I));
183  break;
184  case Intrinsic::coro_suspend:
185  // Make sure that final suspend point is not duplicated as CoroSplit
186  // pass expects that there is at most one final suspend point.
187  if (cast<CoroSuspendInst>(&I)->isFinal())
188  CB->setCannotDuplicate();
189  HasCoroSuspend = true;
190  break;
191  case Intrinsic::coro_end_async:
192  case Intrinsic::coro_end:
193  // Make sure that fallthrough coro.end is not duplicated as CoroSplit
194  // pass expects that there is at most one fallthrough coro.end.
195  if (cast<AnyCoroEndInst>(&I)->isFallthrough())
196  CB->setCannotDuplicate();
197  break;
198  case Intrinsic::coro_noop:
199  lowerCoroNoop(cast<IntrinsicInst>(&I));
200  break;
201  case Intrinsic::coro_id:
202  if (auto *CII = cast<CoroIdInst>(&I)) {
203  if (CII->getInfo().isPreSplit()) {
204  assert(F.isPresplitCoroutine() &&
205  "The frontend uses Swtich-Resumed ABI should emit "
206  "\"coroutine.presplit\" attribute for the coroutine.");
207  setCannotDuplicate(CII);
208  CII->setCoroutineSelf();
209  CoroId = cast<CoroIdInst>(&I);
210  }
211  }
212  break;
213  case Intrinsic::coro_id_retcon:
214  case Intrinsic::coro_id_retcon_once:
215  case Intrinsic::coro_id_async:
216  F.setPresplitCoroutine();
217  break;
218  case Intrinsic::coro_resume:
219  lowerResumeOrDestroy(*CB, CoroSubFnInst::ResumeIndex);
220  break;
221  case Intrinsic::coro_destroy:
222  lowerResumeOrDestroy(*CB, CoroSubFnInst::DestroyIndex);
223  break;
224  case Intrinsic::coro_promise:
225  lowerCoroPromise(cast<CoroPromiseInst>(&I));
226  break;
227  case Intrinsic::coro_done:
228  lowerCoroDone(cast<IntrinsicInst>(&I));
229  break;
230  }
231  }
232 
233  // Make sure that all CoroFree reference the coro.id intrinsic.
234  // Token type is not exposed through coroutine C/C++ builtins to plain C, so
235  // we allow specifying none and fixing it up here.
236  if (CoroId)
237  for (CoroFreeInst *CF : CoroFrees)
238  CF->setArgOperand(0, CoroId);
239 
240  // Coroutine suspention could potentially lead to any argument modified
241  // outside of the function, hence arguments should not have noalias
242  // attributes.
243  if (HasCoroSuspend)
244  for (Argument &A : F.args())
245  if (A.hasNoAliasAttr())
246  A.removeAttr(Attribute::NoAlias);
247 }
248 
249 static bool declaresCoroEarlyIntrinsics(const Module &M) {
251  M, {"llvm.coro.id", "llvm.coro.id.retcon", "llvm.coro.id.retcon.once",
252  "llvm.coro.id.async", "llvm.coro.destroy", "llvm.coro.done",
253  "llvm.coro.end", "llvm.coro.end.async", "llvm.coro.noop",
254  "llvm.coro.free", "llvm.coro.promise", "llvm.coro.resume",
255  "llvm.coro.suspend"});
256 }
257 
260  return PreservedAnalyses::all();
261 
262  Lowerer L(M);
263  for (auto &F : M)
264  L.lowerEarlyIntrinsics(F);
265 
267  PA.preserveSet<CFGAnalyses>();
268  return PA;
269 }
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:156
llvm::CoroSubFnInst::ResumeIndex
@ ResumeIndex
Definition: CoroInstr.h:41
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:65
llvm::DIBuilder::getOrCreateTypeArray
DITypeRefArray getOrCreateTypeArray(ArrayRef< Metadata * > Elements)
Get a DITypeRefArray, create one if required.
Definition: DIBuilder.cpp:685
llvm::DIBuilder::finalize
void finalize()
Construct any deferred debug info descriptors.
Definition: DIBuilder.cpp:79
llvm::StructType::get
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:406
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::DIBuilder
Definition: DIBuilder.h:41
llvm::CoroSubFnInst::DestroyIndex
@ DestroyIndex
Definition: CoroInstr.h:42
InstIterator.h
llvm::Function
Definition: Function.h:60
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1306
llvm::StructType::setBody
void setBody(ArrayRef< Type * > Elements, bool isPacked=false)
Specify a body for an opaque identified type.
Definition: Type.cpp:445
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1181
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:361
llvm::CoroSubFnInst::ResumeKind
ResumeKind
Definition: CoroInstr.h:39
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Module.h
llvm::coro::declaresIntrinsics
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:117
llvm::CoroFreeInst
This represents the llvm.coro.free instruction.
Definition: CoroInstr.h:404
setCannotDuplicate
static void setCannotDuplicate(CoroIdInst *CoroId)
Definition: CoroEarly.cpp:163
llvm::DIBuilder::createSubroutineType
DISubroutineType * createSubroutineType(DITypeRefArray ParameterTypes, DINode::DIFlags Flags=DINode::FlagZero, unsigned CC=0)
Create subroutine type.
Definition: DIBuilder.cpp:544
llvm::StructType::create
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Definition: Type.cpp:513
F
#define F(x, y, z)
Definition: MD5.cpp:55
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
llvm::User
Definition: User.h:44
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::ARM_PROC::A
@ A
Definition: ARMBaseInfo.h:34
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::CoroPromiseInst::getAlignment
Align getAlignment() const
The required alignment of the promise.
Definition: CoroInstr.h:464
llvm::Instruction
Definition: Instruction.h:42
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
CoroInternal.h
llvm::Function::setSubprogram
void setSubprogram(DISubprogram *SP)
Set the attached subprogram.
Definition: Metadata.cpp:1579
llvm::CoroPromiseInst
This represents the llvm.coro.promise instruction.
Definition: CoroInstr.h:452
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:77
llvm::AMDGPU::Hwreg::Offset
Offset
Definition: SIDefines.h:416
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:650
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:596
DIBuilder.h
llvm::DICompileUnit
Compile unit.
Definition: DebugInfoMetadata.h:1322
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:632
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:137
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
buildDebugInfoForNoopResumeDestroyFunc
static void buildDebugInfoForNoopResumeDestroyFunc(Function *NoopFn)
Definition: CoroEarly.cpp:104
llvm::CallBase::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition: Instructions.cpp:312
llvm::CoroPromiseInst::isFromPromise
bool isFromPromise() const
Are we translating from the frame to the promise (false) or from the promise to the frame (true)?
Definition: CoroInstr.h:458
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:213
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:137
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::Function::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: Function.h:242
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition: PassManager.h:113
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:97
llvm::CallingConv::Fast
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::CoroEarlyPass::run
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition: CoroEarly.cpp:258
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
llvm::coro::Shape::SwitchFieldIndex::Resume
@ Resume
Definition: CoroInternal.h:84
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
Function.h
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:774
llvm::ReturnInst::Create
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3079
llvm::IntrinsicInst
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:46
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition: InstrTypes.h:1432
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:222
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::DIBuilder::createFunction
DISubprogram * createFunction(DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine, DINode::DIFlags Flags=DINode::FlagZero, DISubprogram::DISPFlags SPFlags=DISubprogram::SPFlagZero, DITemplateParameterArray TParams=nullptr, DISubprogram *Decl=nullptr, DITypeArray ThrownTypes=nullptr, DINodeArray Annotations=nullptr, StringRef TargetFuncName="")
Create a new descriptor for the specified subprogram.
Definition: DIBuilder.cpp:844
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
llvm::PreservedAnalyses::preserveSet
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:188
declaresCoroEarlyIntrinsics
static bool declaresCoroEarlyIntrinsics(const Module &M)
Definition: CoroEarly.cpp:249
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1341
llvm::coro::LowererBase
Definition: CoroInternal.h:35
CoroEarly.h
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1174
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
llvm::CallBase::setCannotDuplicate
void setCannotDuplicate()
Definition: InstrTypes.h:1904
CU
Definition: AArch64AsmBackend.cpp:504
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1459