LLVM  15.0.0git
CoroEarly.cpp
Go to the documentation of this file.
1 //===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 #include "CoroInternal.h"
11 #include "llvm/IR/Function.h"
12 #include "llvm/IR/IRBuilder.h"
13 #include "llvm/IR/InstIterator.h"
14 #include "llvm/IR/Module.h"
15 
16 using namespace llvm;
17 
18 #define DEBUG_TYPE "coro-early"
19 
20 namespace {
21 // Created on demand if the coro-early pass has work to do.
22 class Lowerer : public coro::LowererBase {
24  PointerType *const AnyResumeFnPtrTy;
25  Constant *NoopCoro = nullptr;
26 
27  void lowerResumeOrDestroy(CallBase &CB, CoroSubFnInst::ResumeKind);
28  void lowerCoroPromise(CoroPromiseInst *Intrin);
29  void lowerCoroDone(IntrinsicInst *II);
30  void lowerCoroNoop(IntrinsicInst *II);
31 
32 public:
33  Lowerer(Module &M)
34  : LowererBase(M), Builder(Context),
35  AnyResumeFnPtrTy(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
36  /*isVarArg=*/false)
37  ->getPointerTo()) {}
38  void lowerEarlyIntrinsics(Function &F);
39 };
40 }
41 
42 // Replace a direct call to coro.resume or coro.destroy with an indirect call to
43 // an address returned by coro.subfn.addr intrinsic. This is done so that
44 // CGPassManager recognizes devirtualization when CoroElide pass replaces a call
45 // to coro.subfn.addr with an appropriate function address.
46 void Lowerer::lowerResumeOrDestroy(CallBase &CB,
48  Value *ResumeAddr = makeSubFnCall(CB.getArgOperand(0), Index, &CB);
49  CB.setCalledOperand(ResumeAddr);
51 }
52 
53 // Coroutine promise field is always at the fixed offset from the beginning of
54 // the coroutine frame. i8* coro.promise(i8*, i1 from) intrinsic adds an offset
55 // to a passed pointer to move from coroutine frame to coroutine promise and
56 // vice versa. Since we don't know exactly which coroutine frame it is, we build
57 // a coroutine frame mock up starting with two function pointers, followed by a
58 // properly aligned coroutine promise field.
59 // TODO: Handle the case when coroutine promise alloca has align override.
60 void Lowerer::lowerCoroPromise(CoroPromiseInst *Intrin) {
61  Value *Operand = Intrin->getArgOperand(0);
62  Align Alignment = Intrin->getAlignment();
63  Type *Int8Ty = Builder.getInt8Ty();
64 
65  auto *SampleStruct =
66  StructType::get(Context, {AnyResumeFnPtrTy, AnyResumeFnPtrTy, Int8Ty});
67  const DataLayout &DL = TheModule.getDataLayout();
68  int64_t Offset = alignTo(
69  DL.getStructLayout(SampleStruct)->getElementOffset(2), Alignment);
70  if (Intrin->isFromPromise())
71  Offset = -Offset;
72 
73  Builder.SetInsertPoint(Intrin);
74  Value *Replacement =
75  Builder.CreateConstInBoundsGEP1_32(Int8Ty, Operand, Offset);
76 
77  Intrin->replaceAllUsesWith(Replacement);
78  Intrin->eraseFromParent();
79 }
80 
81 // When a coroutine reaches final suspend point, it zeros out ResumeFnAddr in
82 // the coroutine frame (it is UB to resume from a final suspend point).
83 // The llvm.coro.done intrinsic is used to check whether a coroutine is
84 // suspended at the final suspend point or not.
85 void Lowerer::lowerCoroDone(IntrinsicInst *II) {
86  Value *Operand = II->getArgOperand(0);
87 
88  // ResumeFnAddr is the first pointer sized element of the coroutine frame.
89  static_assert(coro::Shape::SwitchFieldIndex::Resume == 0,
90  "resume function not at offset zero");
91  auto *FrameTy = Int8Ptr;
92  PointerType *FramePtrTy = FrameTy->getPointerTo();
93 
94  Builder.SetInsertPoint(II);
95  auto *BCI = Builder.CreateBitCast(Operand, FramePtrTy);
96  auto *Load = Builder.CreateLoad(FrameTy, BCI);
97  auto *Cond = Builder.CreateICmpEQ(Load, NullPtr);
98 
100  II->eraseFromParent();
101 }
102 
103 void Lowerer::lowerCoroNoop(IntrinsicInst *II) {
104  if (!NoopCoro) {
105  LLVMContext &C = Builder.getContext();
106  Module &M = *II->getModule();
107 
108  // Create a noop.frame struct type.
109  StructType *FrameTy = StructType::create(C, "NoopCoro.Frame");
110  auto *FramePtrTy = FrameTy->getPointerTo();
111  auto *FnTy = FunctionType::get(Type::getVoidTy(C), FramePtrTy,
112  /*isVarArg=*/false);
113  auto *FnPtrTy = FnTy->getPointerTo();
114  FrameTy->setBody({FnPtrTy, FnPtrTy});
115 
116  // Create a Noop function that does nothing.
117  Function *NoopFn =
118  Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
119  "NoopCoro.ResumeDestroy", &M);
121  auto *Entry = BasicBlock::Create(C, "entry", NoopFn);
122  ReturnInst::Create(C, Entry);
123 
124  // Create a constant struct for the frame.
125  Constant* Values[] = {NoopFn, NoopFn};
126  Constant* NoopCoroConst = ConstantStruct::get(FrameTy, Values);
127  NoopCoro = new GlobalVariable(M, NoopCoroConst->getType(), /*isConstant=*/true,
128  GlobalVariable::PrivateLinkage, NoopCoroConst,
129  "NoopCoro.Frame.Const");
130  }
131 
132  Builder.SetInsertPoint(II);
133  auto *NoopCoroVoidPtr = Builder.CreateBitCast(NoopCoro, Int8Ptr);
134  II->replaceAllUsesWith(NoopCoroVoidPtr);
135  II->eraseFromParent();
136 }
137 
138 // Prior to CoroSplit, calls to coro.begin needs to be marked as NoDuplicate,
139 // as CoroSplit assumes there is exactly one coro.begin. After CoroSplit,
140 // NoDuplicate attribute will be removed from coro.begin otherwise, it will
141 // interfere with inlining.
142 static void setCannotDuplicate(CoroIdInst *CoroId) {
143  for (User *U : CoroId->users())
144  if (auto *CB = dyn_cast<CoroBeginInst>(U))
145  CB->setCannotDuplicate();
146 }
147 
148 void Lowerer::lowerEarlyIntrinsics(Function &F) {
149  CoroIdInst *CoroId = nullptr;
151  bool HasCoroSuspend = false;
153  auto *CB = dyn_cast<CallBase>(&I);
154  if (!CB)
155  continue;
156 
157  switch (CB->getIntrinsicID()) {
158  default:
159  continue;
160  case Intrinsic::coro_free:
161  CoroFrees.push_back(cast<CoroFreeInst>(&I));
162  break;
163  case Intrinsic::coro_suspend:
164  // Make sure that final suspend point is not duplicated as CoroSplit
165  // pass expects that there is at most one final suspend point.
166  if (cast<CoroSuspendInst>(&I)->isFinal())
167  CB->setCannotDuplicate();
168  HasCoroSuspend = true;
169  break;
170  case Intrinsic::coro_end_async:
171  case Intrinsic::coro_end:
172  // Make sure that fallthrough coro.end is not duplicated as CoroSplit
173  // pass expects that there is at most one fallthrough coro.end.
174  if (cast<AnyCoroEndInst>(&I)->isFallthrough())
175  CB->setCannotDuplicate();
176  break;
177  case Intrinsic::coro_noop:
178  lowerCoroNoop(cast<IntrinsicInst>(&I));
179  break;
180  case Intrinsic::coro_id:
181  if (auto *CII = cast<CoroIdInst>(&I)) {
182  if (CII->getInfo().isPreSplit()) {
183  assert(F.hasFnAttribute(CORO_PRESPLIT_ATTR) &&
184  "The frontend uses Swtich-Resumed ABI should emit "
185  "\"coroutine.presplit\" attribute for the coroutine.");
186  setCannotDuplicate(CII);
187  CII->setCoroutineSelf();
188  CoroId = cast<CoroIdInst>(&I);
189  }
190  }
191  break;
192  case Intrinsic::coro_id_retcon:
193  case Intrinsic::coro_id_retcon_once:
194  case Intrinsic::coro_id_async:
195  // TODO: Remove the line once we support it in the corresponding
196  // frontend.
197  F.addFnAttr(CORO_PRESPLIT_ATTR);
198  break;
199  case Intrinsic::coro_resume:
200  lowerResumeOrDestroy(*CB, CoroSubFnInst::ResumeIndex);
201  break;
202  case Intrinsic::coro_destroy:
203  lowerResumeOrDestroy(*CB, CoroSubFnInst::DestroyIndex);
204  break;
205  case Intrinsic::coro_promise:
206  lowerCoroPromise(cast<CoroPromiseInst>(&I));
207  break;
208  case Intrinsic::coro_done:
209  lowerCoroDone(cast<IntrinsicInst>(&I));
210  break;
211  }
212  }
213 
214  // Make sure that all CoroFree reference the coro.id intrinsic.
215  // Token type is not exposed through coroutine C/C++ builtins to plain C, so
216  // we allow specifying none and fixing it up here.
217  if (CoroId)
218  for (CoroFreeInst *CF : CoroFrees)
219  CF->setArgOperand(0, CoroId);
220 
221  // Coroutine suspention could potentially lead to any argument modified
222  // outside of the function, hence arguments should not have noalias
223  // attributes.
224  if (HasCoroSuspend)
225  for (Argument &A : F.args())
226  if (A.hasNoAliasAttr())
227  A.removeAttr(Attribute::NoAlias);
228 }
229 
230 static bool declaresCoroEarlyIntrinsics(const Module &M) {
232  M, {"llvm.coro.id", "llvm.coro.id.retcon", "llvm.coro.id.retcon.once",
233  "llvm.coro.id.async", "llvm.coro.destroy", "llvm.coro.done",
234  "llvm.coro.end", "llvm.coro.end.async", "llvm.coro.noop",
235  "llvm.coro.free", "llvm.coro.promise", "llvm.coro.resume",
236  "llvm.coro.suspend"});
237 }
238 
241  return PreservedAnalyses::all();
242 
243  Lowerer L(M);
244  for (auto &F : M)
245  L.lowerEarlyIntrinsics(F);
246 
248  PA.preserveSet<CFGAnalyses>();
249  return PA;
250 }
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:148
llvm::CoroSubFnInst::ResumeIndex
@ ResumeIndex
Definition: CoroInstr.h:41
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:65
llvm::StructType::get
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:406
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::CoroSubFnInst::DestroyIndex
@ DestroyIndex
Definition: CoroInstr.h:42
InstIterator.h
llvm::Function
Definition: Function.h:60
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1347
llvm::StructType::setBody
void setBody(ArrayRef< Type * > Elements, bool isPacked=false)
Specify a body for an opaque identified type.
Definition: Type.cpp:445
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:361
llvm::CoroSubFnInst::ResumeKind
ResumeKind
Definition: CoroInstr.h:39
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Module.h
llvm::coro::declaresIntrinsics
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:117
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::CoroFreeInst
This represents the llvm.coro.free instruction.
Definition: CoroInstr.h:404
setCannotDuplicate
static void setCannotDuplicate(CoroIdInst *CoroId)
Definition: CoroEarly.cpp:142
llvm::StructType::create
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Definition: Type.cpp:513
F
#define F(x, y, z)
Definition: MD5.cpp:55
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
llvm::User
Definition: User.h:44
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::ARM_PROC::A
@ A
Definition: ARMBaseInfo.h:34
llvm::CoroPromiseInst::getAlignment
Align getAlignment() const
The required alignment of the promise.
Definition: CoroInstr.h:464
llvm::Instruction
Definition: Instruction.h:42
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
CoroInternal.h
llvm::SPIRV::Decoration::Alignment
@ Alignment
llvm::CoroPromiseInst
This represents the llvm.coro.promise instruction.
Definition: CoroInstr.h:452
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::CallingConv::Fast
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:77
llvm::AMDGPU::Hwreg::Offset
Offset
Definition: SIDefines.h:394
llvm::coro::Shape::SwitchFieldIndex::Resume
@ Resume
Definition: CoroInternal.h:95
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:608
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:632
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:137
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::CallBase::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition: Instructions.cpp:311
llvm::CoroPromiseInst::isFromPromise
bool isFromPromise() const
Are we translating from the frame to the promise (false) or from the promise to the frame (true)?
Definition: CoroInstr.h:458
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:213
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:178
llvm::Function::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: Function.h:242
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition: PassManager.h:113
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:529
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:97
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::CoroEarlyPass::run
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition: CoroEarly.cpp:239
CORO_PRESPLIT_ATTR
#define CORO_PRESPLIT_ATTR
Definition: CoroInternal.h:30
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
Function.h
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:774
llvm::ReturnInst::Create
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3044
llvm::IntrinsicInst
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:46
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition: InstrTypes.h:1432
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:222
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
llvm::PreservedAnalyses::preserveSet
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:188
declaresCoroEarlyIntrinsics
static bool declaresCoroEarlyIntrinsics(const Module &M)
Definition: CoroEarly.cpp:230
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1341
llvm::coro::LowererBase
Definition: CoroInternal.h:46
CoroEarly.h
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1174
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
llvm::CallBase::setCannotDuplicate
void setCannotDuplicate()
Definition: InstrTypes.h:1904
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1459