LLVM 18.0.0git
PreISelIntrinsicLowering.cpp
Go to the documentation of this file.
1//===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10// llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11//
12//===----------------------------------------------------------------------===//
13
18#include "llvm/CodeGen/Passes.h"
21#include "llvm/IR/Function.h"
22#include "llvm/IR/IRBuilder.h"
25#include "llvm/IR/Module.h"
26#include "llvm/IR/Type.h"
28#include "llvm/Pass.h"
32
33using namespace llvm;
34
35/// Threshold to leave statically sized memory intrinsic calls. Calls of known
36/// size larger than this will be expanded by the pass. Calls of unknown or
37/// lower size will be left for expansion in codegen.
39 "mem-intrinsic-expand-size",
40 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1),
42
43namespace {
44
45struct PreISelIntrinsicLowering {
46 const TargetMachine &TM;
47 const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
48
49 /// If this is true, assume it's preferably to leave memory intrinsic calls
50 /// for replacement with a library call later. Otherwise this depends on
51 /// TargetLoweringInfo availability of the corresponding function.
52 const bool UseMemIntrinsicLibFunc;
53
54 explicit PreISelIntrinsicLowering(
55 const TargetMachine &TM_,
57 bool UseMemIntrinsicLibFunc_ = true)
58 : TM(TM_), LookupTTI(LookupTTI_),
59 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
60
61 static bool shouldExpandMemIntrinsicWithSize(Value *Size,
63 bool expandMemIntrinsicUses(Function &F) const;
64 bool lowerIntrinsics(Module &M) const;
65};
66
67} // namespace
68
70 if (F.use_empty())
71 return false;
72
73 bool Changed = false;
74 Type *Int32Ty = Type::getInt32Ty(F.getContext());
75 Type *Int8Ty = Type::getInt8Ty(F.getContext());
76
77 for (Use &U : llvm::make_early_inc_range(F.uses())) {
78 auto CI = dyn_cast<CallInst>(U.getUser());
79 if (!CI || CI->getCalledOperand() != &F)
80 continue;
81
82 IRBuilder<> B(CI);
83 Value *OffsetPtr =
84 B.CreateGEP(Int8Ty, CI->getArgOperand(0), CI->getArgOperand(1));
85 Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4));
86
87 Value *ResultPtr = B.CreateGEP(Int8Ty, CI->getArgOperand(0), OffsetI32);
88
89 CI->replaceAllUsesWith(ResultPtr);
90 CI->eraseFromParent();
91 Changed = true;
92 }
93
94 return Changed;
95}
96
97// ObjCARC has knowledge about whether an obj-c runtime function needs to be
98// always tail-called or never tail-called.
101 if (objcarc::IsAlwaysTail(Kind))
102 return CallInst::TCK_Tail;
103 else if (objcarc::IsNeverTail(Kind))
105 return CallInst::TCK_None;
106}
107
108static bool lowerObjCCall(Function &F, const char *NewFn,
109 bool setNonLazyBind = false) {
111 "Pre-ISel intrinsics do lower into regular function calls");
112 if (F.use_empty())
113 return false;
114
115 // If we haven't already looked up this function, check to see if the
116 // program already contains a function with this name.
117 Module *M = F.getParent();
118 FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType());
119
120 if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) {
121 Fn->setLinkage(F.getLinkage());
122 if (setNonLazyBind && !Fn->isWeakForLinker()) {
123 // If we have Native ARC, set nonlazybind attribute for these APIs for
124 // performance.
125 Fn->addFnAttr(Attribute::NonLazyBind);
126 }
127 }
128
130
131 for (Use &U : llvm::make_early_inc_range(F.uses())) {
132 auto *CB = cast<CallBase>(U.getUser());
133
134 if (CB->getCalledFunction() != &F) {
136 (void)Kind;
137 assert((Kind == objcarc::ARCInstKind::RetainRV ||
138 Kind == objcarc::ARCInstKind::UnsafeClaimRV) &&
139 "use expected to be the argument of operand bundle "
140 "\"clang.arc.attachedcall\"");
141 U.set(FCache.getCallee());
142 continue;
143 }
144
145 auto *CI = cast<CallInst>(CB);
146 assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
147
148 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
149 SmallVector<Value *, 8> Args(CI->args());
151 CI->getOperandBundlesAsDefs(BundleList);
152 CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList);
153 NewCI->setName(CI->getName());
154
155 // Try to set the most appropriate TailCallKind based on both the current
156 // attributes and the ones that we could get from ObjCARC's special
157 // knowledge of the runtime functions.
158 //
159 // std::max respects both requirements of notail and tail here:
160 // * notail on either the call or from ObjCARC becomes notail
161 // * tail on either side is stronger than none, but not notail
162 CallInst::TailCallKind TCK = CI->getTailCallKind();
163 NewCI->setTailCallKind(std::max(TCK, OverridingTCK));
164
165 // Transfer the 'returned' attribute from the intrinsic to the call site.
166 // By applying this only to intrinsic call sites, we avoid applying it to
167 // non-ARC explicit calls to things like objc_retain which have not been
168 // auto-upgraded to use the intrinsics.
169 unsigned Index;
170 if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) &&
171 Index)
173 Attribute::Returned);
174
175 if (!CI->use_empty())
176 CI->replaceAllUsesWith(NewCI);
177 CI->eraseFromParent();
178 }
179
180 return true;
181}
182
183// TODO: Should refine based on estimated number of accesses (e.g. does it
184// require splitting based on alignment)
185bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
187 ConstantInt *CI = dyn_cast<ConstantInt>(Size);
188 if (!CI)
189 return true;
190 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
193 uint64_t SizeVal = CI->getZExtValue();
194
195 // Treat a threshold of 0 as a special case to force expansion of all
196 // intrinsics, including size 0.
197 return SizeVal > Threshold || Threshold == 0;
198}
199
201 RTLIB::Libcall LC) {
202 // TODO: Should this consider the address space of the memcpy?
203 const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
204 return TLI->getLibcallName(LC) != nullptr;
205}
206
207// TODO: Handle atomic memcpy and memcpy.inline
208// TODO: Pass ScalarEvolution
209bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
210 Intrinsic::ID ID = F.getIntrinsicID();
211 bool Changed = false;
212
213 for (User *U : llvm::make_early_inc_range(F.users())) {
214 Instruction *Inst = cast<Instruction>(U);
215
216 switch (ID) {
217 case Intrinsic::memcpy: {
218 auto *Memcpy = cast<MemCpyInst>(Inst);
219 Function *ParentFunc = Memcpy->getFunction();
220 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
221 if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
222 if (UseMemIntrinsicLibFunc &&
223 canEmitLibcall(TM, ParentFunc, RTLIB::MEMCPY))
224 break;
225
226 // TODO: For optsize, emit the loop into a separate function
227 expandMemCpyAsLoop(Memcpy, TTI);
228 Changed = true;
229 Memcpy->eraseFromParent();
230 }
231
232 break;
233 }
234 case Intrinsic::memmove: {
235 auto *Memmove = cast<MemMoveInst>(Inst);
236 Function *ParentFunc = Memmove->getFunction();
237 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
238 if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
239 if (UseMemIntrinsicLibFunc &&
240 canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE))
241 break;
242
243 if (expandMemMoveAsLoop(Memmove, TTI)) {
244 Changed = true;
245 Memmove->eraseFromParent();
246 }
247 }
248
249 break;
250 }
251 case Intrinsic::memset: {
252 auto *Memset = cast<MemSetInst>(Inst);
253 Function *ParentFunc = Memset->getFunction();
254 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
255 if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) {
256 if (UseMemIntrinsicLibFunc &&
257 canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET))
258 break;
259
260 expandMemSetAsLoop(Memset);
261 Changed = true;
262 Memset->eraseFromParent();
263 }
264
265 break;
266 }
267 default:
268 llvm_unreachable("unhandled intrinsic");
269 }
270 }
271
272 return Changed;
273}
274
275bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
276 bool Changed = false;
277 for (Function &F : M) {
278 switch (F.getIntrinsicID()) {
279 default:
280 break;
281 case Intrinsic::memcpy:
282 case Intrinsic::memmove:
283 case Intrinsic::memset:
284 Changed |= expandMemIntrinsicUses(F);
285 break;
286 case Intrinsic::load_relative:
287 Changed |= lowerLoadRelative(F);
288 break;
289 case Intrinsic::objc_autorelease:
290 Changed |= lowerObjCCall(F, "objc_autorelease");
291 break;
292 case Intrinsic::objc_autoreleasePoolPop:
293 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPop");
294 break;
295 case Intrinsic::objc_autoreleasePoolPush:
296 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPush");
297 break;
298 case Intrinsic::objc_autoreleaseReturnValue:
299 Changed |= lowerObjCCall(F, "objc_autoreleaseReturnValue");
300 break;
301 case Intrinsic::objc_copyWeak:
302 Changed |= lowerObjCCall(F, "objc_copyWeak");
303 break;
304 case Intrinsic::objc_destroyWeak:
305 Changed |= lowerObjCCall(F, "objc_destroyWeak");
306 break;
307 case Intrinsic::objc_initWeak:
308 Changed |= lowerObjCCall(F, "objc_initWeak");
309 break;
310 case Intrinsic::objc_loadWeak:
311 Changed |= lowerObjCCall(F, "objc_loadWeak");
312 break;
313 case Intrinsic::objc_loadWeakRetained:
314 Changed |= lowerObjCCall(F, "objc_loadWeakRetained");
315 break;
316 case Intrinsic::objc_moveWeak:
317 Changed |= lowerObjCCall(F, "objc_moveWeak");
318 break;
319 case Intrinsic::objc_release:
320 Changed |= lowerObjCCall(F, "objc_release", true);
321 break;
322 case Intrinsic::objc_retain:
323 Changed |= lowerObjCCall(F, "objc_retain", true);
324 break;
325 case Intrinsic::objc_retainAutorelease:
326 Changed |= lowerObjCCall(F, "objc_retainAutorelease");
327 break;
328 case Intrinsic::objc_retainAutoreleaseReturnValue:
329 Changed |= lowerObjCCall(F, "objc_retainAutoreleaseReturnValue");
330 break;
331 case Intrinsic::objc_retainAutoreleasedReturnValue:
332 Changed |= lowerObjCCall(F, "objc_retainAutoreleasedReturnValue");
333 break;
334 case Intrinsic::objc_retainBlock:
335 Changed |= lowerObjCCall(F, "objc_retainBlock");
336 break;
337 case Intrinsic::objc_storeStrong:
338 Changed |= lowerObjCCall(F, "objc_storeStrong");
339 break;
340 case Intrinsic::objc_storeWeak:
341 Changed |= lowerObjCCall(F, "objc_storeWeak");
342 break;
343 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
344 Changed |= lowerObjCCall(F, "objc_unsafeClaimAutoreleasedReturnValue");
345 break;
346 case Intrinsic::objc_retainedObject:
347 Changed |= lowerObjCCall(F, "objc_retainedObject");
348 break;
349 case Intrinsic::objc_unretainedObject:
350 Changed |= lowerObjCCall(F, "objc_unretainedObject");
351 break;
352 case Intrinsic::objc_unretainedPointer:
353 Changed |= lowerObjCCall(F, "objc_unretainedPointer");
354 break;
355 case Intrinsic::objc_retain_autorelease:
356 Changed |= lowerObjCCall(F, "objc_retain_autorelease");
357 break;
358 case Intrinsic::objc_sync_enter:
359 Changed |= lowerObjCCall(F, "objc_sync_enter");
360 break;
361 case Intrinsic::objc_sync_exit:
362 Changed |= lowerObjCCall(F, "objc_sync_exit");
363 break;
364 }
365 }
366 return Changed;
367}
368
369namespace {
370
371class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
372public:
373 static char ID;
374
375 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
376
377 void getAnalysisUsage(AnalysisUsage &AU) const override {
380 }
381
382 bool runOnModule(Module &M) override {
383 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
384 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
385 };
386
387 const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
388 PreISelIntrinsicLowering Lowering(TM, LookupTTI);
389 return Lowering.lowerIntrinsics(M);
390 }
391};
392
393} // end anonymous namespace
394
395char PreISelIntrinsicLoweringLegacyPass::ID;
396
397INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
398 "pre-isel-intrinsic-lowering",
399 "Pre-ISel Intrinsic Lowering", false, false)
402INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
403 "pre-isel-intrinsic-lowering",
404 "Pre-ISel Intrinsic Lowering", false, false)
405
407 return new PreISelIntrinsicLoweringLegacyPass();
408}
409
412 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
413
414 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
416 };
417
418 PreISelIntrinsicLowering Lowering(TM, LookupTTI);
419 if (!Lowering.lowerIntrinsics(M))
420 return PreservedAnalyses::all();
421 else
423}
amdgpu isel
assume Assume Builder
static bool setNonLazyBind(Function &F)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool lowerIntrinsics(Module &M)
uint64_t Size
#define F(x, y, z)
Definition: MD5.cpp:55
Module.h This file contains the declarations for the Module class.
IntegerType * Int32Ty
This file defines ARC utility functions which are used by various parts of the compiler.
FunctionAnalysisManager FAM
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
static cl::opt< int64_t > MemIntrinsicExpandSizeThresholdOpt("mem-intrinsic-expand-size", cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1), cl::Hidden)
Threshold to leave statically sized memory intrinsic calls.
pre isel intrinsic lowering
pre isel intrinsic Pre ISel Intrinsic Lowering
static bool lowerObjCCall(Function &F, const char *NewFn, bool setNonLazyBind=false)
static CallInst::TailCallKind getOverridingTailCallKind(const Function &F)
static bool lowerLoadRelative(Function &F)
static bool canEmitLibcall(const TargetMachine &TM, Function *F, RTLIB::Libcall LC)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
This pass exposes codegen information to IR-level passes.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1541
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:145
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
const Function & getFunction() const
Definition: Function.h:136
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2628
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:933
static bool mayLowerToFunctionCall(Intrinsic::ID IID)
Check if the intrinsic might lower into a regular function call in the course of IR transformations.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: Pass.cpp:98
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
Analysis pass providing the TargetTransformInfo.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
Target-Independent Code Generator Pass Configuration Options.
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
uint64_t getMaxMemIntrinsicInlineSizeThreshold() const
Returns the maximum memset / memcpy size in bytes that still makes it profitable to inline the call.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
LLVM Value Representation.
Definition: Value.h:74
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:378
An efficient, type-erasing, non-owning reference to a callable.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
ARCInstKind getAttachedARCFunctionKind(const CallBase *CB)
This function returns the ARCInstKind of the function attached to operand bundle clang_arc_attachedca...
Definition: ObjCARCUtil.h:60
bool IsNeverTail(ARCInstKind Class)
Test if the given class represents instructions which are never safe to mark with the "tail" keyword.
bool IsAlwaysTail(ARCInstKind Class)
Test if the given class represents instructions which are always safe to mark with the "tail" keyword...
ARCInstKind
Equivalence classes of instructions in the ARC Model.
ARCInstKind GetFunctionClass(const Function *F)
Determine if F is one of the special known Functions.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool expandMemMoveAsLoop(MemMoveInst *MemMove, const TargetTransformInfo &TTI)
Expand MemMove as a loop.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:666
ModulePass * createPreISelIntrinsicLoweringPass()
This pass lowers the @llvm.load.relative and @llvm.objc.
void expandMemCpyAsLoop(MemCpyInst *MemCpy, const TargetTransformInfo &TTI, ScalarEvolution *SE=nullptr)
Expand MemCpy as a loop. MemCpy is not deleted.
void expandMemSetAsLoop(MemSetInst *MemSet)
Expand MemSet as a loop. MemSet is not deleted.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)