LLVM 19.0.0git
ReplaceWithVeclib.cpp
Go to the documentation of this file.
1//=== ReplaceWithVeclib.cpp - Replace vector intrinsics with veclib calls -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Replaces LLVM IR instructions with vector operands (i.e., the frem
10// instruction or calls to LLVM intrinsics) with matching calls to functions
11// from a vector library (e.g libmvec, SVML) using TargetLibraryInfo interface.
12//
13//===----------------------------------------------------------------------===//
14
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/Statistic.h"
18#include "llvm/ADT/StringRef.h"
24#include "llvm/CodeGen/Passes.h"
26#include "llvm/IR/IRBuilder.h"
31
32using namespace llvm;
33
34#define DEBUG_TYPE "replace-with-veclib"
35
36STATISTIC(NumCallsReplaced,
37 "Number of calls to intrinsics that have been replaced.");
38
39STATISTIC(NumTLIFuncDeclAdded,
40 "Number of vector library function declarations added.");
41
42STATISTIC(NumFuncUsedAdded,
43 "Number of functions added to `llvm.compiler.used`");
44
45/// Returns a vector Function that it adds to the Module \p M. When an \p
46/// ScalarFunc is not null, it copies its attributes to the newly created
47/// Function.
49 const StringRef TLIName,
50 Function *ScalarFunc = nullptr) {
51 Function *TLIFunc = M->getFunction(TLIName);
52 if (!TLIFunc) {
53 TLIFunc =
54 Function::Create(VectorFTy, Function::ExternalLinkage, TLIName, *M);
55 if (ScalarFunc)
56 TLIFunc->copyAttributesFrom(ScalarFunc);
57
58 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added vector library function `"
59 << TLIName << "` of type `" << *(TLIFunc->getType())
60 << "` to module.\n");
61
62 ++NumTLIFuncDeclAdded;
63 // Add the freshly created function to llvm.compiler.used, similar to as it
64 // is done in InjectTLIMappings.
65 appendToCompilerUsed(*M, {TLIFunc});
66 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << TLIName
67 << "` to `@llvm.compiler.used`.\n");
68 ++NumFuncUsedAdded;
69 }
70 return TLIFunc;
71}
72
73/// Replace the instruction \p I with a call to the corresponding function from
74/// the vector library (\p TLIVecFunc).
76 Function *TLIVecFunc) {
78 auto *CI = dyn_cast<CallInst>(&I);
79 SmallVector<Value *> Args(CI ? CI->args() : I.operands());
80 if (auto OptMaskpos = Info.getParamIndexForOptionalMask()) {
81 auto *MaskTy =
82 VectorType::get(Type::getInt1Ty(I.getContext()), Info.Shape.VF);
83 Args.insert(Args.begin() + OptMaskpos.value(),
85 }
86
87 // If it is a call instruction, preserve the operand bundles.
89 if (CI)
90 CI->getOperandBundlesAsDefs(OpBundles);
91
92 auto *Replacement = IRBuilder.CreateCall(TLIVecFunc, Args, OpBundles);
93 I.replaceAllUsesWith(Replacement);
94 // Preserve fast math flags for FP math.
95 if (isa<FPMathOperator>(Replacement))
96 Replacement->copyFastMathFlags(&I);
97}
98
99/// Returns true when successfully replaced \p I with a suitable function taking
100/// vector arguments, based on available mappings in the \p TLI. Currently only
101/// works when \p I is a call to vectorized intrinsic or the frem instruction.
103 Instruction &I) {
104 // At the moment VFABI assumes the return type is always widened unless it is
105 // a void type.
106 auto *VTy = dyn_cast<VectorType>(I.getType());
107 ElementCount EC(VTy ? VTy->getElementCount() : ElementCount::getFixed(0));
108
109 // Compute the argument types of the corresponding scalar call and the scalar
110 // function name. For calls, it additionally finds the function to replace
111 // and checks that all vector operands match the previously found EC.
112 SmallVector<Type *, 8> ScalarArgTypes;
113 std::string ScalarName;
114 Function *FuncToReplace = nullptr;
115 auto *CI = dyn_cast<CallInst>(&I);
116 if (CI) {
117 FuncToReplace = CI->getCalledFunction();
118 Intrinsic::ID IID = FuncToReplace->getIntrinsicID();
119 assert(IID != Intrinsic::not_intrinsic && "Not an intrinsic");
120 for (auto Arg : enumerate(CI->args())) {
121 auto *ArgTy = Arg.value()->getType();
122 if (isVectorIntrinsicWithScalarOpAtArg(IID, Arg.index())) {
123 ScalarArgTypes.push_back(ArgTy);
124 } else if (auto *VectorArgTy = dyn_cast<VectorType>(ArgTy)) {
125 ScalarArgTypes.push_back(VectorArgTy->getElementType());
126 // When return type is void, set EC to the first vector argument, and
127 // disallow vector arguments with different ECs.
128 if (EC.isZero())
129 EC = VectorArgTy->getElementCount();
130 else if (EC != VectorArgTy->getElementCount())
131 return false;
132 } else
133 // Exit when it is supposed to be a vector argument but it isn't.
134 return false;
135 }
136 // Try to reconstruct the name for the scalar version of the instruction,
137 // using scalar argument types.
138 ScalarName = Intrinsic::isOverloaded(IID)
139 ? Intrinsic::getName(IID, ScalarArgTypes, I.getModule())
140 : Intrinsic::getName(IID).str();
141 } else {
142 assert(VTy && "Return type must be a vector");
143 auto *ScalarTy = VTy->getScalarType();
144 LibFunc Func;
145 if (!TLI.getLibFunc(I.getOpcode(), ScalarTy, Func))
146 return false;
147 ScalarName = TLI.getName(Func);
148 ScalarArgTypes = {ScalarTy, ScalarTy};
149 }
150
151 // Try to find the mapping for the scalar version of this intrinsic and the
152 // exact vector width of the call operands in the TargetLibraryInfo. First,
153 // check with a non-masked variant, and if that fails try with a masked one.
154 const VecDesc *VD =
155 TLI.getVectorMappingInfo(ScalarName, EC, /*Masked*/ false);
156 if (!VD && !(VD = TLI.getVectorMappingInfo(ScalarName, EC, /*Masked*/ true)))
157 return false;
158
159 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Found TLI mapping from: `" << ScalarName
160 << "` and vector width " << EC << " to: `"
161 << VD->getVectorFnName() << "`.\n");
162
163 // Replace the call to the intrinsic with a call to the vector library
164 // function.
165 Type *ScalarRetTy = I.getType()->getScalarType();
166 FunctionType *ScalarFTy =
167 FunctionType::get(ScalarRetTy, ScalarArgTypes, /*isVarArg*/ false);
168 const std::string MangledName = VD->getVectorFunctionABIVariantString();
169 auto OptInfo = VFABI::tryDemangleForVFABI(MangledName, ScalarFTy);
170 if (!OptInfo)
171 return false;
172
173 // There is no guarantee that the vectorized instructions followed the VFABI
174 // specification when being created, this is why we need to add extra check to
175 // make sure that the operands of the vector function obtained via VFABI match
176 // the operands of the original vector instruction.
177 if (CI) {
178 for (auto &VFParam : OptInfo->Shape.Parameters) {
179 if (VFParam.ParamKind == VFParamKind::GlobalPredicate)
180 continue;
181
182 // tryDemangleForVFABI must return valid ParamPos, otherwise it could be
183 // a bug in the VFABI parser.
184 assert(VFParam.ParamPos < CI->arg_size() &&
185 "ParamPos has invalid range.");
186 Type *OrigTy = CI->getArgOperand(VFParam.ParamPos)->getType();
187 if (OrigTy->isVectorTy() != (VFParam.ParamKind == VFParamKind::Vector)) {
188 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Will not replace: " << ScalarName
189 << ". Wrong type at index " << VFParam.ParamPos
190 << ": " << *OrigTy << "\n");
191 return false;
192 }
193 }
194 }
195
196 FunctionType *VectorFTy = VFABI::createFunctionType(*OptInfo, ScalarFTy);
197 if (!VectorFTy)
198 return false;
199
200 Function *TLIFunc = getTLIFunction(I.getModule(), VectorFTy,
201 VD->getVectorFnName(), FuncToReplace);
202
203 replaceWithTLIFunction(I, *OptInfo, TLIFunc);
204 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Replaced call to `" << ScalarName
205 << "` with call to `" << TLIFunc->getName() << "`.\n");
206 ++NumCallsReplaced;
207 return true;
208}
209
210/// Supported instruction \p I must be a vectorized frem or a call to an
211/// intrinsic that returns either void or a vector.
213 Type *Ty = I->getType();
214 if (auto *CI = dyn_cast<CallInst>(I))
215 return (Ty->isVectorTy() || Ty->isVoidTy()) && CI->getCalledFunction() &&
216 CI->getCalledFunction()->getIntrinsicID() !=
218 if (I->getOpcode() == Instruction::FRem && Ty->isVectorTy())
219 return true;
220 return false;
221}
222
223static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
224 bool Changed = false;
225 SmallVector<Instruction *> ReplacedCalls;
226 for (auto &I : instructions(F)) {
228 continue;
229 if (replaceWithCallToVeclib(TLI, I)) {
230 ReplacedCalls.push_back(&I);
231 Changed = true;
232 }
233 }
234 // Erase the calls to the intrinsics that have been replaced
235 // with calls to the vector library.
236 for (auto *CI : ReplacedCalls)
237 CI->eraseFromParent();
238 return Changed;
239}
240
241////////////////////////////////////////////////////////////////////////////////
242// New pass manager implementation.
243////////////////////////////////////////////////////////////////////////////////
247 auto Changed = runImpl(TLI, F);
248 if (Changed) {
249 LLVM_DEBUG(dbgs() << "Instructions replaced with vector libraries: "
250 << NumCallsReplaced << "\n");
251
259 return PA;
260 }
261
262 // The pass did not replace any calls, hence it preserves all analyses.
263 return PreservedAnalyses::all();
264}
265
266////////////////////////////////////////////////////////////////////////////////
267// Legacy PM Implementation.
268////////////////////////////////////////////////////////////////////////////////
270 const TargetLibraryInfo &TLI =
271 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
272 return runImpl(TLI, F);
273}
274
276 AU.setPreservesCFG();
283}
284
285////////////////////////////////////////////////////////////////////////////////
286// Legacy Pass manager initialization
287////////////////////////////////////////////////////////////////////////////////
289
291 "Replace intrinsics with calls to vector library", false,
292 false)
295 "Replace intrinsics with calls to vector library", false,
296 false)
297
299 return new ReplaceWithVeclibLegacy();
300}
AMDGPU promote alloca to vector or false DEBUG_TYPE to vector
Expand Atomic instructions
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
#define LLVM_DEBUG(X)
Definition: Debug.h:101
static bool runImpl(Function &F, const TargetLowering &TLI)
expand Expand reduction intrinsics
#define DEBUG_TYPE
This is the interface for a simple mod/ref and alias analysis over globals.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
Function * getTLIFunction(Module *M, FunctionType *VectorFTy, const StringRef TLIName, Function *ScalarFunc=nullptr)
Returns a vector Function that it adds to the Module M.
static void replaceWithTLIFunction(Instruction &I, VFInfo &Info, Function *TLIVecFunc)
Replace the instruction I with a call to the corresponding function from the vector library (TLIVecFu...
static bool isSupportedInstruction(Instruction *I)
Supported instruction I must be a vectorized frem or a call to an intrinsic that returns either void ...
Replace intrinsics with calls to vector library
static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI, Instruction &I)
Returns true when successfully replaced I with a suitable function taking vector arguments,...
static bool runImpl(const TargetLibraryInfo &TLI, Function &F)
#define DEBUG_TYPE
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:348
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:500
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:269
Represents analyses that only rely on functions' control flow.
Definition: Analysis.h:70
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:417
An analysis that produces DemandedBits for a function.
Definition: DemandedBits.h:101
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:296
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:162
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:230
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
Definition: Function.cpp:793
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:294
Legacy wrapper pass to provide the GlobalsAAResult object.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2390
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2644
This analysis provides dependence information for the memory accesses of a loop.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
OptimizationRemarkEmitter legacy analysis pass.
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
void preserveSet()
Mark an analysis set as preserved.
Definition: Analysis.h:144
void preserve()
Mark an analysis as preserved.
Definition: Analysis.h:129
Analysis pass that exposes the ScalarEvolution for a function.
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:222
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
StringRef getName(LibFunc F) const
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
static IntegerType * getInt1Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
Provides info so a possible vectorization of a function can be computed.
std::string getVectorFunctionABIVariantString() const
Returns a vector function ABI variant string on the form: ZGV<isa><mask><vlen><vparams><scalarname>(<...
StringRef getVectorFnName() const
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:1025
bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition: Function.cpp:1448
std::optional< VFInfo > tryDemangleForVFABI(StringRef MangledName, const FunctionType *FTy)
Function to construct a VFInfo out of a mangled names in the following format:
FunctionType * createFunctionType(const VFInfo &Info, const FunctionType *ScalarFTy)
Constructs a FunctionType by applying vector function information to the type of a matching scalar fu...
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Definition: STLExtras.h:2415
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
FunctionPass * createReplaceWithVeclibLegacyPass()
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool runOnFunction(Function &F) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Holds the VFShape for a specific scalar to vector function mapping.