70#define DEBUG_TYPE "inline"
72STATISTIC(NumInlined,
"Number of functions inlined");
73STATISTIC(NumDeleted,
"Number of functions deleted because all callers found");
78 "Cost multiplier to multiply onto inlined call sites where the "
79 "new call was previously an intra-SCC call (not relevant when the "
80 "original call was already intra-SCC). This can accumulate over "
81 "multiple inlinings (e.g. if a call site already had a cost "
82 "multiplier and one of its inlined calls was also subject to "
83 "this, the inlined call would have the original multiplier "
84 "multiplied by intra-scc-cost-multiplier). This is to prevent tons of "
85 "inlining through a child SCC which can cause terrible compile times"));
101 "Optimization remarks file containing inline remarks to be replayed "
102 "by cgscc inlining."),
106 "cgscc-inline-replay-scope",
107 cl::init(ReplayInlinerSettings::Scope::Function),
109 "Replay on functions that have remarks associated "
110 "with them (default)"),
111 clEnumValN(ReplayInlinerSettings::Scope::Module,
"Module",
112 "Replay on the entire module")),
113 cl::desc(
"Whether inline replay should be applied to the entire "
114 "Module or just the Functions (default) that are present as "
115 "callers in remarks during cgscc inlining."),
119 "cgscc-inline-replay-fallback",
120 cl::init(ReplayInlinerSettings::Fallback::Original),
123 ReplayInlinerSettings::Fallback::Original,
"Original",
124 "All decisions not in replay send to original advisor (default)"),
125 clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline,
126 "AlwaysInline",
"All decisions not in replay are inlined"),
127 clEnumValN(ReplayInlinerSettings::Fallback::NeverInline,
"NeverInline",
128 "All decisions not in replay are not inlined")),
130 "How cgscc inline replay treats sites that don't come from the replay. "
131 "Original: defers to original advisor, AlwaysInline: inline all sites "
132 "not in replay, NeverInline: inline no sites not in replay"),
136 "cgscc-inline-replay-format",
137 cl::init(CallSiteFormat::Format::LineColumnDiscriminator),
139 clEnumValN(CallSiteFormat::Format::Line,
"Line",
"<Line Number>"),
140 clEnumValN(CallSiteFormat::Format::LineColumn,
"LineColumn",
141 "<Line Number>:<Column Number>"),
142 clEnumValN(CallSiteFormat::Format::LineDiscriminator,
143 "LineDiscriminator",
"<Line Number>.<Discriminator>"),
144 clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator,
145 "LineColumnDiscriminator",
146 "<Line Number>:<Column Number>.<Discriminator> (default)")),
154 while (InlineHistoryID != -1) {
155 assert(
unsigned(InlineHistoryID) < InlineHistory.size() &&
156 "Invalid inline history ID");
157 if (InlineHistory[InlineHistoryID].first ==
F)
159 InlineHistoryID = InlineHistory[InlineHistoryID].second;
168 return *OwnedAdvisor;
180 OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
186 M,
FAM,
M.getContext(), std::move(OwnedAdvisor),
188 CGSCCInlineReplayScope,
189 CGSCCInlineReplayFallback,
190 {CGSCCInlineReplayFormat}},
194 return *OwnedAdvisor;
196 assert(IAA->getAdvisor() &&
197 "Expected a present InlineAdvisorAnalysis also have an "
198 "InlineAdvisor initialized");
199 return *IAA->getAdvisor();
205 const auto &MAMProxy =
207 bool Changed =
false;
209 assert(InitialC.
size() > 0 &&
"Cannot handle an empty SCC!");
210 Module &M = *InitialC.
begin()->getFunction().getParent();
250 for (
auto &
N : InitialC) {
259 if (
auto *CB = dyn_cast<CallBase>(&
I))
261 if (!
Callee->isDeclaration())
262 Calls.push_back({CB, -1});
263 else if (!isa<IntrinsicInst>(
I)) {
268 << NV(
"Callee",
Callee) <<
" will not be inlined into "
269 << NV(
"Caller", CB->getCaller())
270 <<
" because its definition is unavailable"
303 for (
int I = 0;
I < (int)Calls.size(); ++
I) {
310 if (CG.lookupSCC(
N) !=
C)
314 <<
" Function size: " <<
F.getInstructionCount()
324 bool DidInline =
false;
325 for (;
I < (int)Calls.size() && Calls[
I].first->getCaller() == &
F; ++
I) {
328 const int InlineHistoryID =
P.second;
331 if (InlineHistoryID != -1 &&
333 LLVM_DEBUG(
dbgs() <<
"Skipping inlining due to history: " <<
F.getName()
334 <<
" -> " <<
Callee.getName() <<
"\n");
345 if (CalleeSCC ==
C && UR.InlinedInternalEdges.count({&N, C})) {
346 LLVM_DEBUG(
dbgs() <<
"Skipping inlining internal SCC edge from a node "
347 "previously split out of this SCC by inlining: "
348 <<
F.getName() <<
" -> " <<
Callee.getName() <<
"\n");
353 std::unique_ptr<InlineAdvice> Advice =
354 Advisor.getAdvice(*CB, OnlyMandatory);
360 if (!Advice->isInliningRecommended()) {
361 Advice->recordUnattemptedInlining();
373 GetAssumptionCache, PSI,
380 if (!
IR.isSuccess()) {
381 Advice->recordUnsuccessfulInlining(
IR);
390 <<
F.getInstructionCount() <<
"\n");
393 if (!IFI.InlinedCallSites.empty()) {
394 int NewHistoryID = InlineHistory.
size();
398 Function *NewCallee = ICB->getCalledFunction();
400 "Intrinsic calls should not be tracked.");
407 NewCallee = ICB->getCalledFunction();
411 Calls.push_back({ICB, NewHistoryID});
421 if (CalleeSCC !=
C &&
422 CalleeSCC == CG.lookupSCC(CG.get(*NewCallee))) {
427 ICB->addFnAttr(NewCBCostMult);
439 bool CalleeWasDeleted =
false;
440 if (
Callee.isDiscardableIfUnused() &&
Callee.hasZeroLiveUses() &&
441 !CG.isLibFunction(
Callee)) {
444 std::remove_if(Calls.begin() +
I + 1, Calls.end(),
445 [&](
const std::pair<CallBase *, int> &Call) {
446 return Call.first->getCaller() == &Callee;
454 Callee.dropAllReferences();
456 "Cannot put cause a function to become dead twice!");
458 CalleeWasDeleted =
true;
463 if (CalleeWasDeleted)
464 Advice->recordInliningWithCalleeDeleted();
466 Advice->recordInlining();
512 if ((
C != OldC || UR.CWorklist.count(OldC)) &&
514 return CG.lookupSCC(*CG.lookup(*Callee)) == OldC;
516 LLVM_DEBUG(
dbgs() <<
"Inlined an internal call edge and split an SCC, "
517 "retaining this to avoid infinite inlining.\n");
518 UR.InlinedInternalEdges.insert({&
N, OldC});
520 InlinedCallees.
clear();
529 if (!DeadFunctionsInComdats.
empty()) {
531 for (
auto *
Callee : DeadFunctionsInComdats)
532 Callee->dropAllReferences();
533 DeadFunctions.
append(DeadFunctionsInComdats);
543 for (
Function *DeadF : DeadFunctions) {
546 auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
547 FAM.
clear(*DeadF, DeadF->getName());
548 AM.clear(DeadC, DeadC.getName());
549 auto &DeadRC = DeadC.getOuterRefSCC();
550 CG.removeDeadFunction(*DeadF);
554 UR.InvalidatedSCCs.insert(&DeadC);
555 UR.InvalidatedRefSCCs.insert(&DeadRC);
558 if (&DeadC == UR.UpdatedC)
559 UR.UpdatedC =
nullptr;
562 M.getFunctionList().erase(DeadF);
584 : Params(Params), IC(IC), Mode(Mode),
591 if (MandatoryFirst) {
604 if (!IAA.tryCreate(Params, Mode,
605 {CGSCCInlineReplayFile,
606 CGSCCInlineReplayScope,
607 CGSCCInlineReplayFallback,
608 {CGSCCInlineReplayFormat}},
610 M.getContext().emitError(
611 "Could not setup Inlining Advisor for the requested "
612 "mode and/or options");
643 OS, MapClassName2PassName);
645 OS <<
"<only-mandatory>";
658 if (MaxDevirtIterations != 0)
659 OS <<
"devirt<" << MaxDevirtIterations <<
">(";
amdgpu Simplify well known AMD library false FunctionCallee Callee
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
This header provides classes for managing passes over SCCs of the call graph.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file defines the DenseMap class.
static bool inlineHistoryIncludes(Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int > > &InlineHistory)
Return true if the specified inline history ID indicates an inline history that includes the specifie...
static cl::opt< ReplayInlinerSettings::Scope > CGSCCInlineReplayScope("cgscc-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during cgscc inlining."), cl::Hidden)
static cl::opt< bool > KeepAdvisorForPrinting("keep-inline-advisor-for-printing", cl::init(false), cl::Hidden)
A flag for test, so we can print the content of the advisor when running it as part of the default (e...
static cl::opt< std::string > CGSCCInlineReplayFile("cgscc-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by cgscc inlining."), cl::Hidden)
static cl::opt< bool > EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing", cl::init(false), cl::Hidden)
Allows printing the contents of the advisor after each SCC inliner pass.
static cl::opt< int > IntraSCCCostMultiplier("intra-scc-cost-multiplier", cl::init(2), cl::Hidden, cl::desc("Cost multiplier to multiply onto inlined call sites where the " "new call was previously an intra-SCC call (not relevant when the " "original call was already intra-SCC). This can accumulate over " "multiple inlinings (e.g. if a call site already had a cost " "multiplier and one of its inlined calls was also subject to " "this, the inlined call would have the original multiplier " "multiplied by intra-scc-cost-multiplier). This is to prevent tons of " "inlining through a child SCC which can cause terrible compile times"))
static cl::opt< CallSiteFormat::Format > CGSCCInlineReplayFormat("cgscc-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden)
static cl::opt< ReplayInlinerSettings::Fallback > CGSCCInlineReplayFallback("cgscc-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How cgscc inline replay treats sites that don't come from the replay. " "Original: defers to original advisor, AlwaysInline: inline all sites " "not in replay, NeverInline: inline no sites not in replay"), cl::Hidden)
Implements a lazy call graph analysis and related passes for the new pass manager.
Statically lint checks LLVM IR
Module.h This file contains the declarations for the Module class.
print must be executed print the must be executed context for all instructions
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file provides a priority worklist.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A manager for alias analyses.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
A container for analyses that lazily runs them and caches their results.
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Analysis pass which computes BlockFrequencyInfo.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Function * getCaller()
Helper to get the caller (the parent function).
A proxy from a FunctionAnalysisManager to an SCC.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Printer pass for the FunctionPropertiesAnalysis results.
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
virtual void onPassEntry(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
virtual void onPassExit(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
InlineResult is basically true or false.
The inliner pass for the new pass manager.
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
PreservedAnalyses run(Module &, ModuleAnalysisManager &)
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
ModuleInlinerWrapperPass(InlineParams Params=getInlineParams(), bool MandatoryFirst=true, InlineContext IC={}, InliningAdvisorMode Mode=InliningAdvisorMode::Default, unsigned MaxDevirtIterations=0)
A Module instance is used to store all the information related to an LLVM module.
Result proxy object for OuterAnalysisManagerProxy.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same< PassT, PassManager >::value > addPass(PassT &&Pass)
PreservedAnalyses run(IRUnitT &IR, AnalysisManagerT &AM, ExtraArgTs... ExtraArgs)
Run all of the passes in this manager over the given unit of IR.
bool isEmpty() const
Returns if the pass manager contains any passes.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
bool insert(const value_type &X)
Insert a new element into the SetVector.
void clear()
Completely clear the SetVector.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
An efficient, type-erasing, non-owning reference to a callable.
This class implements an extremely fast bulk output stream that can only output to a stream.
@ C
The default llvm calling convention, compatible with C.
const char FunctionInlineCostMultiplierAttributeName[]
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
std::optional< int > getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind)
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT &&Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT &&Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void filterDeadComdatFunctions(SmallVectorImpl< Function * > &DeadComdatFunctions)
Filter out potentially dead comdat functions where other entries keep the entire comdat group alive.
bool tryPromoteCall(CallBase &CB)
Try to promote (devirtualize) a virtual call on an Alloca.
cl::opt< unsigned > MaxDevirtIterations("max-devirt-iterations", cl::ReallyHidden, cl::init(4))
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Thresholds to tune inline cost analysis.
A CRTP mix-in to automatically provide informational APIs needed for passes.