69#define DEBUG_TYPE "inline"
71STATISTIC(NumInlined,
"Number of functions inlined");
72STATISTIC(NumDeleted,
"Number of functions deleted because all callers found");
77 "Cost multiplier to multiply onto inlined call sites where the "
78 "new call was previously an intra-SCC call (not relevant when the "
79 "original call was already intra-SCC). This can accumulate over "
80 "multiple inlinings (e.g. if a call site already had a cost "
81 "multiplier and one of its inlined calls was also subject to "
82 "this, the inlined call would have the original multiplier "
83 "multiplied by intra-scc-cost-multiplier). This is to prevent tons of "
84 "inlining through a child SCC which can cause terrible compile times"));
100 "Optimization remarks file containing inline remarks to be replayed "
101 "by cgscc inlining."),
105 "cgscc-inline-replay-scope",
106 cl::init(ReplayInlinerSettings::Scope::Function),
108 "Replay on functions that have remarks associated "
109 "with them (default)"),
110 clEnumValN(ReplayInlinerSettings::Scope::Module,
"Module",
111 "Replay on the entire module")),
112 cl::desc(
"Whether inline replay should be applied to the entire "
113 "Module or just the Functions (default) that are present as "
114 "callers in remarks during cgscc inlining."),
118 "cgscc-inline-replay-fallback",
119 cl::init(ReplayInlinerSettings::Fallback::Original),
122 ReplayInlinerSettings::Fallback::Original,
"Original",
123 "All decisions not in replay send to original advisor (default)"),
124 clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline,
125 "AlwaysInline",
"All decisions not in replay are inlined"),
126 clEnumValN(ReplayInlinerSettings::Fallback::NeverInline,
"NeverInline",
127 "All decisions not in replay are not inlined")),
129 "How cgscc inline replay treats sites that don't come from the replay. "
130 "Original: defers to original advisor, AlwaysInline: inline all sites "
131 "not in replay, NeverInline: inline no sites not in replay"),
135 "cgscc-inline-replay-format",
136 cl::init(CallSiteFormat::Format::LineColumnDiscriminator),
138 clEnumValN(CallSiteFormat::Format::Line,
"Line",
"<Line Number>"),
139 clEnumValN(CallSiteFormat::Format::LineColumn,
"LineColumn",
140 "<Line Number>:<Column Number>"),
141 clEnumValN(CallSiteFormat::Format::LineDiscriminator,
142 "LineDiscriminator",
"<Line Number>.<Discriminator>"),
143 clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator,
144 "LineColumnDiscriminator",
145 "<Line Number>:<Column Number>.<Discriminator> (default)")),
153 while (InlineHistoryID != -1) {
154 assert(
unsigned(InlineHistoryID) < InlineHistory.size() &&
155 "Invalid inline history ID");
156 if (InlineHistory[InlineHistoryID].first ==
F)
158 InlineHistoryID = InlineHistory[InlineHistoryID].second;
167 return *OwnedAdvisor;
179 OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
185 M,
FAM,
M.getContext(), std::move(OwnedAdvisor),
187 CGSCCInlineReplayScope,
188 CGSCCInlineReplayFallback,
189 {CGSCCInlineReplayFormat}},
193 return *OwnedAdvisor;
195 assert(IAA->getAdvisor() &&
196 "Expected a present InlineAdvisorAnalysis also have an "
197 "InlineAdvisor initialized");
198 return *IAA->getAdvisor();
204 const auto &MAMProxy =
206 bool Changed =
false;
208 assert(InitialC.
size() > 0 &&
"Cannot handle an empty SCC!");
209 Module &M = *InitialC.
begin()->getFunction().getParent();
249 for (
auto &
N : InitialC) {
258 if (
auto *CB = dyn_cast<CallBase>(&
I))
259 if (
Function *Callee = CB->getCalledFunction()) {
260 if (!Callee->isDeclaration())
261 Calls.push_back({CB, -1});
262 else if (!isa<IntrinsicInst>(
I)) {
267 << NV(
"Callee", Callee) <<
" will not be inlined into "
268 << NV(
"Caller", CB->getCaller())
269 <<
" because its definition is unavailable"
302 for (
int I = 0;
I < (int)Calls.size(); ++
I) {
309 if (CG.lookupSCC(
N) !=
C)
313 <<
" Function size: " <<
F.getInstructionCount()
323 bool DidInline =
false;
324 for (;
I < (int)Calls.size() && Calls[
I].first->getCaller() == &
F; ++
I) {
327 const int InlineHistoryID =
P.second;
330 if (InlineHistoryID != -1 &&
332 LLVM_DEBUG(
dbgs() <<
"Skipping inlining due to history: " <<
F.getName()
333 <<
" -> " << Callee.getName() <<
"\n");
347 if (CalleeSCC ==
C && UR.InlinedInternalEdges.count({&N, C})) {
348 LLVM_DEBUG(
dbgs() <<
"Skipping inlining internal SCC edge from a node "
349 "previously split out of this SCC by inlining: "
350 <<
F.getName() <<
" -> " <<
Callee.getName() <<
"\n");
355 std::unique_ptr<InlineAdvice> Advice =
356 Advisor.getAdvice(*CB, OnlyMandatory);
362 if (!Advice->isInliningRecommended()) {
363 Advice->recordUnattemptedInlining();
375 GetAssumptionCache, PSI,
382 if (!
IR.isSuccess()) {
383 Advice->recordUnsuccessfulInlining(
IR);
388 InlinedCallees.
insert(&Callee);
392 <<
F.getInstructionCount() <<
"\n");
395 if (!IFI.InlinedCallSites.empty()) {
396 int NewHistoryID = InlineHistory.
size();
400 Function *NewCallee = ICB->getCalledFunction();
402 "Intrinsic calls should not be tracked.");
409 NewCallee = ICB->getCalledFunction();
413 Calls.push_back({ICB, NewHistoryID});
423 if (CalleeSCC !=
C &&
424 CalleeSCC == CG.lookupSCC(CG.get(*NewCallee))) {
429 ICB->addFnAttr(NewCBCostMult);
441 bool CalleeWasDeleted =
false;
442 if (
Callee.isDiscardableIfUnused() &&
Callee.hasZeroLiveUses() &&
443 !CG.isLibFunction(Callee)) {
446 std::remove_if(Calls.begin() +
I + 1, Calls.end(),
447 [&](
const std::pair<CallBase *, int> &Call) {
448 return Call.first->getCaller() == &Callee;
456 Callee.dropAllReferences();
458 "Cannot put cause a function to become dead twice!");
460 CalleeWasDeleted =
true;
462 DeadFunctionsInComdats.
push_back(&Callee);
465 if (CalleeWasDeleted)
466 Advice->recordInliningWithCalleeDeleted();
468 Advice->recordInlining();
514 if ((
C != OldC || UR.CWorklist.count(OldC)) &&
516 return CG.lookupSCC(*CG.lookup(*Callee)) == OldC;
518 LLVM_DEBUG(
dbgs() <<
"Inlined an internal call edge and split an SCC, "
519 "retaining this to avoid infinite inlining.\n");
520 UR.InlinedInternalEdges.insert({&
N, OldC});
522 InlinedCallees.
clear();
531 if (!DeadFunctionsInComdats.
empty()) {
533 for (
auto *Callee : DeadFunctionsInComdats)
534 Callee->dropAllReferences();
535 DeadFunctions.
append(DeadFunctionsInComdats);
545 for (
Function *DeadF : DeadFunctions) {
548 auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
549 FAM.
clear(*DeadF, DeadF->getName());
550 AM.clear(DeadC, DeadC.getName());
551 auto &DeadRC = DeadC.getOuterRefSCC();
552 CG.removeDeadFunction(*DeadF);
556 UR.InvalidatedSCCs.insert(&DeadC);
557 UR.InvalidatedRefSCCs.insert(&DeadRC);
560 if (&DeadC == UR.UpdatedC)
561 UR.UpdatedC =
nullptr;
564 M.getFunctionList().erase(DeadF);
586 : Params(Params), IC(IC), Mode(Mode),
593 if (MandatoryFirst) {
606 if (!IAA.tryCreate(Params, Mode,
607 {CGSCCInlineReplayFile,
608 CGSCCInlineReplayScope,
609 CGSCCInlineReplayFallback,
610 {CGSCCInlineReplayFormat}},
612 M.getContext().emitError(
613 "Could not setup Inlining Advisor for the requested "
614 "mode and/or options");
645 OS, MapClassName2PassName);
647 OS <<
"<only-mandatory>";
660 if (MaxDevirtIterations != 0)
661 OS <<
"devirt<" << MaxDevirtIterations <<
">(";
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
This header provides classes for managing passes over SCCs of the call graph.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
static bool inlineHistoryIncludes(Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int > > &InlineHistory)
Return true if the specified inline history ID indicates an inline history that includes the specifie...
static cl::opt< ReplayInlinerSettings::Scope > CGSCCInlineReplayScope("cgscc-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during cgscc inlining."), cl::Hidden)
static cl::opt< bool > KeepAdvisorForPrinting("keep-inline-advisor-for-printing", cl::init(false), cl::Hidden)
A flag for test, so we can print the content of the advisor when running it as part of the default (e...
static cl::opt< std::string > CGSCCInlineReplayFile("cgscc-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by cgscc inlining."), cl::Hidden)
static cl::opt< bool > EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing", cl::init(false), cl::Hidden)
Allows printing the contents of the advisor after each SCC inliner pass.
static cl::opt< int > IntraSCCCostMultiplier("intra-scc-cost-multiplier", cl::init(2), cl::Hidden, cl::desc("Cost multiplier to multiply onto inlined call sites where the " "new call was previously an intra-SCC call (not relevant when the " "original call was already intra-SCC). This can accumulate over " "multiple inlinings (e.g. if a call site already had a cost " "multiplier and one of its inlined calls was also subject to " "this, the inlined call would have the original multiplier " "multiplied by intra-scc-cost-multiplier). This is to prevent tons of " "inlining through a child SCC which can cause terrible compile times"))
static cl::opt< CallSiteFormat::Format > CGSCCInlineReplayFormat("cgscc-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden)
static cl::opt< ReplayInlinerSettings::Fallback > CGSCCInlineReplayFallback("cgscc-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How cgscc inline replay treats sites that don't come from the replay. " "Original: defers to original advisor, AlwaysInline: inline all sites " "not in replay, NeverInline: inline no sites not in replay"), cl::Hidden)
Select target instructions out of generic instructions
Implements a lazy call graph analysis and related passes for the new pass manager.
Legalize the Machine IR a function s Machine IR
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file provides a priority worklist.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A manager for alias analyses.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
A container for analyses that lazily runs them and caches their results.
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Analysis pass which computes BlockFrequencyInfo.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Function * getCaller()
Helper to get the caller (the parent function).
A proxy from a FunctionAnalysisManager to an SCC.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Printer pass for the FunctionPropertiesAnalysis results.
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
virtual void onPassEntry(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
virtual void onPassExit(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
InlineResult is basically true or false.
The inliner pass for the new pass manager.
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
PreservedAnalyses run(Module &, ModuleAnalysisManager &)
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
ModuleInlinerWrapperPass(InlineParams Params=getInlineParams(), bool MandatoryFirst=true, InlineContext IC={}, InliningAdvisorMode Mode=InliningAdvisorMode::Default, unsigned MaxDevirtIterations=0)
A Module instance is used to store all the information related to an LLVM module.
Result proxy object for OuterAnalysisManagerProxy.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same< PassT, PassManager >::value > addPass(PassT &&Pass)
PreservedAnalyses run(IRUnitT &IR, AnalysisManagerT &AM, ExtraArgTs... ExtraArgs)
Run all of the passes in this manager over the given unit of IR.
bool isEmpty() const
Returns if the pass manager contains any passes.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
void clear()
Completely clear the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
An efficient, type-erasing, non-owning reference to a callable.
This class implements an extremely fast bulk output stream that can only output to a stream.
@ C
The default llvm calling convention, compatible with C.
const char FunctionInlineCostMultiplierAttributeName[]
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
std::optional< int > getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind)
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT &&Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT &&Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void filterDeadComdatFunctions(SmallVectorImpl< Function * > &DeadComdatFunctions)
Filter out potentially dead comdat functions where other entries keep the entire comdat group alive.
bool tryPromoteCall(CallBase &CB)
Try to promote (devirtualize) a virtual call on an Alloca.
cl::opt< unsigned > MaxDevirtIterations("max-devirt-iterations", cl::ReallyHidden, cl::init(4))
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Thresholds to tune inline cost analysis.
A CRTP mix-in to automatically provide informational APIs needed for passes.