36 "inliner-interactive-channel-base",
cl::Hidden,
38 "Base file path for the interactive mode. The incoming filename should "
39 "have the name <inliner-interactive-channel-base>.in, while the "
40 "outgoing name should be <inliner-interactive-channel-base>.out"));
42 (
Twine(
"In interactive mode, also send the default policy decision: ") +
49#if defined(LLVM_HAVE_TF_AOT_INLINERSIZEMODEL)
51#include "InlinerSizeModel.h"
57std::unique_ptr<InlineAdvisor>
59 std::function<
bool(
CallBase &)> GetDefaultAdvice) {
60 if (!llvm::isEmbeddedModelEvaluatorValid<CompiledModelType>() &&
63 std::unique_ptr<MLModelRunner> AOTRunner;
65 AOTRunner = std::make_unique<ReleaseModeModelRunner<CompiledModelType>>(
71 AOTRunner = std::make_unique<InteractiveModelRunner>(
76 return std::make_unique<MLInlineAdvisor>(M,
MAM, std::move(AOTRunner),
80#define DEBUG_TYPE "inline-ml"
83 "ml-advisor-size-increase-threshold",
cl::Hidden,
84 cl::desc(
"Maximum factor by which expected native size may increase before "
85 "blocking any further inlining."),
91 "For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"),
96#define POPULATE_NAMES(DTYPE, SHAPE, NAME, __) TensorSpec::createSpec<DTYPE>(#NAME, SHAPE),
115 if (
auto *CS = dyn_cast<CallBase>(&
I))
117 if (!
Callee->isDeclaration()) {
126 std::unique_ptr<MLModelRunner> Runner,
127 std::function<
bool(
CallBase &)> GetDefaultAdvice)
130 ModelRunner(
std::
move(Runner)), GetDefaultAdvice(GetDefaultAdvice),
132 InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize) {
143 const std::vector<CallGraphNode *> &CGNodes = *
I;
145 for (
auto *CGNode : CGNodes) {
147 if (!
F ||
F->isDeclaration())
151 auto *Called = CS->getCalledFunction();
152 auto Pos = FunctionLevels.find(&CG.
get(*Called));
156 if (Pos == FunctionLevels.end())
158 Level = std::max(Level, Pos->second + 1);
162 for (
auto *CGNode : CGNodes) {
164 if (
F && !
F->isDeclaration())
165 FunctionLevels[&CG.
get(*
F)] = Level;
168 for (
auto KVP : FunctionLevels) {
169 AllNodes.insert(KVP.first);
172 NodeCount = AllNodes.size();
180 if (!LastSCC || ForceStop)
196 NodeCount -=
static_cast<int64_t
>(NodesInLastSCC.size());
197 while (!NodesInLastSCC.empty()) {
198 const auto *
N = *NodesInLastSCC.begin();
199 NodesInLastSCC.erase(
N);
202 assert(!
N->getFunction().isDeclaration());
207 for (
const auto &
E : *(*
N)) {
208 const auto *AdjNode = &
E.getNode();
209 assert(!AdjNode->isDead() && !AdjNode->getFunction().isDeclaration());
210 auto I = AllNodes.insert(AdjNode);
212 NodesInLastSCC.insert(AdjNode);
216 EdgeCount -= EdgesOfLastSeenNodes;
217 EdgesOfLastSeenNodes = 0;
221 assert(NodesInLastSCC.empty());
222 for (
const auto &
N : *LastSCC)
223 NodesInLastSCC.insert(&
N);
230 if (!LastSCC || ForceStop)
235 EdgesOfLastSeenNodes = 0;
238 for (
auto I = NodesInLastSCC.begin();
I != NodesInLastSCC.end();) {
240 NodesInLastSCC.erase(*
I++);
246 for (
const auto &
N : *LastSCC) {
248 auto I = NodesInLastSCC.insert(&
N);
252 assert(NodeCount >= NodesInLastSCC.size());
253 assert(EdgeCount >= EdgesOfLastSeenNodes);
266 bool CalleeWasDeleted) {
279 int64_t IRSizeAfter =
290 int64_t NewCallerAndCalleeEdges =
293 if (CalleeWasDeleted)
296 NewCallerAndCalleeEdges +=
299 assert(CurrentIRSize >= 0 && EdgeCount >= 0 && NodeCount >= 0);
302int64_t MLInlineAdvisor::getModuleIRSize()
const {
305 if (!
F.isDeclaration())
313 if (!InsertPair.second)
314 return InsertPair.first->second;
316 return InsertPair.first->second;
320 if (
auto Skip = getSkipAdviceIfUnreachableCallsite(CB))
349 <<
"Won't attempt inlining because module size grew too much.";
351 return std::make_unique<InlineAdvice>(
this, CB, ORE, Mandatory);
354 int CostEstimate = 0;
356 auto IsCallSiteInlinable =
358 if (!IsCallSiteInlinable) {
362 return std::make_unique<InlineAdvice>(
this, CB, ORE,
false);
364 CostEstimate = *IsCallSiteInlinable;
367 const auto CostFeatures =
370 return std::make_unique<InlineAdvice>(
this, CB, ORE,
false);
376 auto NrCtantParams = 0;
378 NrCtantParams += (isa<Constant>(*
I));
384 *
ModelRunner->getTensor<int64_t>(FeatureIndex::callee_basic_block_count) =
385 CalleeBefore.BasicBlockCount;
386 *
ModelRunner->getTensor<int64_t>(FeatureIndex::callsite_height) =
388 *
ModelRunner->getTensor<int64_t>(FeatureIndex::node_count) = NodeCount;
389 *
ModelRunner->getTensor<int64_t>(FeatureIndex::nr_ctant_params) =
391 *
ModelRunner->getTensor<int64_t>(FeatureIndex::edge_count) = EdgeCount;
392 *
ModelRunner->getTensor<int64_t>(FeatureIndex::caller_users) =
395 FeatureIndex::caller_conditionally_executed_blocks) =
396 CallerBefore.BlocksReachedFromConditionalInstruction;
397 *
ModelRunner->getTensor<int64_t>(FeatureIndex::caller_basic_block_count) =
398 CallerBefore.BasicBlockCount;
400 FeatureIndex::callee_conditionally_executed_blocks) =
401 CalleeBefore.BlocksReachedFromConditionalInstruction;
402 *
ModelRunner->getTensor<int64_t>(FeatureIndex::callee_users) =
404 *
ModelRunner->getTensor<int64_t>(FeatureIndex::cost_estimate) = CostEstimate;
419std::unique_ptr<MLInlineAdvice>
422 return std::make_unique<MLInlineAdvice>(
423 this, CB, ORE,
static_cast<bool>(
ModelRunner->evaluate<int64_t>()));
426std::unique_ptr<InlineAdvice>
427MLInlineAdvisor::getSkipAdviceIfUnreachableCallsite(
CallBase &CB) {
430 return std::make_unique<InlineAdvice>(
this, CB,
getCallerORE(CB),
false);
437 if (
auto Skip = getSkipAdviceIfUnreachableCallsite(CB))
439 if (Advice && !ForceStop)
446 return std::make_unique<InlineAdvice>(
this, CB,
getCallerORE(CB), Advice);
449std::unique_ptr<MLInlineAdvice>
451 return std::make_unique<MLInlineAdvice>(
this, CB,
getCallerORE(CB),
true);
455 OS <<
"[MLInlineAdvisor] Nodes: " << NodeCount <<
" Edges: " << EdgeCount
456 <<
" EdgesOfLastSeenNodes: " << EdgesOfLastSeenNodes <<
"\n";
457 OS <<
"[MLInlineAdvisor] FPI:\n";
458 for (
auto I : FPICache) {
459 OS <<
I.first->getName() <<
":\n";
470 CallerIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(*Caller)),
471 CalleeIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(*
Callee)),
472 CallerAndCalleeEdges(Advisor->isForcedToStop()
474 : (Advisor->getLocalCalls(*Caller) +
475 Advisor->getLocalCalls(*
Callee))),
476 PreInlineCallerFPI(Advisor->getCachedFPI(*Caller)) {
481void MLInlineAdvice::reportContextForRemark(
487 *getAdvisor()->getModelRunner().getTensor<int64_t>(
I));
498 reportContextForRemark(R);
508 reportContextForRemark(R);
520 reportContextForRemark(R);
528 reportContextForRemark(R);
amdgpu Simplify well known AMD library false FunctionCallee Callee
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
static Function * getFunction(Constant *C)
#define INLINE_COST_FEATURE_ITERATOR(M)
#define INLINE_FEATURE_ITERATOR(M)
Implements a lazy call graph analysis and related passes for the new pass manager.
static cl::opt< bool > KeepFPICache("ml-advisor-keep-fpi-cache", cl::Hidden, cl::desc("For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"), cl::init(false))
CallBase * getInlinableCS(Instruction &I)
static cl::opt< std::string > InteractiveChannelBaseName("inliner-interactive-channel-base", cl::Hidden, cl::desc("Base file path for the interactive mode. The incoming filename should " "have the name <inliner-interactive-channel-base>.in, while the " "outgoing name should be <inliner-interactive-channel-base>.out"))
#define POPULATE_NAMES(DTYPE, SHAPE, NAME, __)
static cl::opt< float > SizeIncreaseThreshold("ml-advisor-size-increase-threshold", cl::Hidden, cl::desc("Maximum factor by which expected native size may increase before " "blocking any further inlining."), cl::init(2.0))
static const std::string InclDefaultMsg
static cl::opt< bool > InteractiveIncludeDefault("inliner-interactive-include-default", cl::Hidden, cl::desc(InclDefaultMsg))
print must be executed print the must be executed context for all instructions
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A container for analyses that lazily runs them and caches their results.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Function * getCaller()
Helper to get the caller (the parent function).
The basic data container for the call graph of a Module of IR.
Common features for diagnostics dealing with optimization remarks that are used by both IR and MIR pa...
Analysis pass which computes a DominatorTree.
int64_t DirectCallsToDefinedFunctions
Number of direct calls made from this function to other functions defined in this module.
Capture state between an inlining decision having had been made, and its impact being observable.
Function *const Caller
Caller and Callee are pre-inlining.
const BasicBlock *const Block
OptimizationRemarkEmitter & ORE
InlineAdvisor *const Advisor
bool isInliningRecommended() const
Get the inlining recommendation.
Interface for deciding whether to inline a call site or not.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
FunctionAnalysisManager & FAM
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
InlineResult is basically true or false.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const BasicBlock * getParent() const
An analysis pass which computes the call graph for a module.
An SCC of the call graph.
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
Node * lookup(const Function &F) const
Lookup a function in the graph which has already been scanned and added.
Analysis pass that exposes the LoopInfo for a function.
InlineAdvice that tracks changes post inlining.
void updateCachedCallerFPI(FunctionAnalysisManager &FAM) const
const int64_t CallerIRSize
MLInlineAdvice(MLInlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool Recommendation)
const int64_t CalleeIRSize
void recordInliningImpl() override
Function * getCaller() const
const int64_t CallerAndCalleeEdges
void recordUnsuccessfulInliningImpl(const InlineResult &Result) override
Function * getCallee() const
void recordInliningWithCalleeDeletedImpl() override
void recordUnattemptedInliningImpl() override
std::unique_ptr< MLModelRunner > ModelRunner
FunctionPropertiesInfo & getCachedFPI(Function &) const
void onPassExit(LazyCallGraph::SCC *SCC) override
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM, std::unique_ptr< MLModelRunner > ModelRunner, std::function< bool(CallBase &)> GetDefaultAdvice)
void onSuccessfulInlining(const MLInlineAdvice &Advice, bool CalleeWasDeleted)
virtual std::unique_ptr< MLInlineAdvice > getMandatoryAdviceImpl(CallBase &CB)
void onPassEntry(LazyCallGraph::SCC *SCC) override
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
int64_t getLocalCalls(Function &F)
virtual std::unique_ptr< MLInlineAdvice > getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE)
int64_t getIRSize(Function &F) const
std::function< bool(CallBase &)> GetDefaultAdvice
std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB) override
std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice) override
unsigned getInitialFunctionLevel(const Function &F) const
A Module instance is used to store all the information related to an LLVM module.
A mock class satisfying the interface expected by ReleaseModeModelRunner for its TGen parameter.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void abandon()
Mark an analysis as abandoned.
Analysis pass providing the TargetTransformInfo.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
StringRef getName() const
Return a constant reference to the value's name.
This class implements an extremely fast bulk output stream that can only output to a stream.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
constexpr FeatureIndex inlineCostFeatureToMlFeature(InlineCostFeatureIndex Feature)
const char *const DefaultDecisionName
constexpr size_t NumberOfFeatures
std::optional< InlineCostFeatures > getInliningCostFeatures(CallBase &Call, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get the expanded cost features.
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
const TensorSpec DefaultDecisionSpec
const char *const DecisionName
const std::vector< TensorSpec > FeatureMap
const TensorSpec InlineDecisionSpec
const char *const RewardName
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
std::optional< int > getInliningCostEstimate(CallBase &Call, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get the cost estimate ignoring thresholds.