36 "inliner-interactive-channel-base",
cl::Hidden,
38 "Base file path for the interactive mode. The incoming filename should "
39 "have the name <inliner-interactive-channel-base>.in, while the "
40 "outgoing name should be <inliner-interactive-channel-base>.out"));
42 (
Twine(
"In interactive mode, also send the default policy decision: ") +
49#if defined(LLVM_HAVE_TF_AOT_INLINERSIZEMODEL)
51#include "InlinerSizeModel.h"
57std::unique_ptr<InlineAdvisor>
59 std::function<
bool(
CallBase &)> GetDefaultAdvice) {
60 if (!llvm::isEmbeddedModelEvaluatorValid<CompiledModelType>() &&
63 std::unique_ptr<MLModelRunner> AOTRunner;
65 AOTRunner = std::make_unique<ReleaseModeModelRunner<CompiledModelType>>(
71 AOTRunner = std::make_unique<InteractiveModelRunner>(
76 return std::make_unique<MLInlineAdvisor>(M,
MAM, std::move(AOTRunner),
80#define DEBUG_TYPE "inline-ml"
83 "ml-advisor-size-increase-threshold",
cl::Hidden,
84 cl::desc(
"Maximum factor by which expected native size may increase before "
85 "blocking any further inlining."),
91 "For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"),
96#define POPULATE_NAMES(DTYPE, SHAPE, NAME, __) TensorSpec::createSpec<DTYPE>(#NAME, SHAPE),
115 if (
auto *CS = dyn_cast<CallBase>(&
I))
116 if (
Function *Callee = CS->getCalledFunction()) {
117 if (!Callee->isDeclaration()) {
126 std::unique_ptr<MLModelRunner> Runner,
127 std::function<
bool(
CallBase &)> GetDefaultAdvice)
130 ModelRunner(
std::
move(Runner)), GetDefaultAdvice(GetDefaultAdvice),
132 InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize) {
143 const std::vector<CallGraphNode *> &CGNodes = *
I;
145 for (
auto *CGNode : CGNodes) {
147 if (!
F ||
F->isDeclaration())
151 auto *Called = CS->getCalledFunction();
152 auto Pos = FunctionLevels.find(&CG.
get(*Called));
156 if (Pos == FunctionLevels.end())
158 Level = std::max(Level, Pos->second + 1);
162 for (
auto *CGNode : CGNodes) {
164 if (
F && !
F->isDeclaration())
165 FunctionLevels[&CG.
get(*
F)] = Level;
168 for (
auto KVP : FunctionLevels) {
169 AllNodes.insert(KVP.first);
172 NodeCount = AllNodes.size();
180 if (!LastSCC || ForceStop)
198 NodeCount -=
static_cast<int64_t
>(NodesInLastSCC.size());
199 while (!NodesInLastSCC.empty()) {
200 const auto *
N = *NodesInLastSCC.begin();
201 NodesInLastSCC.erase(
N);
204 assert(!
N->getFunction().isDeclaration());
209 const auto NLevel = FunctionLevels.at(
N);
210 for (
const auto &
E : *(*
N)) {
211 const auto *AdjNode = &
E.getNode();
212 assert(!AdjNode->isDead() && !AdjNode->getFunction().isDeclaration());
213 auto I = AllNodes.insert(AdjNode);
215 NodesInLastSCC.insert(AdjNode);
216 FunctionLevels[AdjNode] = NLevel;
221 EdgeCount -= EdgesOfLastSeenNodes;
222 EdgesOfLastSeenNodes = 0;
226 assert(NodesInLastSCC.empty());
227 for (
const auto &
N : *LastSCC)
228 NodesInLastSCC.insert(&
N);
235 if (!LastSCC || ForceStop)
240 EdgesOfLastSeenNodes = 0;
243 for (
auto I = NodesInLastSCC.begin();
I != NodesInLastSCC.end();) {
245 NodesInLastSCC.erase(*
I++);
251 for (
const auto &
N : *LastSCC) {
253 auto I = NodesInLastSCC.insert(&
N);
257 assert(NodeCount >= NodesInLastSCC.size());
258 assert(EdgeCount >= EdgesOfLastSeenNodes);
271 bool CalleeWasDeleted) {
284 int64_t IRSizeAfter =
295 int64_t NewCallerAndCalleeEdges =
298 if (CalleeWasDeleted)
301 NewCallerAndCalleeEdges +=
304 assert(CurrentIRSize >= 0 && EdgeCount >= 0 && NodeCount >= 0);
307int64_t MLInlineAdvisor::getModuleIRSize()
const {
310 if (!
F.isDeclaration())
318 if (!InsertPair.second)
319 return InsertPair.first->second;
321 return InsertPair.first->second;
325 if (
auto Skip = getSkipAdviceIfUnreachableCallsite(CB))
354 <<
"Won't attempt inlining because module size grew too much.";
356 return std::make_unique<InlineAdvice>(
this, CB, ORE, Mandatory);
359 int CostEstimate = 0;
361 auto IsCallSiteInlinable =
363 if (!IsCallSiteInlinable) {
367 return std::make_unique<InlineAdvice>(
this, CB, ORE,
false);
369 CostEstimate = *IsCallSiteInlinable;
372 const auto CostFeatures =
375 return std::make_unique<InlineAdvice>(
this, CB, ORE,
false);
381 auto NrCtantParams = 0;
383 NrCtantParams += (isa<Constant>(*
I));
389 *
ModelRunner->getTensor<int64_t>(FeatureIndex::callee_basic_block_count) =
390 CalleeBefore.BasicBlockCount;
391 *
ModelRunner->getTensor<int64_t>(FeatureIndex::callsite_height) =
393 *
ModelRunner->getTensor<int64_t>(FeatureIndex::node_count) = NodeCount;
394 *
ModelRunner->getTensor<int64_t>(FeatureIndex::nr_ctant_params) =
396 *
ModelRunner->getTensor<int64_t>(FeatureIndex::edge_count) = EdgeCount;
397 *
ModelRunner->getTensor<int64_t>(FeatureIndex::caller_users) =
400 FeatureIndex::caller_conditionally_executed_blocks) =
401 CallerBefore.BlocksReachedFromConditionalInstruction;
402 *
ModelRunner->getTensor<int64_t>(FeatureIndex::caller_basic_block_count) =
403 CallerBefore.BasicBlockCount;
405 FeatureIndex::callee_conditionally_executed_blocks) =
406 CalleeBefore.BlocksReachedFromConditionalInstruction;
407 *
ModelRunner->getTensor<int64_t>(FeatureIndex::callee_users) =
409 *
ModelRunner->getTensor<int64_t>(FeatureIndex::cost_estimate) = CostEstimate;
424std::unique_ptr<MLInlineAdvice>
427 return std::make_unique<MLInlineAdvice>(
428 this, CB, ORE,
static_cast<bool>(
ModelRunner->evaluate<int64_t>()));
431std::unique_ptr<InlineAdvice>
432MLInlineAdvisor::getSkipAdviceIfUnreachableCallsite(
CallBase &CB) {
435 return std::make_unique<InlineAdvice>(
this, CB,
getCallerORE(CB),
false);
442 if (
auto Skip = getSkipAdviceIfUnreachableCallsite(CB))
444 if (Advice && !ForceStop)
451 return std::make_unique<InlineAdvice>(
this, CB,
getCallerORE(CB), Advice);
454std::unique_ptr<MLInlineAdvice>
456 return std::make_unique<MLInlineAdvice>(
this, CB,
getCallerORE(CB),
true);
460 OS <<
"[MLInlineAdvisor] Nodes: " << NodeCount <<
" Edges: " << EdgeCount
461 <<
" EdgesOfLastSeenNodes: " << EdgesOfLastSeenNodes <<
"\n";
462 OS <<
"[MLInlineAdvisor] FPI:\n";
463 for (
auto I : FPICache) {
464 OS <<
I.first->getName() <<
":\n";
469 OS <<
"[MLInlineAdvisor] FuncLevels:\n";
470 for (
auto I : FunctionLevels)
471 OS << (
I.first->isDead() ?
"<deleted>" :
I.first->getFunction().getName())
472 <<
" : " <<
I.second <<
"\n";
481 CallerIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(*Caller)),
482 CalleeIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(*Callee)),
483 CallerAndCalleeEdges(Advisor->isForcedToStop()
485 : (Advisor->getLocalCalls(*Caller) +
486 Advisor->getLocalCalls(*Callee))),
487 PreInlineCallerFPI(Advisor->getCachedFPI(*Caller)) {
492void MLInlineAdvice::reportContextForRemark(
498 *getAdvisor()->getModelRunner().getTensor<int64_t>(
I));
509 reportContextForRemark(R);
519 reportContextForRemark(R);
531 reportContextForRemark(R);
539 reportContextForRemark(R);
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
static Function * getFunction(Constant *C)
#define INLINE_COST_FEATURE_ITERATOR(M)
#define INLINE_FEATURE_ITERATOR(M)
Select target instructions out of generic instructions
Implements a lazy call graph analysis and related passes for the new pass manager.
static cl::opt< bool > KeepFPICache("ml-advisor-keep-fpi-cache", cl::Hidden, cl::desc("For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"), cl::init(false))
CallBase * getInlinableCS(Instruction &I)
static cl::opt< std::string > InteractiveChannelBaseName("inliner-interactive-channel-base", cl::Hidden, cl::desc("Base file path for the interactive mode. The incoming filename should " "have the name <inliner-interactive-channel-base>.in, while the " "outgoing name should be <inliner-interactive-channel-base>.out"))
#define POPULATE_NAMES(DTYPE, SHAPE, NAME, __)
static cl::opt< float > SizeIncreaseThreshold("ml-advisor-size-increase-threshold", cl::Hidden, cl::desc("Maximum factor by which expected native size may increase before " "blocking any further inlining."), cl::init(2.0))
static const std::string InclDefaultMsg
static cl::opt< bool > InteractiveIncludeDefault("inliner-interactive-include-default", cl::Hidden, cl::desc(InclDefaultMsg))
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A container for analyses that lazily runs them and caches their results.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Function * getCaller()
Helper to get the caller (the parent function).
The basic data container for the call graph of a Module of IR.
Common features for diagnostics dealing with optimization remarks that are used by both IR and MIR pa...
Analysis pass which computes a DominatorTree.
int64_t DirectCallsToDefinedFunctions
Number of direct calls made from this function to other functions defined in this module.
Capture state between an inlining decision having had been made, and its impact being observable.
Function *const Caller
Caller and Callee are pre-inlining.
const BasicBlock *const Block
OptimizationRemarkEmitter & ORE
InlineAdvisor *const Advisor
bool isInliningRecommended() const
Get the inlining recommendation.
Interface for deciding whether to inline a call site or not.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
FunctionAnalysisManager & FAM
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
InlineResult is basically true or false.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const BasicBlock * getParent() const
An analysis pass which computes the call graph for a module.
An SCC of the call graph.
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
Node * lookup(const Function &F) const
Lookup a function in the graph which has already been scanned and added.
Analysis pass that exposes the LoopInfo for a function.
InlineAdvice that tracks changes post inlining.
void updateCachedCallerFPI(FunctionAnalysisManager &FAM) const
const int64_t CallerIRSize
MLInlineAdvice(MLInlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool Recommendation)
const int64_t CalleeIRSize
void recordInliningImpl() override
Function * getCaller() const
const int64_t CallerAndCalleeEdges
void recordUnsuccessfulInliningImpl(const InlineResult &Result) override
Function * getCallee() const
void recordInliningWithCalleeDeletedImpl() override
void recordUnattemptedInliningImpl() override
std::unique_ptr< MLModelRunner > ModelRunner
FunctionPropertiesInfo & getCachedFPI(Function &) const
void onPassExit(LazyCallGraph::SCC *SCC) override
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM, std::unique_ptr< MLModelRunner > ModelRunner, std::function< bool(CallBase &)> GetDefaultAdvice)
void onSuccessfulInlining(const MLInlineAdvice &Advice, bool CalleeWasDeleted)
virtual std::unique_ptr< MLInlineAdvice > getMandatoryAdviceImpl(CallBase &CB)
void onPassEntry(LazyCallGraph::SCC *SCC) override
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
int64_t getLocalCalls(Function &F)
virtual std::unique_ptr< MLInlineAdvice > getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE)
int64_t getIRSize(Function &F) const
std::function< bool(CallBase &)> GetDefaultAdvice
std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB) override
std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice) override
unsigned getInitialFunctionLevel(const Function &F) const
A Module instance is used to store all the information related to an LLVM module.
A mock class satisfying the interface expected by ReleaseModeModelRunner for its TGen parameter.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void abandon()
Mark an analysis as abandoned.
Analysis pass providing the TargetTransformInfo.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
StringRef getName() const
Return a constant reference to the value's name.
This class implements an extremely fast bulk output stream that can only output to a stream.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
constexpr FeatureIndex inlineCostFeatureToMlFeature(InlineCostFeatureIndex Feature)
const char *const DefaultDecisionName
constexpr size_t NumberOfFeatures
std::optional< InlineCostFeatures > getInliningCostFeatures(CallBase &Call, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get the expanded cost features.
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
const TensorSpec DefaultDecisionSpec
const char *const DecisionName
const std::vector< TensorSpec > FeatureMap
const TensorSpec InlineDecisionSpec
const char *const RewardName
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
std::optional< int > getInliningCostEstimate(CallBase &Call, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get the cost estimate ignoring thresholds.
Implement std::hash so that hash_code can be used in STL containers.