36 "inliner-interactive-channel-base",
cl::Hidden,
38 "Base file path for the interactive mode. The incoming filename should "
39 "have the name <inliner-interactive-channel-base>.in, while the "
40 "outgoing name should be <inliner-interactive-channel-base>.out"));
42#if defined(LLVM_HAVE_TF_AOT_INLINERSIZEMODEL)
44#include "InlinerSizeModel.h"
50std::unique_ptr<InlineAdvisor>
52 if (!llvm::isEmbeddedModelEvaluatorValid<CompiledModelType>() &&
55 std::unique_ptr<MLModelRunner> AOTRunner;
57 AOTRunner = std::make_unique<ReleaseModeModelRunner<CompiledModelType>>(
60 AOTRunner = std::make_unique<InteractiveModelRunner>(
64 return std::make_unique<MLInlineAdvisor>(M,
MAM, std::move(AOTRunner));
67#define DEBUG_TYPE "inline-ml"
70 "ml-advisor-size-increase-threshold",
cl::Hidden,
71 cl::desc(
"Maximum factor by which expected native size may increase before "
72 "blocking any further inlining."),
78 "For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"),
83#define POPULATE_NAMES(_, NAME) TensorSpec::createSpec<int64_t>(NAME, {1} ),
89#define POPULATE_NAMES(_, NAME, __) TensorSpec::createSpec<int64_t>(NAME, {1} ),
102 if (
auto *CS = dyn_cast<CallBase>(&
I))
104 if (!
Callee->isDeclaration()) {
112 std::unique_ptr<MLModelRunner> Runner)
115 ModelRunner(
std::
move(Runner)),
117 InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize) {
128 const std::vector<CallGraphNode *> &CGNodes = *
I;
130 for (
auto *CGNode : CGNodes) {
132 if (!
F ||
F->isDeclaration())
136 auto *Called = CS->getCalledFunction();
137 auto Pos = FunctionLevels.find(&CG.
get(*Called));
141 if (Pos == FunctionLevels.end())
143 Level = std::max(Level, Pos->second + 1);
147 for (
auto *CGNode : CGNodes) {
149 if (
F && !
F->isDeclaration())
150 FunctionLevels[&CG.
get(*
F)] = Level;
153 for (
auto KVP : FunctionLevels) {
154 AllNodes.insert(KVP.first);
157 NodeCount = AllNodes.size();
165 if (!LastSCC || ForceStop)
181 NodeCount -=
static_cast<int64_t
>(NodesInLastSCC.size());
182 while (!NodesInLastSCC.empty()) {
183 const auto *
N = *NodesInLastSCC.begin();
184 NodesInLastSCC.erase(
N);
187 assert(!
N->getFunction().isDeclaration());
192 for (
const auto &
E : *(*
N)) {
193 const auto *AdjNode = &
E.getNode();
194 assert(!AdjNode->isDead() && !AdjNode->getFunction().isDeclaration());
195 auto I = AllNodes.insert(AdjNode);
197 NodesInLastSCC.insert(AdjNode);
201 EdgeCount -= EdgesOfLastSeenNodes;
202 EdgesOfLastSeenNodes = 0;
206 assert(NodesInLastSCC.empty());
207 for (
const auto &
N : *LastSCC)
208 NodesInLastSCC.insert(&
N);
215 if (!LastSCC || ForceStop)
220 EdgesOfLastSeenNodes = 0;
223 for (
auto I = NodesInLastSCC.begin();
I != NodesInLastSCC.end();) {
225 NodesInLastSCC.erase(*
I++);
231 for (
const auto &
N : *LastSCC) {
233 auto I = NodesInLastSCC.insert(&
N);
237 assert(NodeCount >= NodesInLastSCC.size());
238 assert(EdgeCount >= EdgesOfLastSeenNodes);
251 bool CalleeWasDeleted) {
264 int64_t IRSizeAfter =
275 int64_t NewCallerAndCalleeEdges =
278 if (CalleeWasDeleted)
281 NewCallerAndCalleeEdges +=
284 assert(CurrentIRSize >= 0 && EdgeCount >= 0 && NodeCount >= 0);
287int64_t MLInlineAdvisor::getModuleIRSize()
const {
290 if (!
F.isDeclaration())
298 if (!InsertPair.second)
299 return InsertPair.first->second;
301 return InsertPair.first->second;
305 if (
auto Skip = getSkipAdviceIfUnreachableCallsite(CB))
334 <<
"Won't attempt inlining because module size grew too much.";
336 return std::make_unique<InlineAdvice>(
this, CB, ORE, Mandatory);
339 int CostEstimate = 0;
341 auto IsCallSiteInlinable =
343 if (!IsCallSiteInlinable) {
347 return std::make_unique<InlineAdvice>(
this, CB, ORE,
false);
349 CostEstimate = *IsCallSiteInlinable;
352 const auto CostFeatures =
355 return std::make_unique<InlineAdvice>(
this, CB, ORE,
false);
361 auto NrCtantParams = 0;
363 NrCtantParams += (isa<Constant>(*
I));
369 *
ModelRunner->getTensor<int64_t>(FeatureIndex::CalleeBasicBlockCount) =
370 CalleeBefore.BasicBlockCount;
371 *
ModelRunner->getTensor<int64_t>(FeatureIndex::CallSiteHeight) =
373 *
ModelRunner->getTensor<int64_t>(FeatureIndex::NodeCount) = NodeCount;
374 *
ModelRunner->getTensor<int64_t>(FeatureIndex::NrCtantParams) = NrCtantParams;
375 *
ModelRunner->getTensor<int64_t>(FeatureIndex::EdgeCount) = EdgeCount;
376 *
ModelRunner->getTensor<int64_t>(FeatureIndex::CallerUsers) =
379 FeatureIndex::CallerConditionallyExecutedBlocks) =
380 CallerBefore.BlocksReachedFromConditionalInstruction;
381 *
ModelRunner->getTensor<int64_t>(FeatureIndex::CallerBasicBlockCount) =
382 CallerBefore.BasicBlockCount;
384 FeatureIndex::CalleeConditionallyExecutedBlocks) =
385 CalleeBefore.BlocksReachedFromConditionalInstruction;
386 *
ModelRunner->getTensor<int64_t>(FeatureIndex::CalleeUsers) =
388 *
ModelRunner->getTensor<int64_t>(FeatureIndex::CostEstimate) = CostEstimate;
400std::unique_ptr<MLInlineAdvice>
403 return std::make_unique<MLInlineAdvice>(
404 this, CB, ORE,
static_cast<bool>(
ModelRunner->evaluate<int64_t>()));
407std::unique_ptr<InlineAdvice>
408MLInlineAdvisor::getSkipAdviceIfUnreachableCallsite(
CallBase &CB) {
411 return std::make_unique<InlineAdvice>(
this, CB,
getCallerORE(CB),
false);
418 if (
auto Skip = getSkipAdviceIfUnreachableCallsite(CB))
420 if (Advice && !ForceStop)
427 return std::make_unique<InlineAdvice>(
this, CB,
getCallerORE(CB), Advice);
430std::unique_ptr<MLInlineAdvice>
432 return std::make_unique<MLInlineAdvice>(
this, CB,
getCallerORE(CB),
true);
435void MLInlineAdvisor::print(
raw_ostream &OS)
const {
436 OS <<
"[MLInlineAdvisor] Nodes: " << NodeCount <<
" Edges: " << EdgeCount
437 <<
" EdgesOfLastSeenNodes: " << EdgesOfLastSeenNodes <<
"\n";
438 OS <<
"[MLInlineAdvisor] FPI:\n";
439 for (
auto I : FPICache) {
440 OS <<
I.first->getName() <<
":\n";
451 CallerIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(*Caller)),
452 CalleeIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(*
Callee)),
453 CallerAndCalleeEdges(Advisor->isForcedToStop()
455 : (Advisor->getLocalCalls(*Caller) +
456 Advisor->getLocalCalls(*
Callee))),
457 PreInlineCallerFPI(Advisor->getCachedFPI(*Caller)) {
462void MLInlineAdvice::reportContextForRemark(
468 *getAdvisor()->getModelRunner().getTensor<int64_t>(
I));
479 reportContextForRemark(R);
489 reportContextForRemark(R);
501 reportContextForRemark(R);
509 reportContextForRemark(R);
amdgpu Simplify well known AMD library false FunctionCallee Callee
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
static Function * getFunction(Constant *C)
#define INLINE_COST_FEATURE_ITERATOR(M)
#define INLINE_FEATURE_ITERATOR(M)
Implements a lazy call graph analysis and related passes for the new pass manager.
#define POPULATE_NAMES(_, NAME)
static cl::opt< bool > KeepFPICache("ml-advisor-keep-fpi-cache", cl::Hidden, cl::desc("For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"), cl::init(false))
CallBase * getInlinableCS(Instruction &I)
static cl::opt< std::string > InteractiveChannelBaseName("inliner-interactive-channel-base", cl::Hidden, cl::desc("Base file path for the interactive mode. The incoming filename should " "have the name <inliner-interactive-channel-base>.in, while the " "outgoing name should be <inliner-interactive-channel-base>.out"))
static cl::opt< float > SizeIncreaseThreshold("ml-advisor-size-increase-threshold", cl::Hidden, cl::desc("Maximum factor by which expected native size may increase before " "blocking any further inlining."), cl::init(2.0))
print must be executed print the must be executed context for all instructions
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A container for analyses that lazily runs them and caches their results.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Function * getCaller()
Helper to get the caller (the parent function).
The basic data container for the call graph of a Module of IR.
Common features for diagnostics dealing with optimization remarks that are used by both IR and MIR pa...
Analysis pass which computes a DominatorTree.
int64_t DirectCallsToDefinedFunctions
Number of direct calls made from this function to other functions defined in this module.
Capture state between an inlining decision having had been made, and its impact being observable.
Function *const Caller
Caller and Callee are pre-inlining.
const BasicBlock *const Block
OptimizationRemarkEmitter & ORE
InlineAdvisor *const Advisor
bool isInliningRecommended() const
Get the inlining recommendation.
Interface for deciding whether to inline a call site or not.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
FunctionAnalysisManager & FAM
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
InlineResult is basically true or false.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const BasicBlock * getParent() const
An analysis pass which computes the call graph for a module.
An SCC of the call graph.
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
Node * lookup(const Function &F) const
Lookup a function in the graph which has already been scanned and added.
Analysis pass that exposes the LoopInfo for a function.
InlineAdvice that tracks changes post inlining.
void updateCachedCallerFPI(FunctionAnalysisManager &FAM) const
const int64_t CallerIRSize
MLInlineAdvice(MLInlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool Recommendation)
const int64_t CalleeIRSize
void recordInliningImpl() override
Function * getCaller() const
const int64_t CallerAndCalleeEdges
void recordUnsuccessfulInliningImpl(const InlineResult &Result) override
Function * getCallee() const
void recordInliningWithCalleeDeletedImpl() override
void recordUnattemptedInliningImpl() override
MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM, std::unique_ptr< MLModelRunner > ModelRunner)
std::unique_ptr< MLModelRunner > ModelRunner
FunctionPropertiesInfo & getCachedFPI(Function &) const
void onPassExit(LazyCallGraph::SCC *SCC) override
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
void onSuccessfulInlining(const MLInlineAdvice &Advice, bool CalleeWasDeleted)
virtual std::unique_ptr< MLInlineAdvice > getMandatoryAdviceImpl(CallBase &CB)
void onPassEntry(LazyCallGraph::SCC *SCC) override
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
int64_t getLocalCalls(Function &F)
virtual std::unique_ptr< MLInlineAdvice > getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE)
int64_t getIRSize(Function &F) const
std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB) override
std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice) override
unsigned getInitialFunctionLevel(const Function &F) const
A Module instance is used to store all the information related to an LLVM module.
A mock class satisfying the interface expected by ReleaseModeModelRunner for its TGen parameter.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void abandon()
Mark an analysis as abandoned.
Analysis pass providing the TargetTransformInfo.
StringRef getName() const
Return a constant reference to the value's name.
This class implements an extremely fast bulk output stream that can only output to a stream.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
constexpr FeatureIndex inlineCostFeatureToMlFeature(InlineCostFeatureIndex Feature)
const char *const DefaultDecisionName
constexpr size_t NumberOfFeatures
std::optional< InlineCostFeatures > getInliningCostFeatures(CallBase &Call, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get the expanded cost features.
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
const char *const DecisionName
const std::vector< TensorSpec > FeatureMap
const TensorSpec InlineDecisionSpec
const char *const RewardName
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM)
std::optional< int > getInliningCostEstimate(CallBase &Call, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get the cost estimate ignoring thresholds.