30#define DEBUG_TYPE "inline"
31#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
32#define LLVM_HAVE_TF_AOT
38STATISTIC(NumCallerCallersAnalyzed,
"Number of caller-callers analyzed");
44 cl::desc(
"Enable adding inline-remark attribute to"
45 " callsites processed by inliner but decided"
46 " to be not inlined"));
50 cl::desc(
"Enable deferred inlining"));
56 cl::desc(
"Scale to limit the cost of inline deferral"),
61 cl::desc(
"If true, annotate inline advisor remarks "
62 "with LTO and pass information."));
74 bool IsInliningMandatory)
81 if (IsInliningRecommended)
84 Remark <<
": always inline attribute";
89 if (IsInliningRecommended)
92 "NotInlined", DLoc, Block)
93 <<
"'" <<
NV(
"Callee",
Callee) <<
"' is not AlwaysInline into '"
94 <<
NV(
"Caller", Caller)
95 <<
"': " <<
NV(
"Reason", Result.getFailureReason());
100 assert(!IsInliningRecommended &&
"Expected to attempt inlining");
105void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
113 <<
"'" <<
NV(
"Callee",
Callee) <<
"' is not inlined into '"
115 <<
"': " <<
NV(
"Reason",
Result.getFailureReason());
119void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
126void DefaultInlineAdvice::recordInliningImpl() {
138 .getCachedResult<ProfileSummaryAnalysis>(
152 auto GetInlineCost = [&](
CallBase &CB) {
155 bool RemarksEnabled =
156 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
158 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
159 GetBFI, PSI, RemarksEnabled ? &ORE :
nullptr);
162 CB, GetInlineCost, ORE,
166std::unique_ptr<InlineAdvice>
167DefaultInlineAdvisor::getAdviceImpl(
CallBase &CB) {
169 return std::make_unique<DefaultInlineAdvice>(
176 bool IsInliningRecommended)
179 IsInliningRecommended(IsInliningRecommended) {}
181void InlineAdvice::recordInlineStatsIfNeeded() {
188 recordInlineStatsIfNeeded();
194 recordInlineStatsIfNeeded();
208 Advisor.reset(DA.Factory(M,
FAM, Params, IC));
219 std::move(Advisor), ReplaySettings,
224#ifdef LLVM_HAVE_TFLITE
229 return OIC.has_value();
251 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
274 TotalSecondaryCost = 0;
276 int CandidateCost = IC.
getCost() - 1;
279 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
281 bool InliningPreventsSomeOuterInline =
false;
282 unsigned NumCallerUsers = 0;
283 for (
User *U : Caller->users()) {
284 CallBase *CS2 = dyn_cast<CallBase>(U);
290 ApplyLastCallBonus =
false;
295 ++NumCallerCallersAnalyzed;
297 ApplyLastCallBonus =
false;
307 InliningPreventsSomeOuterInline =
true;
308 TotalSecondaryCost += IC2.
getCost();
313 if (!InliningPreventsSomeOuterInline)
320 if (ApplyLastCallBonus)
326 return TotalSecondaryCost < IC.
getCost();
328 int TotalCost = TotalSecondaryCost + IC.
getCost() * NumCallerUsers;
330 return TotalCost < Allowance;
338template <
class RemarkT>
342 R <<
"(cost=always)";
350 R <<
": " <<
ore::NV(
"Reason", Reason);
374std::optional<InlineCost>
387 <<
", Call: " << CB <<
"\n");
393 <<
", Call: " << CB <<
"\n");
397 <<
"'" <<
NV(
"Callee",
Callee) <<
"' not inlined into '"
398 <<
NV(
"Caller", Caller)
399 <<
"' because it should never be inlined " << IC;
404 <<
"'" <<
NV(
"Callee",
Callee) <<
"' not inlined into '"
405 <<
NV(
"Caller", Caller) <<
"' because too costly to inline "
413 int TotalSecondaryCost = 0;
414 if (EnableDeferral &&
418 <<
", outer Cost = " << TotalSecondaryCost <<
'\n');
422 <<
"Not inlining. Cost of inlining '" <<
NV(
"Callee",
Callee)
423 <<
"' increases the cost of inlining '" <<
NV(
"Caller", Caller)
424 <<
"' in other contexts";
440 for (
DILocation *DIL = DLoc.
get(); DIL; DIL = DIL->getInlinedAt()) {
442 CallSiteLoc <<
" @ ";
447 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
448 uint32_t Discriminator = DIL->getBaseDiscriminator();
449 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
451 Name = DIL->getScope()->getSubprogram()->getName();
452 CallSiteLoc <<
Name.str() <<
":" << llvm::utostr(
Offset);
453 if (
Format.outputColumn())
454 CallSiteLoc <<
":" << llvm::utostr(DIL->getColumn());
455 if (
Format.outputDiscriminator() && Discriminator)
456 CallSiteLoc <<
"." << llvm::utostr(Discriminator);
460 return CallSiteLoc.
str();
469 Remark <<
" at callsite ";
470 for (
DILocation *DIL = DLoc.
get(); DIL; DIL = DIL->getInlinedAt()) {
473 unsigned int Offset = DIL->getLine();
474 Offset -= DIL->getScope()->getSubprogram()->getLine();
475 unsigned int Discriminator = DIL->getBaseDiscriminator();
476 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
478 Name = DIL->getScope()->getSubprogram()->getName();
480 <<
ore::NV(
"Column", DIL->getColumn());
495 StringRef RemarkName = AlwaysInline ?
"AlwaysInline" :
"Inlined";
499 <<
ore::NV(
"Caller", &Caller) <<
"'";
510 bool ForProfileContext,
const char *
PassName) {
514 if (ForProfileContext)
515 Remark <<
" to match profiling context";
516 Remark <<
" with " << IC;
522 std::optional<InlineContext> IC)
529 std::make_unique<ImportedFunctionsInliningStatistics>();
544 return std::make_unique<MandatoryInlineAdvice>(
this, CB,
getCallerORE(CB),
565 return "always-inline";
567 return "cgscc-inline";
569 return "early-inline";
573 return "module-inline";
575 return "replay-cgscc-inline";
577 return "replay-sample-profile-inline";
579 return "sample-profile-inline";
601 auto TrivialDecision =
604 if (TrivialDecision) {
605 if (TrivialDecision->isSuccess())
614 bool MandatoryOnly) {
631 OS <<
"No Inline Advisor\n";
633 IA->getAdvisor()->print(OS);
640 const auto &MAMProxy =
643 if (InitialC.
size() == 0) {
644 OS <<
"SCC is empty!\n";
647 Module &M = *InitialC.
begin()->getFunction().getParent();
650 OS <<
"No Inline Advisor\n";
652 IA->getAdvisor()->print(OS);
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static const Function * getParent(const Value *V)
static cl::opt< int > InlineDeferralScale("inline-deferral-scale", cl::desc("Scale to limit the cost of inline deferral"), cl::init(2), cl::Hidden)
static cl::opt< bool > InlineRemarkAttribute("inline-remark-attribute", cl::init(false), cl::Hidden, cl::desc("Enable adding inline-remark attribute to" " callsites processed by inliner but decided" " to be not inlined"))
Flag to add inline messages as callsite attributes 'inline-remark'.
static cl::opt< bool > EnableInlineDeferral("inline-deferral", cl::init(false), cl::Hidden, cl::desc("Enable deferred inlining"))
static cl::opt< bool > AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), cl::desc("If true, annotate inline advisor remarks " "with LTO and pass information."))
static const char * getLTOPhase(ThinOrFullLTOPhase LTOPhase)
static std::optional< llvm::InlineCost > getDefaultInlineAdvice(CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params)
static const char * getInlineAdvisorContext(InlinePass IP)
static bool shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, function_ref< InlineCost(CallBase &CB)> GetInlineCost)
Return true if inlining of CB can block the caller from being inlined which is proved to be more bene...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static const char PassName[]
A container for analyses that lazily runs them and caches their results.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Function * getCaller()
Helper to get the caller (the parent function).
DILocation * get() const
Get the underlying DILocation.
The default (manual heuristics) implementation of the InlineAdvisor.
Module * getParent()
Get the module that this global value is contained inside of...
Capture state between an inlining decision having had been made, and its impact being observable.
Function *const Caller
Caller and Callee are pre-inlining.
virtual void recordInliningWithCalleeDeletedImpl()
virtual void recordInliningImpl()
const BasicBlock *const Block
void recordInlining()
Exactly one of the record* APIs must be called.
OptimizationRemarkEmitter & ORE
virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result)
InlineAdvisor *const Advisor
InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool IsInliningRecommended)
virtual void recordUnattemptedInliningImpl()
void recordInliningWithCalleeDeleted()
Call after inlining succeeded, and results in the callee being delete-able, meaning,...
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
FunctionAnalysisManager & FAM
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
virtual std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice)
std::unique_ptr< ImportedFunctionsInliningStatistics > ImportedFunctionsStats
const char * getAnnotatedInlinePassName() const
NOTE pass name is annotated only when inline advisor constructor provides InlineContext.
InlineAdvisor(InlineAdvisor &&)=delete
virtual std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB)=0
std::unique_ptr< InlineAdvice > getAdvice(CallBase &CB, bool MandatoryOnly=false)
Get an InlineAdvice containing a recommendation on whether to inline or not.
Represents the cost of inlining a function.
int getThreshold() const
Get the threshold against which the cost was computed.
const char * getReason() const
Get the reason of Always or Never.
int getCost() const
Get the inline cost estimate.
int getCostDelta() const
Get the cost delta from the threshold for inlining.
InlineResult is basically true or false.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const BasicBlock * getParent() const
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
A Module instance is used to store all the information related to an LLVM module.
LLVMContext & getContext() const
Get the global data context.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Used for dynamically registering InlineAdvisors as plugins.
static bool HasBeenRegistered
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Analysis providing profile information.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
LLVMContext & getContext() const
All values hold a context through their type.
An efficient, type-erasing, non-owning reference to a callable.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
std::string & str()
Returns the string's reference.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const int LastCallToStaticBonus
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
@ ReplaySampleProfileInliner
void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, bool IsMandatory, function_ref< void(OptimizationRemark &)> ExtraContext={}, const char *PassName=nullptr)
Emit ORE message.
std::string inlineCostStr(const InlineCost &IC)
Utility for extracting the inline cost message to a string.
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
std::string AnnotateInlinePassName(InlineContext IC)
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
@ FullLTOPreLink
Full LTO prelink phase.
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
@ None
No LTO/ThinLTO behavior needed.
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
std::optional< InlineResult > getAttributeBasedInliningDecision(CallBase &Call, Function *Callee, TargetTransformInfo &CalleeTTI, function_ref< const TargetLibraryInfo &(Function &)> GetTLI)
Returns InlineResult::success() if the call site should be always inlined because of user directives,...
cl::opt< InlinerFunctionImportStatsOpts > InlinerFunctionImportStats("inliner-function-import-stats", cl::init(InlinerFunctionImportStatsOpts::No), cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic", "basic statistics"), clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose", "printing of statistics for each inlined function")), cl::Hidden, cl::desc("Enable inliner stats for imported functions"))
std::optional< InlineCost > shouldInline(CallBase &CB, function_ref< InlineCost(CallBase &CB)> GetInlineCost, OptimizationRemarkEmitter &ORE, bool EnableDeferral=true)
Return the cost only if the inliner should attempt to inline at the given CallSite.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
std::unique_ptr< InlineAdvisor > getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM)
void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc)
Add location info to ORE message.
std::string formatCallSiteLocation(DebugLoc DLoc, const CallSiteFormat &Format)
Get call site location as a string with the given format.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Used in the streaming interface as the general argument type.
bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, const ReplayInlinerSettings &ReplaySettings, InlineContext IC)
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
ThinOrFullLTOPhase LTOPhase
Thresholds to tune inline cost analysis.
std::optional< bool > EnableDeferral
Indicate whether we should allow inline deferral.