Go to the documentation of this file.
30 #define DEBUG_TYPE "inline"
31 #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
32 #define LLVM_HAVE_TF_AOT
38 STATISTIC(NumCallerCallersAnalyzed,
"Number of caller-callers analyzed");
44 cl::desc(
"Enable adding inline-remark attribute to"
45 " callsites processed by inliner but decided"
46 " to be not inlined"));
50 cl::desc(
"Enable deferred inlining"));
56 cl::desc(
"Scale to limit the cost of inline deferral"),
61 cl::desc(
"If true, annotate inline advisor remarks "
62 "with LTO and pass information."));
72 bool IsInliningMandatory)
76 void recordInliningWithCalleeDeletedImpl()
override { recordInliningImpl(); }
78 void recordInliningImpl()
override {
79 if (IsInliningRecommended)
80 emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended,
82 Remark <<
": always inline attribute";
86 void recordUnsuccessfulInliningImpl(
const InlineResult &Result)
override {
87 if (IsInliningRecommended)
90 "NotInlined", DLoc, Block)
91 <<
"'" <<
NV(
"Callee", Callee) <<
"' is not AlwaysInline into '"
92 <<
NV(
"Caller", Caller)
93 <<
"': " <<
NV(
"Reason",
Result.getFailureReason());
97 void recordUnattemptedInliningImpl()
override {
98 assert(!IsInliningRecommended &&
"Expected to attempt inlining");
103 void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
111 <<
"'" <<
NV(
"Callee",
Callee) <<
"' is not inlined into '"
113 <<
"': " <<
NV(
"Reason",
Result.getFailureReason());
117 void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
124 void DefaultInlineAdvice::recordInliningImpl() {
136 .getCachedResult<ProfileSummaryAnalysis>(
150 auto GetInlineCost = [&](
CallBase &CB) {
153 bool RemarksEnabled =
156 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
157 GetBFI, PSI, RemarksEnabled ? &
ORE :
nullptr);
160 CB, GetInlineCost,
ORE,
164 std::unique_ptr<InlineAdvice>
165 DefaultInlineAdvisor::getAdviceImpl(
CallBase &CB) {
167 return std::make_unique<DefaultInlineAdvice>(
179 void InlineAdvice::recordInlineStatsIfNeeded() {
186 recordInlineStatsIfNeeded();
192 recordInlineStatsIfNeeded();
215 #ifdef LLVM_HAVE_TF_API
220 return OIC.hasValue();
225 #ifdef LLVM_HAVE_TF_AOT
244 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
267 TotalSecondaryCost = 0;
269 int CandidateCost = IC.
getCost() - 1;
272 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
274 bool InliningPreventsSomeOuterInline =
false;
275 unsigned NumCallerUsers = 0;
276 for (
User *U : Caller->users()) {
277 CallBase *CS2 = dyn_cast<CallBase>(U);
283 ApplyLastCallBonus =
false;
288 ++NumCallerCallersAnalyzed;
290 ApplyLastCallBonus =
false;
300 InliningPreventsSomeOuterInline =
true;
301 TotalSecondaryCost += IC2.
getCost();
306 if (!InliningPreventsSomeOuterInline)
313 if (ApplyLastCallBonus)
319 return TotalSecondaryCost < IC.
getCost();
321 int TotalCost = TotalSecondaryCost + IC.
getCost() * NumCallerUsers;
323 return TotalCost < Allowance;
331 template <
class RemarkT>
335 R <<
"(cost=always)";
343 R <<
": " <<
ore::NV(
"Reason", Reason);
380 <<
", Call: " << CB <<
"\n");
386 <<
", Call: " << CB <<
"\n");
390 <<
"'" <<
NV(
"Callee",
Callee) <<
"' not inlined into '"
391 <<
NV(
"Caller", Caller)
392 <<
"' because it should never be inlined " << IC;
397 <<
"'" <<
NV(
"Callee",
Callee) <<
"' not inlined into '"
398 <<
NV(
"Caller", Caller) <<
"' because too costly to inline "
406 int TotalSecondaryCost = 0;
407 if (EnableDeferral &&
411 <<
", outer Cost = " << TotalSecondaryCost <<
'\n');
415 <<
"Not inlining. Cost of inlining '" <<
NV(
"Callee",
Callee)
416 <<
"' increases the cost of inlining '" <<
NV(
"Caller", Caller)
417 <<
"' in other contexts";
433 for (
DILocation *DIL = DLoc.
get(); DIL; DIL = DIL->getInlinedAt()) {
435 CallSiteLoc <<
" @ ";
440 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
441 uint32_t Discriminator = DIL->getBaseDiscriminator();
442 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
444 Name = DIL->getScope()->getSubprogram()->getName();
445 CallSiteLoc <<
Name.str() <<
":" << llvm::utostr(Offset);
446 if (Format.outputColumn())
447 CallSiteLoc <<
":" << llvm::utostr(DIL->getColumn());
448 if (Format.outputDiscriminator() && Discriminator)
449 CallSiteLoc <<
"." << llvm::utostr(Discriminator);
453 return CallSiteLoc.
str();
462 Remark <<
" at callsite ";
463 for (
DILocation *DIL = DLoc.
get(); DIL; DIL = DIL->getInlinedAt()) {
466 unsigned int Offset = DIL->getLine();
467 Offset -= DIL->getScope()->getSubprogram()->getLine();
468 unsigned int Discriminator = DIL->getBaseDiscriminator();
469 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
471 Name = DIL->getScope()->getSubprogram()->getName();
473 <<
ore::NV(
"Column", DIL->getColumn());
488 StringRef RemarkName = AlwaysInline ?
"AlwaysInline" :
"Inlined";
492 <<
ore::NV(
"Caller", &Caller) <<
"'";
503 bool ForProfileContext,
const char *
PassName) {
507 if (ForProfileContext)
508 Remark <<
" to match profiling context";
509 Remark <<
" with " << IC;
521 std::make_unique<ImportedFunctionsInliningStatistics>();
536 return std::make_unique<MandatoryInlineAdvice>(
this, CB,
getCallerORE(CB),
557 return "always-inline";
559 return "cgscc-inline";
561 return "early-inline";
565 return "module-inline";
567 return "replay-cgscc-inline";
569 return "replay-sample-profile-inline";
571 return "sample-profile-inline";
593 auto TrivialDecision =
596 if (TrivialDecision) {
597 if (TrivialDecision->isSuccess())
606 bool MandatoryOnly) {
623 OS <<
"No Inline Advisor\n";
625 IA->getAdvisor()->print(OS);
632 const auto &MAMProxy =
635 if (InitialC.size() == 0) {
636 OS <<
"SCC is empty!\n";
639 Module &
M = *InitialC.begin()->getFunction().getParent();
642 OS <<
"No Inline Advisor\n";
644 IA->getAdvisor()->print(OS);
A set of analyses that are preserved following a run of a transformation pass.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
int getCost() const
Get the inline cost estimate.
Analysis pass providing the TargetTransformInfo.
void recordInlining()
Exactly one of the record* APIs must be called.
This is an optimization pass for GlobalISel generic memory operations.
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM)
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
std::unique_ptr< ImportedFunctionsInliningStatistics > ImportedFunctionsStats
const Function * getParent() const
Return the enclosing method, or null if none.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
virtual bool isMissedOptRemarkEnabled(StringRef PassName) const
Return true if missed optimization remarks are enabled, override to provide different implementation.
A raw_ostream that writes to an std::string.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
Function *const Caller
Caller and Callee are pre-inlining.
FunctionAnalysisManager FAM
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Thresholds to tune inline cost analysis.
DiagnosticInfoOptimizationBase::Argument NV
int getThreshold() const
Get the threshold against which the cost was computed.
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc)
Add location info to ORE message.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
LLVM Basic Block Representation.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, bool IsMandatory, function_ref< void(OptimizationRemark &)> ExtraContext={}, const char *PassName=nullptr)
Emit ORE message.
const bool IsInliningRecommended
An SCC of the call graph.
static cl::opt< bool > InlineRemarkAttribute("inline-remark-attribute", cl::init(false), cl::Hidden, cl::desc("Enable adding inline-remark attribute to" " callsites processed by inliner but decided" " to be not inlined"))
Flag to add inline messages as callsite attributes 'inline-remark'.
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
OptimizationRemarkEmitter & ORE
The default (manual heuristics) implementation of the InlineAdvisor.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
ModuleAnalysisManager MAM
Represents the cost of inlining a function.
static const char * getLTOPhase(ThinOrFullLTOPhase LTOPhase)
into llvm powi allowing the code generator to produce balanced multiplication trees First
static const char * getInlineAdvisorContext(InlinePass IP)
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
STATISTIC(NumFunctions, "Total number of functions")
This class implements an extremely fast bulk output stream that can only output to a stream.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Analysis pass which computes BlockFrequencyInfo.
Function * getCaller()
Helper to get the caller (the parent function).
std::string formatCallSiteLocation(DebugLoc DLoc, const CallSiteFormat &Format)
Get call site location as a string with the given format.
Analysis providing profile information.
An efficient, type-erasing, non-owning reference to a callable.
const DiagnosticHandler * getDiagHandlerPtr() const
getDiagHandlerPtr - Returns const raw pointer of DiagnosticHandler set by setDiagnosticHandler.
Capture state between an inlining decision having had been made, and its impact being observable.
InliningAdvisorMode
There are 3 scenarios we can use the InlineAdvisor:
static bool shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, function_ref< InlineCost(CallBase &CB)> GetInlineCost)
Return true if inlining of CB can block the caller from being inlined which is proved to be more bene...
Used in the streaming interface as the general argument type.
const char * getAnnotatedInlinePassName() const
NOTE pass name is annotated only when inline advisor constructor provides InlineContext.
constexpr LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
InlineAdvisor(InlineAdvisor &&)=delete
Module * getParent()
Get the module that this global value is contained inside of...
void recordInliningWithCalleeDeleted()
Call after inlining succeeded, and results in the callee being delete-able, meaning,...
A function analysis which provides an AssumptionCache.
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
A special type used by analysis passes to provide an address that identifies that particular analysis...
DILocation * get() const
Get the underlying DILocation.
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
const BasicBlock *const Block
initializer< Ty > init(const Ty &Val)
int getCostDelta() const
Get the cost delta from the threshold for inlining.
virtual void recordInliningImpl()
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
A Module instance is used to store all the information related to an LLVM module.
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, const ReplayInlinerSettings &ReplaySettings, InlineContext IC)
StringRef - Represent a constant reference to a string, i.e.
const char * getReason() const
Get the reason of Always or Never.
A cache of @llvm.assume calls within a function.
Interface for deciding whether to inline a call site or not.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static const Function * getParent(const Value *V)
LLVMContext & getContext() const
All values hold a context through their type.
virtual void recordInliningWithCalleeDeletedImpl()
cl::opt< InlinerFunctionImportStatsOpts > InlinerFunctionImportStats
@ ReplaySampleProfileInliner
static cl::opt< int > InlineDeferralScale("inline-deferral-scale", cl::desc("Scale to limit the cost of inline deferral"), cl::init(2), cl::Hidden)
amdgpu Simplify well known AMD library false FunctionCallee Callee
static cl::opt< bool > EnableInlineDeferral("inline-deferral", cl::init(false), cl::Hidden, cl::desc("Enable deferred inlining"))
virtual std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice)
Optional< InlineResult > getAttributeBasedInliningDecision(CallBase &Call, Function *Callee, TargetTransformInfo &CalleeTTI, function_ref< const TargetLibraryInfo &(Function &)> GetTLI)
Returns InlineResult::success() if the call site should be always inlined because of user directives,...
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
FunctionAnalysisManager & FAM
std::unique_ptr< InlineAdvice > getAdvice(CallBase &CB, bool MandatoryOnly=false)
Get an InlineAdvice containing a recommendation on whether to inline or not.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
LLVMContext & getContext() const
Get the global data context.
static cl::opt< bool > AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), cl::desc("If true, annotate inline advisor remarks " "with LTO and pass information."))
virtual std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB)=0
Provides information about what library functions are available for the current target.
@ None
No LTO/ThinLTO behavior needed.
std::string inlineCostStr(const InlineCost &IC)
Utility for extracting the inline cost message to a string.
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
std::string AnnotateInlinePassName(InlineContext IC)
@ FullLTOPreLink
Full LTO prelink phase.
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
const int LastCallToStaticBonus
const BasicBlock * getParent() const
std::unique_ptr< InlineAdvisor > getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool IsInliningRecommended)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
A container for analyses that lazily runs them and caches their results.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
A lazily constructed view of the call graph of a module.
Optional< bool > EnableDeferral
Indicate whether we should allow inline deferral.
static llvm::Optional< llvm::InlineCost > getDefaultInlineAdvice(CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params)
std::string & str()
Returns the string's reference.
InlineAdvisor *const Advisor
constexpr T value_or(U &&alt) const &
InlineResult is basically true or false.
Analysis pass providing the TargetLibraryInfo.
static const char PassName[]
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
Optional< InlineCost > shouldInline(CallBase &CB, function_ref< InlineCost(CallBase &CB)> GetInlineCost, OptimizationRemarkEmitter &ORE, bool EnableDeferral=true)
Return the cost only if the inliner should attempt to inline at the given CallSite.
ThinOrFullLTOPhase LTOPhase