LLVM 19.0.0git
InlineAdvisor.cpp
Go to the documentation of this file.
1//===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
10// related types.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/Statistic.h"
26#include "llvm/IR/PassManager.h"
29
30using namespace llvm;
31#define DEBUG_TYPE "inline"
32#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
33#define LLVM_HAVE_TF_AOT
34#endif
35
36// This weirdly named statistic tracks the number of times that, when attempting
37// to inline a function A into B, we analyze the callers of B in order to see
38// if those would be more profitable and blocked inline steps.
39STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
40
41/// Flag to add inline messages as callsite attributes 'inline-remark'.
42static cl::opt<bool>
43 InlineRemarkAttribute("inline-remark-attribute", cl::init(false),
45 cl::desc("Enable adding inline-remark attribute to"
46 " callsites processed by inliner but decided"
47 " to be not inlined"));
48
49static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false),
51 cl::desc("Enable deferred inlining"));
52
53// An integer used to limit the cost of inline deferral. The default negative
54// number tells shouldBeDeferred to only take the secondary cost into account.
55static cl::opt<int>
56 InlineDeferralScale("inline-deferral-scale",
57 cl::desc("Scale to limit the cost of inline deferral"),
59
60static cl::opt<bool>
61 AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false),
62 cl::desc("If true, annotate inline advisor remarks "
63 "with LTO and pass information."));
64
65namespace llvm {
67} // namespace llvm
68
69namespace {
70using namespace llvm::ore;
71class MandatoryInlineAdvice : public InlineAdvice {
72public:
73 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
75 bool IsInliningMandatory)
76 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
77
78private:
80
81 void recordInliningImpl() override {
82 if (IsInliningRecommended)
83 emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended,
85 Remark << ": always inline attribute";
86 });
87 }
88
89 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
90 if (IsInliningRecommended)
91 ORE.emit([&]() {
93 "NotInlined", DLoc, Block)
94 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
95 << NV("Caller", Caller)
96 << "': " << NV("Reason", Result.getFailureReason());
97 });
98 }
99
100 void recordUnattemptedInliningImpl() override {
101 assert(!IsInliningRecommended && "Expected to attempt inlining");
102 }
103};
104} // namespace
105
106void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
107 const InlineResult &Result) {
108 using namespace ore;
109 llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
110 "; " + inlineCostStr(*OIC));
111 ORE.emit([&]() {
113 "NotInlined", DLoc, Block)
114 << "'" << NV("Callee", Callee) << "' is not inlined into '"
115 << NV("Caller", Caller)
116 << "': " << NV("Reason", Result.getFailureReason());
117 });
118}
119
120void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
121 if (EmitRemarks)
123 /* ForProfileContext= */ false,
125}
126
127void DefaultInlineAdvice::recordInliningImpl() {
128 if (EmitRemarks)
130 /* ForProfileContext= */ false,
132}
133
134std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
135 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
136 Function &Caller = *CB.getCaller();
137 ProfileSummaryInfo *PSI =
139 .getCachedResult<ProfileSummaryAnalysis>(
140 *CB.getParent()->getParent()->getParent());
141
143 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
145 };
146 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
148 };
149 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
151 };
152
153 auto GetInlineCost = [&](CallBase &CB) {
154 Function &Callee = *CB.getCalledFunction();
155 auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
156 bool RemarksEnabled =
157 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
158 DEBUG_TYPE);
159 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
160 GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
161 };
162 return llvm::shouldInline(
163 CB, GetInlineCost, ORE,
164 Params.EnableDeferral.value_or(EnableInlineDeferral));
165}
166
167std::unique_ptr<InlineAdvice>
168DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
169 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
170 return std::make_unique<DefaultInlineAdvice>(
171 this, CB, OIC,
173}
174
177 bool IsInliningRecommended)
178 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
179 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
180 IsInliningRecommended(IsInliningRecommended) {}
181
182void InlineAdvice::recordInlineStatsIfNeeded() {
184 Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee);
185}
186
188 markRecorded();
189 recordInlineStatsIfNeeded();
191}
192
194 markRecorded();
195 recordInlineStatsIfNeeded();
197}
198
202
205 const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
206 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
208 auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(M);
209 Advisor.reset(DA.Factory(M, FAM, Params, IC));
210 return !!Advisor;
211 }
212 auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) {
213 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
214 return OIC.has_value();
215 };
216 switch (Mode) {
218 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
219 Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC));
220 // Restrict replay to default advisor, ML advisors are stateful so
221 // replay will need augmentations to interleave with them correctly.
222 if (!ReplaySettings.ReplayFile.empty()) {
224 std::move(Advisor), ReplaySettings,
225 /* EmitRemarks =*/true, IC);
226 }
227 break;
229#ifdef LLVM_HAVE_TFLITE
230 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
231 Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
232#endif
233 break;
235 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
236 Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
237 break;
238 }
239
240 return !!Advisor;
241}
242
243/// Return true if inlining of CB can block the caller from being
244/// inlined which is proved to be more beneficial. \p IC is the
245/// estimated inline cost associated with callsite \p CB.
246/// \p TotalSecondaryCost will be set to the estimated cost of inlining the
247/// caller if \p CB is suppressed for inlining.
248static bool
249shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
250 function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
251 // For now we only handle local or inline functions.
252 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
253 return false;
254 // If the cost of inlining CB is non-positive, it is not going to prevent the
255 // caller from being inlined into its callers and hence we don't need to
256 // defer.
257 if (IC.getCost() <= 0)
258 return false;
259 // Try to detect the case where the current inlining candidate caller (call
260 // it B) is a static or linkonce-ODR function and is an inlining candidate
261 // elsewhere, and the current candidate callee (call it C) is large enough
262 // that inlining it into B would make B too big to inline later. In these
263 // circumstances it may be best not to inline C into B, but to inline B into
264 // its callers.
265 //
266 // This only applies to static and linkonce-ODR functions because those are
267 // expected to be available for inlining in the translation units where they
268 // are used. Thus we will always have the opportunity to make local inlining
269 // decisions. Importantly the linkonce-ODR linkage covers inline functions
270 // and templates in C++.
271 //
272 // FIXME: All of this logic should be sunk into getInlineCost. It relies on
273 // the internal implementation of the inline cost metrics rather than
274 // treating them as truly abstract units etc.
275 TotalSecondaryCost = 0;
276 // The candidate cost to be imposed upon the current function.
277 int CandidateCost = IC.getCost() - 1;
278 // If the caller has local linkage and can be inlined to all its callers, we
279 // can apply a huge negative bonus to TotalSecondaryCost.
280 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
281 // This bool tracks what happens if we DO inline C into B.
282 bool InliningPreventsSomeOuterInline = false;
283 unsigned NumCallerUsers = 0;
284 for (User *U : Caller->users()) {
285 CallBase *CS2 = dyn_cast<CallBase>(U);
286
287 // If this isn't a call to Caller (it could be some other sort
288 // of reference) skip it. Such references will prevent the caller
289 // from being removed.
290 if (!CS2 || CS2->getCalledFunction() != Caller) {
291 ApplyLastCallBonus = false;
292 continue;
293 }
294
295 InlineCost IC2 = GetInlineCost(*CS2);
296 ++NumCallerCallersAnalyzed;
297 if (!IC2) {
298 ApplyLastCallBonus = false;
299 continue;
300 }
301 if (IC2.isAlways())
302 continue;
303
304 // See if inlining of the original callsite would erase the cost delta of
305 // this callsite. We subtract off the penalty for the call instruction,
306 // which we would be deleting.
307 if (IC2.getCostDelta() <= CandidateCost) {
308 InliningPreventsSomeOuterInline = true;
309 TotalSecondaryCost += IC2.getCost();
310 NumCallerUsers++;
311 }
312 }
313
314 if (!InliningPreventsSomeOuterInline)
315 return false;
316
317 // If all outer calls to Caller would get inlined, the cost for the last
318 // one is set very low by getInlineCost, in anticipation that Caller will
319 // be removed entirely. We did not account for this above unless there
320 // is only one caller of Caller.
321 if (ApplyLastCallBonus)
322 TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
323
324 // If InlineDeferralScale is negative, then ignore the cost of primary
325 // inlining -- IC.getCost() multiplied by the number of callers to Caller.
326 if (InlineDeferralScale < 0)
327 return TotalSecondaryCost < IC.getCost();
328
329 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
330 int Allowance = IC.getCost() * InlineDeferralScale;
331 return TotalCost < Allowance;
332}
333
334namespace llvm {
335static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {
336 return R << Arg.Val;
337}
338
339template <class RemarkT>
340RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) {
341 using namespace ore;
342 if (IC.isAlways()) {
343 R << "(cost=always)";
344 } else if (IC.isNever()) {
345 R << "(cost=never)";
346 } else {
347 R << "(cost=" << ore::NV("Cost", IC.getCost())
348 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
349 }
350 if (const char *Reason = IC.getReason())
351 R << ": " << ore::NV("Reason", Reason);
352 return R;
353}
354} // namespace llvm
355
356std::string llvm::inlineCostStr(const InlineCost &IC) {
357 std::string Buffer;
359 Remark << IC;
360 return Remark.str();
361}
362
365 return;
366
367 Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message);
368 CB.addFnAttr(Attr);
369}
370
371/// Return the cost only if the inliner should attempt to inline at the given
372/// CallSite. If we return the cost, we will emit an optimisation remark later
373/// using that cost, so we won't do so from this function. Return std::nullopt
374/// if inlining should not be attempted.
375std::optional<InlineCost>
377 function_ref<InlineCost(CallBase &CB)> GetInlineCost,
378 OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
379 using namespace ore;
380
381 InlineCost IC = GetInlineCost(CB);
382 Instruction *Call = &CB;
383 Function *Callee = CB.getCalledFunction();
384 Function *Caller = CB.getCaller();
385
386 if (IC.isAlways()) {
387 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC)
388 << ", Call: " << CB << "\n");
389 return IC;
390 }
391
392 if (!IC) {
393 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC)
394 << ", Call: " << CB << "\n");
395 if (IC.isNever()) {
396 ORE.emit([&]() {
397 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
398 << "'" << NV("Callee", Callee) << "' not inlined into '"
399 << NV("Caller", Caller)
400 << "' because it should never be inlined " << IC;
401 });
402 } else {
403 ORE.emit([&]() {
404 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
405 << "'" << NV("Callee", Callee) << "' not inlined into '"
406 << NV("Caller", Caller) << "' because too costly to inline "
407 << IC;
408 });
409 }
411 return std::nullopt;
412 }
413
414 int TotalSecondaryCost = 0;
415 if (EnableDeferral &&
416 shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
417 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB
418 << " Cost = " << IC.getCost()
419 << ", outer Cost = " << TotalSecondaryCost << '\n');
420 ORE.emit([&]() {
421 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
422 Call)
423 << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
424 << "' increases the cost of inlining '" << NV("Caller", Caller)
425 << "' in other contexts";
426 });
427 setInlineRemark(CB, "deferred");
428 return std::nullopt;
429 }
430
431 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB
432 << '\n');
433 return IC;
434}
435
437 const CallSiteFormat &Format) {
438 std::string Buffer;
439 raw_string_ostream CallSiteLoc(Buffer);
440 bool First = true;
441 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
442 if (!First)
443 CallSiteLoc << " @ ";
444 // Note that negative line offset is actually possible, but we use
445 // unsigned int to match line offset representation in remarks so
446 // it's directly consumable by relay advisor.
448 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
449 uint32_t Discriminator = DIL->getBaseDiscriminator();
450 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
451 if (Name.empty())
452 Name = DIL->getScope()->getSubprogram()->getName();
453 CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset);
454 if (Format.outputColumn())
455 CallSiteLoc << ":" << llvm::utostr(DIL->getColumn());
456 if (Format.outputDiscriminator() && Discriminator)
457 CallSiteLoc << "." << llvm::utostr(Discriminator);
458 First = false;
459 }
460
461 return CallSiteLoc.str();
462}
463
465 if (!DLoc) {
466 return;
467 }
468
469 bool First = true;
470 Remark << " at callsite ";
471 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
472 if (!First)
473 Remark << " @ ";
474 unsigned int Offset = DIL->getLine();
475 Offset -= DIL->getScope()->getSubprogram()->getLine();
476 unsigned int Discriminator = DIL->getBaseDiscriminator();
477 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
478 if (Name.empty())
479 Name = DIL->getScope()->getSubprogram()->getName();
480 Remark << Name << ":" << ore::NV("Line", Offset) << ":"
481 << ore::NV("Column", DIL->getColumn());
482 if (Discriminator)
483 Remark << "." << ore::NV("Disc", Discriminator);
484 First = false;
485 }
486
487 Remark << ";";
488}
489
492 const Function &Callee, const Function &Caller, bool AlwaysInline,
493 function_ref<void(OptimizationRemark &)> ExtraContext,
494 const char *PassName) {
495 ORE.emit([&]() {
496 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
498 DLoc, Block);
499 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
500 << ore::NV("Caller", &Caller) << "'";
501 if (ExtraContext)
502 ExtraContext(Remark);
504 return Remark;
505 });
506}
507
510 const Function &Callee, const Function &Caller, const InlineCost &IC,
511 bool ForProfileContext, const char *PassName) {
513 ORE, DLoc, Block, Callee, Caller, IC.isAlways(),
515 if (ForProfileContext)
516 Remark << " to match profiling context";
517 Remark << " with " << IC;
518 },
519 PassName);
520}
521
523 std::optional<InlineContext> IC)
524 : M(M), FAM(FAM), IC(IC),
525 AnnotatedInlinePassName((IC && AnnotateInlinePhase)
527 : DEBUG_TYPE) {
530 std::make_unique<ImportedFunctionsInliningStatistics>();
531 ImportedFunctionsStats->setModuleInfo(M);
532 }
533}
534
540 }
541}
542
543std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
544 bool Advice) {
545 return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB),
546 Advice);
547}
548
549static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
550 switch (LTOPhase) {
552 return "main";
555 return "prelink";
558 return "postlink";
559 }
560 llvm_unreachable("unreachable");
561}
562
563static inline const char *getInlineAdvisorContext(InlinePass IP) {
564 switch (IP) {
566 return "always-inline";
568 return "cgscc-inline";
570 return "early-inline";
572 return "ml-inline";
574 return "module-inline";
576 return "replay-cgscc-inline";
578 return "replay-sample-profile-inline";
580 return "sample-profile-inline";
581 }
582
583 llvm_unreachable("unreachable");
584}
585
587 return std::string(getLTOPhase(IC.LTOPhase)) + "-" +
588 std::string(getInlineAdvisorContext(IC.Pass));
589}
590
594 auto &Callee = *CB.getCalledFunction();
595
596 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
598 };
599
600 auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
601
602 auto TrivialDecision =
603 llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);
604
605 if (TrivialDecision) {
606 if (TrivialDecision->isSuccess())
608 else
610 }
612}
613
614std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
615 bool MandatoryOnly) {
616 if (!MandatoryOnly)
617 return getAdviceImpl(CB);
618 bool Advice = CB.getCaller() != CB.getCalledFunction() &&
621 return getMandatoryAdvice(CB, Advice);
622}
623
626}
627
630 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
631 if (!IA)
632 OS << "No Inline Advisor\n";
633 else
634 IA->getAdvisor()->print(OS);
635 return PreservedAnalyses::all();
636}
637
640 CGSCCUpdateResult &UR) {
641 const auto &MAMProxy =
643
644 if (InitialC.size() == 0) {
645 OS << "SCC is empty!\n";
646 return PreservedAnalyses::all();
647 }
648 Module &M = *InitialC.begin()->getFunction().getParent();
649 const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M);
650 if (!IA)
651 OS << "No Inline Advisor\n";
652 else
653 IA->getAdvisor()->print(OS);
654 return PreservedAnalyses::all();
655}
static const Function * getParent(const Value *V)
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
#define DEBUG_TYPE
static cl::opt< int > InlineDeferralScale("inline-deferral-scale", cl::desc("Scale to limit the cost of inline deferral"), cl::init(2), cl::Hidden)
static cl::opt< bool > InlineRemarkAttribute("inline-remark-attribute", cl::init(false), cl::Hidden, cl::desc("Enable adding inline-remark attribute to" " callsites processed by inliner but decided" " to be not inlined"))
Flag to add inline messages as callsite attributes 'inline-remark'.
static cl::opt< bool > EnableInlineDeferral("inline-deferral", cl::init(false), cl::Hidden, cl::desc("Enable deferred inlining"))
static cl::opt< bool > AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), cl::desc("If true, annotate inline advisor remarks " "with LTO and pass information."))
static const char * getLTOPhase(ThinOrFullLTOPhase LTOPhase)
static std::optional< llvm::InlineCost > getDefaultInlineAdvice(CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params)
static const char * getInlineAdvisorContext(InlinePass IP)
static bool shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, function_ref< InlineCost(CallBase &CB)> GetInlineCost)
Return true if inlining of CB can block the caller from being inlined which is proved to be more bene...
#define F(x, y, z)
Definition: MD5.cpp:55
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This file contains some functions that are useful when dealing with strings.
This pass exposes codegen information to IR-level passes.
static const char PassName[]
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:242
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
Definition: PassManager.h:413
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:394
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:94
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
Definition: InstrTypes.h:1574
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
Function * getCaller()
Helper to get the caller (the parent function).
Debug location.
A debug info location.
Definition: DebugLoc.h:33
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:20
The default (manual heuristics) implementation of the InlineAdvisor.
Capture state between an inlining decision having had been made, and its impact being observable.
Definition: InlineAdvisor.h:75
Function *const Callee
Function *const Caller
Caller and Callee are pre-inlining.
virtual void recordInliningWithCalleeDeletedImpl()
virtual void recordInliningImpl()
const BasicBlock *const Block
void recordInlining()
Exactly one of the record* APIs must be called.
OptimizationRemarkEmitter & ORE
virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result)
InlineAdvisor *const Advisor
InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool IsInliningRecommended)
virtual void recordUnattemptedInliningImpl()
const DebugLoc DLoc
void recordInliningWithCalleeDeleted()
Call after inlining succeeded, and results in the callee being delete-able, meaning,...
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
static AnalysisKey Key
Interface for deciding whether to inline a call site or not.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
FunctionAnalysisManager & FAM
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
virtual std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice)
std::unique_ptr< ImportedFunctionsInliningStatistics > ImportedFunctionsStats
const char * getAnnotatedInlinePassName() const
NOTE pass name is annotated only when inline advisor constructor provides InlineContext.
InlineAdvisor(InlineAdvisor &&)=delete
virtual std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB)=0
std::unique_ptr< InlineAdvice > getAdvice(CallBase &CB, bool MandatoryOnly=false)
Get an InlineAdvice containing a recommendation on whether to inline or not.
Represents the cost of inlining a function.
Definition: InlineCost.h:90
int getThreshold() const
Get the threshold against which the cost was computed.
Definition: InlineCost.h:151
const char * getReason() const
Get the reason of Always or Never.
Definition: InlineCost.h:166
int getCost() const
Get the inline cost estimate.
Definition: InlineCost.h:145
bool isAlways() const
Definition: InlineCost.h:139
int getCostDelta() const
Get the cost delta from the threshold for inlining.
Definition: InlineCost.h:175
bool isNever() const
Definition: InlineCost.h:140
InlineResult is basically true or false.
Definition: InlineCost.h:180
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:552
An SCC of the call graph.
iterator begin() const
A lazily constructed view of the call graph of a module.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:301
The optimization diagnostic interface.
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for missed-optimization remarks.
Diagnostic information for applied optimization remarks.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Definition: PassManager.h:677
Used for dynamically registering InlineAdvisors as plugins.
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
Analysis providing profile information.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition: ilist_node.h:32
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:679
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const int LastCallToStaticBonus
Definition: InlineCost.h:50
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, bool IsMandatory, function_ref< void(OptimizationRemark &)> ExtraContext={}, const char *PassName=nullptr)
Emit ORE message.
std::string inlineCostStr(const InlineCost &IC)
Utility for extracting the inline cost message to a string.
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
Definition: InlineAdvisor.h:43
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
std::string AnnotateInlinePassName(InlineContext IC)
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
Definition: Pass.h:76
@ FullLTOPreLink
Full LTO prelink phase.
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
@ None
No LTO/ThinLTO behavior needed.
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
std::optional< InlineResult > getAttributeBasedInliningDecision(CallBase &Call, Function *Callee, TargetTransformInfo &CalleeTTI, function_ref< const TargetLibraryInfo &(Function &)> GetTLI)
Returns InlineResult::success() if the call site should be always inlined because of user directives,...
cl::opt< InlinerFunctionImportStatsOpts > InlinerFunctionImportStats("inliner-function-import-stats", cl::init(InlinerFunctionImportStatsOpts::No), cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic", "basic statistics"), clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose", "printing of statistics for each inlined function")), cl::Hidden, cl::desc("Enable inliner stats for imported functions"))
std::optional< InlineCost > shouldInline(CallBase &CB, function_ref< InlineCost(CallBase &CB)> GetInlineCost, OptimizationRemarkEmitter &ORE, bool EnableDeferral=true)
Return the cost only if the inliner should attempt to inline at the given CallSite.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:293
std::unique_ptr< InlineAdvisor > getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc)
Add location info to ORE message.
std::string formatCallSiteLocation(DebugLoc DLoc, const CallSiteFormat &Format)
Get call site location as a string with the given format.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: Analysis.h:26
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Used in the streaming interface as the general argument type.
bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, const ReplayInlinerSettings &ReplaySettings, InlineContext IC)
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Definition: InlineAdvisor.h:59
ThinOrFullLTOPhase LTOPhase
Definition: InlineAdvisor.h:60
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:206
std::optional< bool > EnableDeferral
Indicate whether we should allow inline deferral.
Definition: InlineCost.h:236