LLVM 19.0.0git
SampleProfile.cpp
Go to the documentation of this file.
1//===- SampleProfile.cpp - Incorporate sample profiles into the IR --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SampleProfileLoader transformation. This pass
10// reads a profile file generated by a sampling profiler (e.g. Linux Perf -
11// http://perf.wiki.kernel.org/) and generates IR metadata to reflect the
12// profile information in the given profile.
13//
14// This pass generates branch weight annotations on the IR:
15//
16// - prof: Represents branch weights. This annotation is added to branches
17// to indicate the weights of each edge coming out of the branch.
18// The weight of each edge is the weight of the target block for
19// that edge. The weight of a block B is computed as the maximum
20// number of samples found in B.
21//
22//===----------------------------------------------------------------------===//
23
25#include "llvm/ADT/ArrayRef.h"
26#include "llvm/ADT/DenseMap.h"
27#include "llvm/ADT/DenseSet.h"
28#include "llvm/ADT/MapVector.h"
32#include "llvm/ADT/Statistic.h"
33#include "llvm/ADT/StringMap.h"
34#include "llvm/ADT/StringRef.h"
35#include "llvm/ADT/Twine.h"
46#include "llvm/IR/BasicBlock.h"
47#include "llvm/IR/DebugLoc.h"
49#include "llvm/IR/Function.h"
50#include "llvm/IR/GlobalValue.h"
51#include "llvm/IR/InstrTypes.h"
52#include "llvm/IR/Instruction.h"
55#include "llvm/IR/LLVMContext.h"
56#include "llvm/IR/MDBuilder.h"
57#include "llvm/IR/Module.h"
58#include "llvm/IR/PassManager.h"
60#include "llvm/IR/PseudoProbe.h"
67#include "llvm/Support/Debug.h"
71#include "llvm/Transforms/IPO.h"
82#include <algorithm>
83#include <cassert>
84#include <cstdint>
85#include <functional>
86#include <limits>
87#include <map>
88#include <memory>
89#include <queue>
90#include <string>
91#include <system_error>
92#include <utility>
93#include <vector>
94
95using namespace llvm;
96using namespace sampleprof;
97using namespace llvm::sampleprofutil;
99#define DEBUG_TYPE "sample-profile"
100#define CSINLINE_DEBUG DEBUG_TYPE "-inline"
101
102STATISTIC(NumCSInlined,
103 "Number of functions inlined with context sensitive profile");
104STATISTIC(NumCSNotInlined,
105 "Number of functions not inlined with context sensitive profile");
106STATISTIC(NumMismatchedProfile,
107 "Number of functions with CFG mismatched profile");
108STATISTIC(NumMatchedProfile, "Number of functions with CFG matched profile");
109STATISTIC(NumDuplicatedInlinesite,
110 "Number of inlined callsites with a partial distribution factor");
111
112STATISTIC(NumCSInlinedHitMinLimit,
113 "Number of functions with FDO inline stopped due to min size limit");
114STATISTIC(NumCSInlinedHitMaxLimit,
115 "Number of functions with FDO inline stopped due to max size limit");
117 NumCSInlinedHitGrowthLimit,
118 "Number of functions with FDO inline stopped due to growth size limit");
119
120// Command line option to specify the file to read samples from. This is
121// mainly used for debugging.
123 "sample-profile-file", cl::init(""), cl::value_desc("filename"),
124 cl::desc("Profile file loaded by -sample-profile"), cl::Hidden);
125
126// The named file contains a set of transformations that may have been applied
127// to the symbol names between the program from which the sample data was
128// collected and the current program's symbols.
130 "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"),
131 cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden);
132
134 "salvage-stale-profile", cl::Hidden, cl::init(false),
135 cl::desc("Salvage stale profile by fuzzy matching and use the remapped "
136 "location for sample profile query."));
137
139 "report-profile-staleness", cl::Hidden, cl::init(false),
140 cl::desc("Compute and report stale profile statistical metrics."));
141
143 "persist-profile-staleness", cl::Hidden, cl::init(false),
144 cl::desc("Compute stale profile statistical metrics and write it into the "
145 "native object file(.llvm_stats section)."));
146
148 "profile-sample-accurate", cl::Hidden, cl::init(false),
149 cl::desc("If the sample profile is accurate, we will mark all un-sampled "
150 "callsite and function as having 0 samples. Otherwise, treat "
151 "un-sampled callsites and functions conservatively as unknown. "));
152
154 "profile-sample-block-accurate", cl::Hidden, cl::init(false),
155 cl::desc("If the sample profile is accurate, we will mark all un-sampled "
156 "branches and calls as having 0 samples. Otherwise, treat "
157 "them conservatively as unknown. "));
158
160 "profile-accurate-for-symsinlist", cl::Hidden, cl::init(true),
161 cl::desc("For symbols in profile symbol list, regard their profiles to "
162 "be accurate. It may be overriden by profile-sample-accurate. "));
163
165 "sample-profile-merge-inlinee", cl::Hidden, cl::init(true),
166 cl::desc("Merge past inlinee's profile to outline version if sample "
167 "profile loader decided not to inline a call site. It will "
168 "only be enabled when top-down order of profile loading is "
169 "enabled. "));
170
172 "sample-profile-top-down-load", cl::Hidden, cl::init(true),
173 cl::desc("Do profile annotation and inlining for functions in top-down "
174 "order of call graph during sample profile loading. It only "
175 "works for new pass manager. "));
176
177static cl::opt<bool>
178 UseProfiledCallGraph("use-profiled-call-graph", cl::init(true), cl::Hidden,
179 cl::desc("Process functions in a top-down order "
180 "defined by the profiled call graph when "
181 "-sample-profile-top-down-load is on."));
182
184 "sample-profile-inline-size", cl::Hidden, cl::init(false),
185 cl::desc("Inline cold call sites in profile loader if it's beneficial "
186 "for code size."));
187
188// Since profiles are consumed by many passes, turning on this option has
189// side effects. For instance, pre-link SCC inliner would see merged profiles
190// and inline the hot functions (that are skipped in this pass).
192 "disable-sample-loader-inlining", cl::Hidden, cl::init(false),
193 cl::desc("If true, artifically skip inline transformation in sample-loader "
194 "pass, and merge (or scale) profiles (as configured by "
195 "--sample-profile-merge-inlinee)."));
196
197namespace llvm {
199 SortProfiledSCC("sort-profiled-scc-member", cl::init(true), cl::Hidden,
200 cl::desc("Sort profiled recursion by edge weights."));
201
203 "sample-profile-inline-growth-limit", cl::Hidden, cl::init(12),
204 cl::desc("The size growth ratio limit for proirity-based sample profile "
205 "loader inlining."));
206
208 "sample-profile-inline-limit-min", cl::Hidden, cl::init(100),
209 cl::desc("The lower bound of size growth limit for "
210 "proirity-based sample profile loader inlining."));
211
213 "sample-profile-inline-limit-max", cl::Hidden, cl::init(10000),
214 cl::desc("The upper bound of size growth limit for "
215 "proirity-based sample profile loader inlining."));
216
218 "sample-profile-hot-inline-threshold", cl::Hidden, cl::init(3000),
219 cl::desc("Hot callsite threshold for proirity-based sample profile loader "
220 "inlining."));
221
223 "sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45),
224 cl::desc("Threshold for inlining cold callsites"));
225} // namespace llvm
226
228 "sample-profile-icp-relative-hotness", cl::Hidden, cl::init(25),
229 cl::desc(
230 "Relative hotness percentage threshold for indirect "
231 "call promotion in proirity-based sample profile loader inlining."));
232
234 "sample-profile-icp-relative-hotness-skip", cl::Hidden, cl::init(1),
235 cl::desc(
236 "Skip relative hotness check for ICP up to given number of targets."));
237
239 "hot-func-cutoff-for-staleness-error", cl::Hidden, cl::init(800000),
240 cl::desc("A function is considered hot for staleness error check if its "
241 "total sample count is above the specified percentile"));
242
244 "min-functions-for-staleness-error", cl::Hidden, cl::init(50),
245 cl::desc("Skip the check if the number of hot functions is smaller than "
246 "the specified number."));
247
249 "precent-mismatch-for-staleness-error", cl::Hidden, cl::init(80),
250 cl::desc("Reject the profile if the mismatch percent is higher than the "
251 "given number."));
252
254 "sample-profile-prioritized-inline", cl::Hidden,
255 cl::desc("Use call site prioritized inlining for sample profile loader."
256 "Currently only CSSPGO is supported."));
257
259 "sample-profile-use-preinliner", cl::Hidden,
260 cl::desc("Use the preinliner decisions stored in profile context."));
261
263 "sample-profile-recursive-inline", cl::Hidden,
264 cl::desc("Allow sample loader inliner to inline recursive calls."));
265
267 "sample-profile-remove-probe", cl::Hidden, cl::init(false),
268 cl::desc("Remove pseudo-probe after sample profile annotation."));
269
271 "sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"),
272 cl::desc(
273 "Optimization remarks file containing inline remarks to be replayed "
274 "by inlining from sample profile loader."),
275 cl::Hidden);
276
278 "sample-profile-inline-replay-scope",
279 cl::init(ReplayInlinerSettings::Scope::Function),
280 cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function",
281 "Replay on functions that have remarks associated "
282 "with them (default)"),
283 clEnumValN(ReplayInlinerSettings::Scope::Module, "Module",
284 "Replay on the entire module")),
285 cl::desc("Whether inline replay should be applied to the entire "
286 "Module or just the Functions (default) that are present as "
287 "callers in remarks during sample profile inlining."),
288 cl::Hidden);
289
291 "sample-profile-inline-replay-fallback",
292 cl::init(ReplayInlinerSettings::Fallback::Original),
295 ReplayInlinerSettings::Fallback::Original, "Original",
296 "All decisions not in replay send to original advisor (default)"),
297 clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline,
298 "AlwaysInline", "All decisions not in replay are inlined"),
299 clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline",
300 "All decisions not in replay are not inlined")),
301 cl::desc("How sample profile inline replay treats sites that don't come "
302 "from the replay. Original: defers to original advisor, "
303 "AlwaysInline: inline all sites not in replay, NeverInline: "
304 "inline no sites not in replay"),
305 cl::Hidden);
306
308 "sample-profile-inline-replay-format",
309 cl::init(CallSiteFormat::Format::LineColumnDiscriminator),
311 clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"),
312 clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn",
313 "<Line Number>:<Column Number>"),
314 clEnumValN(CallSiteFormat::Format::LineDiscriminator,
315 "LineDiscriminator", "<Line Number>.<Discriminator>"),
316 clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator,
317 "LineColumnDiscriminator",
318 "<Line Number>:<Column Number>.<Discriminator> (default)")),
319 cl::desc("How sample profile inline replay file is formatted"), cl::Hidden);
320
322 MaxNumPromotions("sample-profile-icp-max-prom", cl::init(3), cl::Hidden,
323 cl::desc("Max number of promotions for a single indirect "
324 "call callsite in sample profile loader"));
325
327 "overwrite-existing-weights", cl::Hidden, cl::init(false),
328 cl::desc("Ignore existing branch weights on IR and always overwrite."));
329
331 "annotate-sample-profile-inline-phase", cl::Hidden, cl::init(false),
332 cl::desc("Annotate LTO phase (prelink / postlink), or main (no LTO) for "
333 "sample-profile inline pass name."));
334
335namespace llvm {
337}
338
339namespace {
340
341using BlockWeightMap = DenseMap<const BasicBlock *, uint64_t>;
342using EquivalenceClassMap = DenseMap<const BasicBlock *, const BasicBlock *>;
343using Edge = std::pair<const BasicBlock *, const BasicBlock *>;
344using EdgeWeightMap = DenseMap<Edge, uint64_t>;
345using BlockEdgeMap =
347
348class GUIDToFuncNameMapper {
349public:
350 GUIDToFuncNameMapper(Module &M, SampleProfileReader &Reader,
351 DenseMap<uint64_t, StringRef> &GUIDToFuncNameMap)
352 : CurrentReader(Reader), CurrentModule(M),
353 CurrentGUIDToFuncNameMap(GUIDToFuncNameMap) {
354 if (!CurrentReader.useMD5())
355 return;
356
357 for (const auto &F : CurrentModule) {
358 StringRef OrigName = F.getName();
359 CurrentGUIDToFuncNameMap.insert(
360 {Function::getGUID(OrigName), OrigName});
361
362 // Local to global var promotion used by optimization like thinlto
363 // will rename the var and add suffix like ".llvm.xxx" to the
364 // original local name. In sample profile, the suffixes of function
365 // names are all stripped. Since it is possible that the mapper is
366 // built in post-thin-link phase and var promotion has been done,
367 // we need to add the substring of function name without the suffix
368 // into the GUIDToFuncNameMap.
370 if (CanonName != OrigName)
371 CurrentGUIDToFuncNameMap.insert(
372 {Function::getGUID(CanonName), CanonName});
373 }
374
375 // Update GUIDToFuncNameMap for each function including inlinees.
376 SetGUIDToFuncNameMapForAll(&CurrentGUIDToFuncNameMap);
377 }
378
379 ~GUIDToFuncNameMapper() {
380 if (!CurrentReader.useMD5())
381 return;
382
383 CurrentGUIDToFuncNameMap.clear();
384
385 // Reset GUIDToFuncNameMap for of each function as they're no
386 // longer valid at this point.
387 SetGUIDToFuncNameMapForAll(nullptr);
388 }
389
390private:
391 void SetGUIDToFuncNameMapForAll(DenseMap<uint64_t, StringRef> *Map) {
392 std::queue<FunctionSamples *> FSToUpdate;
393 for (auto &IFS : CurrentReader.getProfiles()) {
394 FSToUpdate.push(&IFS.second);
395 }
396
397 while (!FSToUpdate.empty()) {
398 FunctionSamples *FS = FSToUpdate.front();
399 FSToUpdate.pop();
400 FS->GUIDToFuncNameMap = Map;
401 for (const auto &ICS : FS->getCallsiteSamples()) {
402 const FunctionSamplesMap &FSMap = ICS.second;
403 for (const auto &IFS : FSMap) {
404 FunctionSamples &FS = const_cast<FunctionSamples &>(IFS.second);
405 FSToUpdate.push(&FS);
406 }
407 }
408 }
409 }
410
412 Module &CurrentModule;
413 DenseMap<uint64_t, StringRef> &CurrentGUIDToFuncNameMap;
414};
415
416// Inline candidate used by iterative callsite prioritized inliner
417struct InlineCandidate {
418 CallBase *CallInstr;
419 const FunctionSamples *CalleeSamples;
420 // Prorated callsite count, which will be used to guide inlining. For example,
421 // if a callsite is duplicated in LTO prelink, then in LTO postlink the two
422 // copies will get their own distribution factors and their prorated counts
423 // will be used to decide if they should be inlined independently.
424 uint64_t CallsiteCount;
425 // Call site distribution factor to prorate the profile samples for a
426 // duplicated callsite. Default value is 1.0.
427 float CallsiteDistribution;
428};
429
430// Inline candidate comparer using call site weight
431struct CandidateComparer {
432 bool operator()(const InlineCandidate &LHS, const InlineCandidate &RHS) {
433 if (LHS.CallsiteCount != RHS.CallsiteCount)
434 return LHS.CallsiteCount < RHS.CallsiteCount;
435
436 const FunctionSamples *LCS = LHS.CalleeSamples;
437 const FunctionSamples *RCS = RHS.CalleeSamples;
438 assert(LCS && RCS && "Expect non-null FunctionSamples");
439
440 // Tie breaker using number of samples try to favor smaller functions first
441 if (LCS->getBodySamples().size() != RCS->getBodySamples().size())
442 return LCS->getBodySamples().size() > RCS->getBodySamples().size();
443
444 // Tie breaker using GUID so we have stable/deterministic inlining order
445 return LCS->getGUID() < RCS->getGUID();
446 }
447};
448
449using CandidateQueue =
451 CandidateComparer>;
452
453/// Sample profile pass.
454///
455/// This pass reads profile data from the file specified by
456/// -sample-profile-file and annotates every affected function with the
457/// profile information found in that file.
458class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> {
459public:
460 SampleProfileLoader(
461 StringRef Name, StringRef RemapName, ThinOrFullLTOPhase LTOPhase,
463 std::function<AssumptionCache &(Function &)> GetAssumptionCache,
464 std::function<TargetTransformInfo &(Function &)> GetTargetTransformInfo,
465 std::function<const TargetLibraryInfo &(Function &)> GetTLI)
467 std::move(FS)),
468 GetAC(std::move(GetAssumptionCache)),
469 GetTTI(std::move(GetTargetTransformInfo)), GetTLI(std::move(GetTLI)),
470 LTOPhase(LTOPhase),
471 AnnotatedPassName(AnnotateSampleProfileInlinePhase
474 : CSINLINE_DEBUG) {}
475
476 bool doInitialization(Module &M, FunctionAnalysisManager *FAM = nullptr);
477 bool runOnModule(Module &M, ModuleAnalysisManager *AM,
479
480protected:
482 bool emitAnnotations(Function &F);
484 const FunctionSamples *findCalleeFunctionSamples(const CallBase &I) const;
485 const FunctionSamples *
486 findFunctionSamples(const Instruction &I) const override;
487 std::vector<const FunctionSamples *>
488 findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const;
489 void findExternalInlineCandidate(CallBase *CB, const FunctionSamples *Samples,
490 DenseSet<GlobalValue::GUID> &InlinedGUIDs,
491 uint64_t Threshold);
492 // Attempt to promote indirect call and also inline the promoted call
493 bool tryPromoteAndInlineCandidate(
494 Function &F, InlineCandidate &Candidate, uint64_t SumOrigin,
495 uint64_t &Sum, SmallVector<CallBase *, 8> *InlinedCallSites = nullptr);
496
497 bool inlineHotFunctions(Function &F,
498 DenseSet<GlobalValue::GUID> &InlinedGUIDs);
499 std::optional<InlineCost> getExternalInlineAdvisorCost(CallBase &CB);
500 bool getExternalInlineAdvisorShouldInline(CallBase &CB);
501 InlineCost shouldInlineCandidate(InlineCandidate &Candidate);
502 bool getInlineCandidate(InlineCandidate *NewCandidate, CallBase *CB);
503 bool
504 tryInlineCandidate(InlineCandidate &Candidate,
505 SmallVector<CallBase *, 8> *InlinedCallSites = nullptr);
506 bool
507 inlineHotFunctionsWithPriority(Function &F,
508 DenseSet<GlobalValue::GUID> &InlinedGUIDs);
509 // Inline cold/small functions in addition to hot ones
510 bool shouldInlineColdCallee(CallBase &CallInst);
511 void emitOptimizationRemarksForInlineCandidates(
512 const SmallVectorImpl<CallBase *> &Candidates, const Function &F,
513 bool Hot);
514 void promoteMergeNotInlinedContextSamples(
516 const Function &F);
517 std::vector<Function *> buildFunctionOrder(Module &M, LazyCallGraph &CG);
518 std::unique_ptr<ProfiledCallGraph> buildProfiledCallGraph(Module &M);
519 void generateMDProfMetadata(Function &F);
520 bool rejectHighStalenessProfile(Module &M, ProfileSummaryInfo *PSI,
521 const SampleProfileMap &Profiles);
522 void removePseudoProbeInsts(Module &M);
523
524 /// Map from function name to Function *. Used to find the function from
525 /// the function name. If the function name contains suffix, additional
526 /// entry is added to map from the stripped name to the function if there
527 /// is one-to-one mapping.
529
530 std::function<AssumptionCache &(Function &)> GetAC;
531 std::function<TargetTransformInfo &(Function &)> GetTTI;
532 std::function<const TargetLibraryInfo &(Function &)> GetTLI;
533
534 /// Profile tracker for different context.
535 std::unique_ptr<SampleContextTracker> ContextTracker;
536
537 /// Flag indicating which LTO/ThinLTO phase the pass is invoked in.
538 ///
539 /// We need to know the LTO phase because for example in ThinLTOPrelink
540 /// phase, in annotation, we should not promote indirect calls. Instead,
541 /// we will mark GUIDs that needs to be annotated to the function.
542 const ThinOrFullLTOPhase LTOPhase;
543 const std::string AnnotatedPassName;
544
545 /// Profle Symbol list tells whether a function name appears in the binary
546 /// used to generate the current profile.
547 std::unique_ptr<ProfileSymbolList> PSL;
548
549 /// Total number of samples collected in this profile.
550 ///
551 /// This is the sum of all the samples collected in all the functions executed
552 /// at runtime.
553 uint64_t TotalCollectedSamples = 0;
554
555 // Information recorded when we declined to inline a call site
556 // because we have determined it is too cold is accumulated for
557 // each callee function. Initially this is just the entry count.
558 struct NotInlinedProfileInfo {
559 uint64_t entryCount;
560 };
562
563 // GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
564 // all the function symbols defined or declared in current module.
565 DenseMap<uint64_t, StringRef> GUIDToFuncNameMap;
566
567 // All the Names used in FunctionSamples including outline function
568 // names, inline instance names and call target names.
569 StringSet<> NamesInProfile;
570 // MD5 version of NamesInProfile. Either NamesInProfile or GUIDsInProfile is
571 // populated, depends on whether the profile uses MD5. Because the name table
572 // generally contains several magnitude more entries than the number of
573 // functions, we do not want to convert all names from one form to another.
574 llvm::DenseSet<uint64_t> GUIDsInProfile;
575
576 // For symbol in profile symbol list, whether to regard their profiles
577 // to be accurate. It is mainly decided by existance of profile symbol
578 // list and -profile-accurate-for-symsinlist flag, but it can be
579 // overriden by -profile-sample-accurate or profile-sample-accurate
580 // attribute.
581 bool ProfAccForSymsInList;
582
583 // External inline advisor used to replay inline decision from remarks.
584 std::unique_ptr<InlineAdvisor> ExternalInlineAdvisor;
585
586 // A helper to implement the sample profile matching algorithm.
587 std::unique_ptr<SampleProfileMatcher> MatchingManager;
588
589private:
590 const char *getAnnotatedRemarkPassName() const {
591 return AnnotatedPassName.c_str();
592 }
593};
594} // end anonymous namespace
595
596namespace llvm {
597template <>
599 return succ_empty(BB);
600}
601
602template <>
604 const std::vector<const BasicBlockT *> &BasicBlocks,
605 BlockEdgeMap &Successors, FlowFunction &Func) {
606 for (auto &Jump : Func.Jumps) {
607 const auto *BB = BasicBlocks[Jump.Source];
608 const auto *Succ = BasicBlocks[Jump.Target];
609 const Instruction *TI = BB->getTerminator();
610 // Check if a block ends with InvokeInst and mark non-taken branch unlikely.
611 // In that case block Succ should be a landing pad
612 if (Successors[BB].size() == 2 && Successors[BB].back() == Succ) {
613 if (isa<InvokeInst>(TI)) {
614 Jump.IsUnlikely = true;
615 }
616 }
617 const Instruction *SuccTI = Succ->getTerminator();
618 // Check if the target block contains UnreachableInst and mark it unlikely
619 if (SuccTI->getNumSuccessors() == 0) {
620 if (isa<UnreachableInst>(SuccTI)) {
621 Jump.IsUnlikely = true;
622 }
623 }
624 }
625}
626
627template <>
629 Function &F) {
630 DT.reset(new DominatorTree);
631 DT->recalculate(F);
632
633 PDT.reset(new PostDominatorTree(F));
634
635 LI.reset(new LoopInfo);
636 LI->analyze(*DT);
637}
638} // namespace llvm
639
640ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) {
642 return getProbeWeight(Inst);
643
644 const DebugLoc &DLoc = Inst.getDebugLoc();
645 if (!DLoc)
646 return std::error_code();
647
648 // Ignore all intrinsics, phinodes and branch instructions.
649 // Branch and phinodes instruction usually contains debug info from sources
650 // outside of the residing basic block, thus we ignore them during annotation.
651 if (isa<BranchInst>(Inst) || isa<IntrinsicInst>(Inst) || isa<PHINode>(Inst))
652 return std::error_code();
653
654 // For non-CS profile, if a direct call/invoke instruction is inlined in
655 // profile (findCalleeFunctionSamples returns non-empty result), but not
656 // inlined here, it means that the inlined callsite has no sample, thus the
657 // call instruction should have 0 count.
658 // For CS profile, the callsite count of previously inlined callees is
659 // populated with the entry count of the callees.
661 if (const auto *CB = dyn_cast<CallBase>(&Inst))
662 if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
663 return 0;
664
665 return getInstWeightImpl(Inst);
666}
667
668/// Get the FunctionSamples for a call instruction.
669///
670/// The FunctionSamples of a call/invoke instruction \p Inst is the inlined
671/// instance in which that call instruction is calling to. It contains
672/// all samples that resides in the inlined instance. We first find the
673/// inlined instance in which the call instruction is from, then we
674/// traverse its children to find the callsite with the matching
675/// location.
676///
677/// \param Inst Call/Invoke instruction to query.
678///
679/// \returns The FunctionSamples pointer to the inlined instance.
680const FunctionSamples *
681SampleProfileLoader::findCalleeFunctionSamples(const CallBase &Inst) const {
682 const DILocation *DIL = Inst.getDebugLoc();
683 if (!DIL) {
684 return nullptr;
685 }
686
687 StringRef CalleeName;
688 if (Function *Callee = Inst.getCalledFunction())
689 CalleeName = Callee->getName();
690
692 return ContextTracker->getCalleeContextSamplesFor(Inst, CalleeName);
693
694 const FunctionSamples *FS = findFunctionSamples(Inst);
695 if (FS == nullptr)
696 return nullptr;
697
698 return FS->findFunctionSamplesAt(FunctionSamples::getCallSiteIdentifier(DIL),
699 CalleeName, Reader->getRemapper());
700}
701
702/// Returns a vector of FunctionSamples that are the indirect call targets
703/// of \p Inst. The vector is sorted by the total number of samples. Stores
704/// the total call count of the indirect call in \p Sum.
705std::vector<const FunctionSamples *>
706SampleProfileLoader::findIndirectCallFunctionSamples(
707 const Instruction &Inst, uint64_t &Sum) const {
708 const DILocation *DIL = Inst.getDebugLoc();
709 std::vector<const FunctionSamples *> R;
710
711 if (!DIL) {
712 return R;
713 }
714
715 auto FSCompare = [](const FunctionSamples *L, const FunctionSamples *R) {
716 assert(L && R && "Expect non-null FunctionSamples");
717 if (L->getHeadSamplesEstimate() != R->getHeadSamplesEstimate())
718 return L->getHeadSamplesEstimate() > R->getHeadSamplesEstimate();
719 return L->getGUID() < R->getGUID();
720 };
721
723 auto CalleeSamples =
724 ContextTracker->getIndirectCalleeContextSamplesFor(DIL);
725 if (CalleeSamples.empty())
726 return R;
727
728 // For CSSPGO, we only use target context profile's entry count
729 // as that already includes both inlined callee and non-inlined ones..
730 Sum = 0;
731 for (const auto *const FS : CalleeSamples) {
732 Sum += FS->getHeadSamplesEstimate();
733 R.push_back(FS);
734 }
735 llvm::sort(R, FSCompare);
736 return R;
737 }
738
739 const FunctionSamples *FS = findFunctionSamples(Inst);
740 if (FS == nullptr)
741 return R;
742
744 Sum = 0;
745 if (auto T = FS->findCallTargetMapAt(CallSite))
746 for (const auto &T_C : *T)
747 Sum += T_C.second;
748 if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(CallSite)) {
749 if (M->empty())
750 return R;
751 for (const auto &NameFS : *M) {
752 Sum += NameFS.second.getHeadSamplesEstimate();
753 R.push_back(&NameFS.second);
754 }
755 llvm::sort(R, FSCompare);
756 }
757 return R;
758}
759
760const FunctionSamples *
761SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
763 std::optional<PseudoProbe> Probe = extractProbe(Inst);
764 if (!Probe)
765 return nullptr;
766 }
767
768 const DILocation *DIL = Inst.getDebugLoc();
769 if (!DIL)
770 return Samples;
771
772 auto it = DILocation2SampleMap.try_emplace(DIL,nullptr);
773 if (it.second) {
775 it.first->second = ContextTracker->getContextSamplesFor(DIL);
776 else
777 it.first->second =
778 Samples->findFunctionSamples(DIL, Reader->getRemapper());
779 }
780 return it.first->second;
781}
782
783/// Check whether the indirect call promotion history of \p Inst allows
784/// the promotion for \p Candidate.
785/// If the profile count for the promotion candidate \p Candidate is
786/// NOMORE_ICP_MAGICNUM, it means \p Candidate has already been promoted
787/// for \p Inst. If we already have at least MaxNumPromotions
788/// NOMORE_ICP_MAGICNUM count values in the value profile of \p Inst, we
789/// cannot promote for \p Inst anymore.
790static bool doesHistoryAllowICP(const Instruction &Inst, StringRef Candidate) {
791 uint32_t NumVals = 0;
792 uint64_t TotalCount = 0;
793 auto ValueData =
794 getValueProfDataFromInst(Inst, IPVK_IndirectCallTarget, MaxNumPromotions,
795 NumVals, TotalCount, true);
796 // No valid value profile so no promoted targets have been recorded
797 // before. Ok to do ICP.
798 if (!ValueData)
799 return true;
800
801 unsigned NumPromoted = 0;
802 for (uint32_t I = 0; I < NumVals; I++) {
803 if (ValueData[I].Count != NOMORE_ICP_MAGICNUM)
804 continue;
805
806 // If the promotion candidate has NOMORE_ICP_MAGICNUM count in the
807 // metadata, it means the candidate has been promoted for this
808 // indirect call.
809 if (ValueData[I].Value == Function::getGUID(Candidate))
810 return false;
811 NumPromoted++;
812 // If already have MaxNumPromotions promotion, don't do it anymore.
813 if (NumPromoted == MaxNumPromotions)
814 return false;
815 }
816 return true;
817}
818
819/// Update indirect call target profile metadata for \p Inst.
820/// Usually \p Sum is the sum of counts of all the targets for \p Inst.
821/// If it is 0, it means updateIDTMetaData is used to mark a
822/// certain target to be promoted already. If it is not zero,
823/// we expect to use it to update the total count in the value profile.
824static void
826 const SmallVectorImpl<InstrProfValueData> &CallTargets,
827 uint64_t Sum) {
828 // Bail out early if MaxNumPromotions is zero.
829 // This prevents allocating an array of zero length below.
830 //
831 // Note `updateIDTMetaData` is called in two places so check
832 // `MaxNumPromotions` inside it.
833 if (MaxNumPromotions == 0)
834 return;
835 uint32_t NumVals = 0;
836 // OldSum is the existing total count in the value profile data.
837 uint64_t OldSum = 0;
838 auto ValueData = getValueProfDataFromInst(
839 Inst, IPVK_IndirectCallTarget, MaxNumPromotions, NumVals, OldSum, true);
840
841 DenseMap<uint64_t, uint64_t> ValueCountMap;
842 if (Sum == 0) {
843 assert((CallTargets.size() == 1 &&
844 CallTargets[0].Count == NOMORE_ICP_MAGICNUM) &&
845 "If sum is 0, assume only one element in CallTargets "
846 "with count being NOMORE_ICP_MAGICNUM");
847 // Initialize ValueCountMap with existing value profile data.
848 if (ValueData) {
849 for (uint32_t I = 0; I < NumVals; I++)
850 ValueCountMap[ValueData[I].Value] = ValueData[I].Count;
851 }
852 auto Pair =
853 ValueCountMap.try_emplace(CallTargets[0].Value, CallTargets[0].Count);
854 // If the target already exists in value profile, decrease the total
855 // count OldSum and reset the target's count to NOMORE_ICP_MAGICNUM.
856 if (!Pair.second) {
857 OldSum -= Pair.first->second;
858 Pair.first->second = NOMORE_ICP_MAGICNUM;
859 }
860 Sum = OldSum;
861 } else {
862 // Initialize ValueCountMap with existing NOMORE_ICP_MAGICNUM
863 // counts in the value profile.
864 if (ValueData) {
865 for (uint32_t I = 0; I < NumVals; I++) {
866 if (ValueData[I].Count == NOMORE_ICP_MAGICNUM)
867 ValueCountMap[ValueData[I].Value] = ValueData[I].Count;
868 }
869 }
870
871 for (const auto &Data : CallTargets) {
872 auto Pair = ValueCountMap.try_emplace(Data.Value, Data.Count);
873 if (Pair.second)
874 continue;
875 // The target represented by Data.Value has already been promoted.
876 // Keep the count as NOMORE_ICP_MAGICNUM in the profile and decrease
877 // Sum by Data.Count.
878 assert(Sum >= Data.Count && "Sum should never be less than Data.Count");
879 Sum -= Data.Count;
880 }
881 }
882
884 for (const auto &ValueCount : ValueCountMap) {
885 NewCallTargets.emplace_back(
886 InstrProfValueData{ValueCount.first, ValueCount.second});
887 }
888
889 llvm::sort(NewCallTargets,
890 [](const InstrProfValueData &L, const InstrProfValueData &R) {
891 if (L.Count != R.Count)
892 return L.Count > R.Count;
893 return L.Value > R.Value;
894 });
895
896 uint32_t MaxMDCount =
897 std::min(NewCallTargets.size(), static_cast<size_t>(MaxNumPromotions));
898 annotateValueSite(*Inst.getParent()->getParent()->getParent(), Inst,
899 NewCallTargets, Sum, IPVK_IndirectCallTarget, MaxMDCount);
900}
901
902/// Attempt to promote indirect call and also inline the promoted call.
903///
904/// \param F Caller function.
905/// \param Candidate ICP and inline candidate.
906/// \param SumOrigin Original sum of target counts for indirect call before
907/// promoting given candidate.
908/// \param Sum Prorated sum of remaining target counts for indirect call
909/// after promoting given candidate.
910/// \param InlinedCallSite Output vector for new call sites exposed after
911/// inlining.
912bool SampleProfileLoader::tryPromoteAndInlineCandidate(
913 Function &F, InlineCandidate &Candidate, uint64_t SumOrigin, uint64_t &Sum,
914 SmallVector<CallBase *, 8> *InlinedCallSite) {
915 // Bail out early if sample-loader inliner is disabled.
917 return false;
918
919 // Bail out early if MaxNumPromotions is zero.
920 // This prevents allocating an array of zero length in callees below.
921 if (MaxNumPromotions == 0)
922 return false;
923 auto CalleeFunctionName = Candidate.CalleeSamples->getFunction();
924 auto R = SymbolMap.find(CalleeFunctionName);
925 if (R == SymbolMap.end() || !R->second)
926 return false;
927
928 auto &CI = *Candidate.CallInstr;
929 if (!doesHistoryAllowICP(CI, R->second->getName()))
930 return false;
931
932 const char *Reason = "Callee function not available";
933 // R->getValue() != &F is to prevent promoting a recursive call.
934 // If it is a recursive call, we do not inline it as it could bloat
935 // the code exponentially. There is way to better handle this, e.g.
936 // clone the caller first, and inline the cloned caller if it is
937 // recursive. As llvm does not inline recursive calls, we will
938 // simply ignore it instead of handling it explicitly.
939 if (!R->second->isDeclaration() && R->second->getSubprogram() &&
940 R->second->hasFnAttribute("use-sample-profile") &&
941 R->second != &F && isLegalToPromote(CI, R->second, &Reason)) {
942 // For promoted target, set its value with NOMORE_ICP_MAGICNUM count
943 // in the value profile metadata so the target won't be promoted again.
944 SmallVector<InstrProfValueData, 1> SortedCallTargets = {InstrProfValueData{
945 Function::getGUID(R->second->getName()), NOMORE_ICP_MAGICNUM}};
946 updateIDTMetaData(CI, SortedCallTargets, 0);
947
948 auto *DI = &pgo::promoteIndirectCall(
949 CI, R->second, Candidate.CallsiteCount, Sum, false, ORE);
950 if (DI) {
951 Sum -= Candidate.CallsiteCount;
952 // Do not prorate the indirect callsite distribution since the original
953 // distribution will be used to scale down non-promoted profile target
954 // counts later. By doing this we lose track of the real callsite count
955 // for the leftover indirect callsite as a trade off for accurate call
956 // target counts.
957 // TODO: Ideally we would have two separate factors, one for call site
958 // counts and one is used to prorate call target counts.
959 // Do not update the promoted direct callsite distribution at this
960 // point since the original distribution combined with the callee profile
961 // will be used to prorate callsites from the callee if inlined. Once not
962 // inlined, the direct callsite distribution should be prorated so that
963 // the it will reflect the real callsite counts.
964 Candidate.CallInstr = DI;
965 if (isa<CallInst>(DI) || isa<InvokeInst>(DI)) {
966 bool Inlined = tryInlineCandidate(Candidate, InlinedCallSite);
967 if (!Inlined) {
968 // Prorate the direct callsite distribution so that it reflects real
969 // callsite counts.
971 *DI, static_cast<float>(Candidate.CallsiteCount) / SumOrigin);
972 }
973 return Inlined;
974 }
975 }
976 } else {
977 LLVM_DEBUG(dbgs() << "\nFailed to promote indirect call to "
979 Candidate.CallInstr->getName())<< " because "
980 << Reason << "\n");
981 }
982 return false;
983}
984
985bool SampleProfileLoader::shouldInlineColdCallee(CallBase &CallInst) {
987 return false;
988
990 if (Callee == nullptr)
991 return false;
992
994 GetAC, GetTLI);
995
996 if (Cost.isNever())
997 return false;
998
999 if (Cost.isAlways())
1000 return true;
1001
1002 return Cost.getCost() <= SampleColdCallSiteThreshold;
1003}
1004
1005void SampleProfileLoader::emitOptimizationRemarksForInlineCandidates(
1006 const SmallVectorImpl<CallBase *> &Candidates, const Function &F,
1007 bool Hot) {
1008 for (auto *I : Candidates) {
1009 Function *CalledFunction = I->getCalledFunction();
1010 if (CalledFunction) {
1011 ORE->emit(OptimizationRemarkAnalysis(getAnnotatedRemarkPassName(),
1012 "InlineAttempt", I->getDebugLoc(),
1013 I->getParent())
1014 << "previous inlining reattempted for "
1015 << (Hot ? "hotness: '" : "size: '")
1016 << ore::NV("Callee", CalledFunction) << "' into '"
1017 << ore::NV("Caller", &F) << "'");
1018 }
1019 }
1020}
1021
1022void SampleProfileLoader::findExternalInlineCandidate(
1023 CallBase *CB, const FunctionSamples *Samples,
1024 DenseSet<GlobalValue::GUID> &InlinedGUIDs, uint64_t Threshold) {
1025
1026 // If ExternalInlineAdvisor(ReplayInlineAdvisor) wants to inline an external
1027 // function make sure it's imported
1028 if (CB && getExternalInlineAdvisorShouldInline(*CB)) {
1029 // Samples may not exist for replayed function, if so
1030 // just add the direct GUID and move on
1031 if (!Samples) {
1032 InlinedGUIDs.insert(
1033 Function::getGUID(CB->getCalledFunction()->getName()));
1034 return;
1035 }
1036 // Otherwise, drop the threshold to import everything that we can
1037 Threshold = 0;
1038 }
1039
1040 // In some rare cases, call instruction could be changed after being pushed
1041 // into inline candidate queue, this is because earlier inlining may expose
1042 // constant propagation which can change indirect call to direct call. When
1043 // this happens, we may fail to find matching function samples for the
1044 // candidate later, even if a match was found when the candidate was enqueued.
1045 if (!Samples)
1046 return;
1047
1048 // For AutoFDO profile, retrieve candidate profiles by walking over
1049 // the nested inlinee profiles.
1051 // Set threshold to zero to honor pre-inliner decision.
1053 Threshold = 0;
1054 Samples->findInlinedFunctions(InlinedGUIDs, SymbolMap, Threshold);
1055 return;
1056 }
1057
1058 ContextTrieNode *Caller = ContextTracker->getContextNodeForProfile(Samples);
1059 std::queue<ContextTrieNode *> CalleeList;
1060 CalleeList.push(Caller);
1061 while (!CalleeList.empty()) {
1062 ContextTrieNode *Node = CalleeList.front();
1063 CalleeList.pop();
1064 FunctionSamples *CalleeSample = Node->getFunctionSamples();
1065 // For CSSPGO profile, retrieve candidate profile by walking over the
1066 // trie built for context profile. Note that also take call targets
1067 // even if callee doesn't have a corresponding context profile.
1068 if (!CalleeSample)
1069 continue;
1070
1071 // If pre-inliner decision is used, honor that for importing as well.
1072 bool PreInline =
1075 if (!PreInline && CalleeSample->getHeadSamplesEstimate() < Threshold)
1076 continue;
1077
1078 Function *Func = SymbolMap.lookup(CalleeSample->getFunction());
1079 // Add to the import list only when it's defined out of module.
1080 if (!Func || Func->isDeclaration())
1081 InlinedGUIDs.insert(CalleeSample->getGUID());
1082
1083 // Import hot CallTargets, which may not be available in IR because full
1084 // profile annotation cannot be done until backend compilation in ThinLTO.
1085 for (const auto &BS : CalleeSample->getBodySamples())
1086 for (const auto &TS : BS.second.getCallTargets())
1087 if (TS.second > Threshold) {
1088 const Function *Callee = SymbolMap.lookup(TS.first);
1089 if (!Callee || Callee->isDeclaration())
1090 InlinedGUIDs.insert(TS.first.getHashCode());
1091 }
1092
1093 // Import hot child context profile associted with callees. Note that this
1094 // may have some overlap with the call target loop above, but doing this
1095 // based child context profile again effectively allow us to use the max of
1096 // entry count and call target count to determine importing.
1097 for (auto &Child : Node->getAllChildContext()) {
1098 ContextTrieNode *CalleeNode = &Child.second;
1099 CalleeList.push(CalleeNode);
1100 }
1101 }
1102}
1103
1104/// Iteratively inline hot callsites of a function.
1105///
1106/// Iteratively traverse all callsites of the function \p F, so as to
1107/// find out callsites with corresponding inline instances.
1108///
1109/// For such callsites,
1110/// - If it is hot enough, inline the callsites and adds callsites of the callee
1111/// into the caller. If the call is an indirect call, first promote
1112/// it to direct call. Each indirect call is limited with a single target.
1113///
1114/// - If a callsite is not inlined, merge the its profile to the outline
1115/// version (if --sample-profile-merge-inlinee is true), or scale the
1116/// counters of standalone function based on the profile of inlined
1117/// instances (if --sample-profile-merge-inlinee is false).
1118///
1119/// Later passes may consume the updated profiles.
1120///
1121/// \param F function to perform iterative inlining.
1122/// \param InlinedGUIDs a set to be updated to include all GUIDs that are
1123/// inlined in the profiled binary.
1124///
1125/// \returns True if there is any inline happened.
1126bool SampleProfileLoader::inlineHotFunctions(
1127 Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
1128 // ProfAccForSymsInList is used in callsiteIsHot. The assertion makes sure
1129 // Profile symbol list is ignored when profile-sample-accurate is on.
1130 assert((!ProfAccForSymsInList ||
1132 !F.hasFnAttribute("profile-sample-accurate"))) &&
1133 "ProfAccForSymsInList should be false when profile-sample-accurate "
1134 "is enabled");
1135
1136 MapVector<CallBase *, const FunctionSamples *> LocalNotInlinedCallSites;
1137 bool Changed = false;
1138 bool LocalChanged = true;
1139 while (LocalChanged) {
1140 LocalChanged = false;
1142 for (auto &BB : F) {
1143 bool Hot = false;
1144 SmallVector<CallBase *, 10> AllCandidates;
1145 SmallVector<CallBase *, 10> ColdCandidates;
1146 for (auto &I : BB) {
1147 const FunctionSamples *FS = nullptr;
1148 if (auto *CB = dyn_cast<CallBase>(&I)) {
1149 if (!isa<IntrinsicInst>(I)) {
1150 if ((FS = findCalleeFunctionSamples(*CB))) {
1151 assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&
1152 "GUIDToFuncNameMap has to be populated");
1153 AllCandidates.push_back(CB);
1154 if (FS->getHeadSamplesEstimate() > 0 ||
1156 LocalNotInlinedCallSites.insert({CB, FS});
1157 if (callsiteIsHot(FS, PSI, ProfAccForSymsInList))
1158 Hot = true;
1159 else if (shouldInlineColdCallee(*CB))
1160 ColdCandidates.push_back(CB);
1161 } else if (getExternalInlineAdvisorShouldInline(*CB)) {
1162 AllCandidates.push_back(CB);
1163 }
1164 }
1165 }
1166 }
1167 if (Hot || ExternalInlineAdvisor) {
1168 CIS.insert(CIS.begin(), AllCandidates.begin(), AllCandidates.end());
1169 emitOptimizationRemarksForInlineCandidates(AllCandidates, F, true);
1170 } else {
1171 CIS.insert(CIS.begin(), ColdCandidates.begin(), ColdCandidates.end());
1172 emitOptimizationRemarksForInlineCandidates(ColdCandidates, F, false);
1173 }
1174 }
1175 for (CallBase *I : CIS) {
1176 Function *CalledFunction = I->getCalledFunction();
1177 InlineCandidate Candidate = {I, LocalNotInlinedCallSites.lookup(I),
1178 0 /* dummy count */,
1179 1.0 /* dummy distribution factor */};
1180 // Do not inline recursive calls.
1181 if (CalledFunction == &F)
1182 continue;
1183 if (I->isIndirectCall()) {
1184 uint64_t Sum;
1185 for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) {
1186 uint64_t SumOrigin = Sum;
1187 if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1188 findExternalInlineCandidate(I, FS, InlinedGUIDs,
1189 PSI->getOrCompHotCountThreshold());
1190 continue;
1191 }
1192 if (!callsiteIsHot(FS, PSI, ProfAccForSymsInList))
1193 continue;
1194
1195 Candidate = {I, FS, FS->getHeadSamplesEstimate(), 1.0};
1196 if (tryPromoteAndInlineCandidate(F, Candidate, SumOrigin, Sum)) {
1197 LocalNotInlinedCallSites.erase(I);
1198 LocalChanged = true;
1199 }
1200 }
1201 } else if (CalledFunction && CalledFunction->getSubprogram() &&
1202 !CalledFunction->isDeclaration()) {
1203 if (tryInlineCandidate(Candidate)) {
1204 LocalNotInlinedCallSites.erase(I);
1205 LocalChanged = true;
1206 }
1207 } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1208 findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
1209 InlinedGUIDs,
1210 PSI->getOrCompHotCountThreshold());
1211 }
1212 }
1213 Changed |= LocalChanged;
1214 }
1215
1216 // For CS profile, profile for not inlined context will be merged when
1217 // base profile is being retrieved.
1219 promoteMergeNotInlinedContextSamples(LocalNotInlinedCallSites, F);
1220 return Changed;
1221}
1222
1223bool SampleProfileLoader::tryInlineCandidate(
1224 InlineCandidate &Candidate, SmallVector<CallBase *, 8> *InlinedCallSites) {
1225 // Do not attempt to inline a candidate if
1226 // --disable-sample-loader-inlining is true.
1228 return false;
1229
1230 CallBase &CB = *Candidate.CallInstr;
1231 Function *CalledFunction = CB.getCalledFunction();
1232 assert(CalledFunction && "Expect a callee with definition");
1233 DebugLoc DLoc = CB.getDebugLoc();
1234 BasicBlock *BB = CB.getParent();
1235
1236 InlineCost Cost = shouldInlineCandidate(Candidate);
1237 if (Cost.isNever()) {
1238 ORE->emit(OptimizationRemarkAnalysis(getAnnotatedRemarkPassName(),
1239 "InlineFail", DLoc, BB)
1240 << "incompatible inlining");
1241 return false;
1242 }
1243
1244 if (!Cost)
1245 return false;
1246
1247 InlineFunctionInfo IFI(GetAC);
1248 IFI.UpdateProfile = false;
1249 InlineResult IR = InlineFunction(CB, IFI,
1250 /*MergeAttributes=*/true);
1251 if (!IR.isSuccess())
1252 return false;
1253
1254 // The call to InlineFunction erases I, so we can't pass it here.
1255 emitInlinedIntoBasedOnCost(*ORE, DLoc, BB, *CalledFunction, *BB->getParent(),
1256 Cost, true, getAnnotatedRemarkPassName());
1257
1258 // Now populate the list of newly exposed call sites.
1259 if (InlinedCallSites) {
1260 InlinedCallSites->clear();
1261 for (auto &I : IFI.InlinedCallSites)
1262 InlinedCallSites->push_back(I);
1263 }
1264
1266 ContextTracker->markContextSamplesInlined(Candidate.CalleeSamples);
1267 ++NumCSInlined;
1268
1269 // Prorate inlined probes for a duplicated inlining callsite which probably
1270 // has a distribution less than 100%. Samples for an inlinee should be
1271 // distributed among the copies of the original callsite based on each
1272 // callsite's distribution factor for counts accuracy. Note that an inlined
1273 // probe may come with its own distribution factor if it has been duplicated
1274 // in the inlinee body. The two factor are multiplied to reflect the
1275 // aggregation of duplication.
1276 if (Candidate.CallsiteDistribution < 1) {
1277 for (auto &I : IFI.InlinedCallSites) {
1278 if (std::optional<PseudoProbe> Probe = extractProbe(*I))
1279 setProbeDistributionFactor(*I, Probe->Factor *
1280 Candidate.CallsiteDistribution);
1281 }
1282 NumDuplicatedInlinesite++;
1283 }
1284
1285 return true;
1286}
1287
1288bool SampleProfileLoader::getInlineCandidate(InlineCandidate *NewCandidate,
1289 CallBase *CB) {
1290 assert(CB && "Expect non-null call instruction");
1291
1292 if (isa<IntrinsicInst>(CB))
1293 return false;
1294
1295 // Find the callee's profile. For indirect call, find hottest target profile.
1296 const FunctionSamples *CalleeSamples = findCalleeFunctionSamples(*CB);
1297 // If ExternalInlineAdvisor wants to inline this site, do so even
1298 // if Samples are not present.
1299 if (!CalleeSamples && !getExternalInlineAdvisorShouldInline(*CB))
1300 return false;
1301
1302 float Factor = 1.0;
1303 if (std::optional<PseudoProbe> Probe = extractProbe(*CB))
1304 Factor = Probe->Factor;
1305
1306 uint64_t CallsiteCount =
1307 CalleeSamples ? CalleeSamples->getHeadSamplesEstimate() * Factor : 0;
1308 *NewCandidate = {CB, CalleeSamples, CallsiteCount, Factor};
1309 return true;
1310}
1311
1312std::optional<InlineCost>
1313SampleProfileLoader::getExternalInlineAdvisorCost(CallBase &CB) {
1314 std::unique_ptr<InlineAdvice> Advice = nullptr;
1315 if (ExternalInlineAdvisor) {
1316 Advice = ExternalInlineAdvisor->getAdvice(CB);
1317 if (Advice) {
1318 if (!Advice->isInliningRecommended()) {
1319 Advice->recordUnattemptedInlining();
1320 return InlineCost::getNever("not previously inlined");
1321 }
1322 Advice->recordInlining();
1323 return InlineCost::getAlways("previously inlined");
1324 }
1325 }
1326
1327 return {};
1328}
1329
1330bool SampleProfileLoader::getExternalInlineAdvisorShouldInline(CallBase &CB) {
1331 std::optional<InlineCost> Cost = getExternalInlineAdvisorCost(CB);
1332 return Cost ? !!*Cost : false;
1333}
1334
1336SampleProfileLoader::shouldInlineCandidate(InlineCandidate &Candidate) {
1337 if (std::optional<InlineCost> ReplayCost =
1338 getExternalInlineAdvisorCost(*Candidate.CallInstr))
1339 return *ReplayCost;
1340 // Adjust threshold based on call site hotness, only do this for callsite
1341 // prioritized inliner because otherwise cost-benefit check is done earlier.
1342 int SampleThreshold = SampleColdCallSiteThreshold;
1344 if (Candidate.CallsiteCount > PSI->getHotCountThreshold())
1345 SampleThreshold = SampleHotCallSiteThreshold;
1346 else if (!ProfileSizeInline)
1347 return InlineCost::getNever("cold callsite");
1348 }
1349
1350 Function *Callee = Candidate.CallInstr->getCalledFunction();
1351 assert(Callee && "Expect a definition for inline candidate of direct call");
1352
1353 InlineParams Params = getInlineParams();
1354 // We will ignore the threshold from inline cost, so always get full cost.
1355 Params.ComputeFullInlineCost = true;
1357 // Checks if there is anything in the reachable portion of the callee at
1358 // this callsite that makes this inlining potentially illegal. Need to
1359 // set ComputeFullInlineCost, otherwise getInlineCost may return early
1360 // when cost exceeds threshold without checking all IRs in the callee.
1361 // The acutal cost does not matter because we only checks isNever() to
1362 // see if it is legal to inline the callsite.
1363 InlineCost Cost = getInlineCost(*Candidate.CallInstr, Callee, Params,
1364 GetTTI(*Callee), GetAC, GetTLI);
1365
1366 // Honor always inline and never inline from call analyzer
1367 if (Cost.isNever() || Cost.isAlways())
1368 return Cost;
1369
1370 // With CSSPGO, the preinliner in llvm-profgen can estimate global inline
1371 // decisions based on hotness as well as accurate function byte sizes for
1372 // given context using function/inlinee sizes from previous build. It
1373 // stores the decision in profile, and also adjust/merge context profile
1374 // aiming at better context-sensitive post-inline profile quality, assuming
1375 // all inline decision estimates are going to be honored by compiler. Here
1376 // we replay that inline decision under `sample-profile-use-preinliner`.
1377 // Note that we don't need to handle negative decision from preinliner as
1378 // context profile for not inlined calls are merged by preinliner already.
1379 if (UsePreInlinerDecision && Candidate.CalleeSamples) {
1380 // Once two node are merged due to promotion, we're losing some context
1381 // so the original context-sensitive preinliner decision should be ignored
1382 // for SyntheticContext.
1383 SampleContext &Context = Candidate.CalleeSamples->getContext();
1384 if (!Context.hasState(SyntheticContext) &&
1386 return InlineCost::getAlways("preinliner");
1387 }
1388
1389 // For old FDO inliner, we inline the call site if it is below hot threshold,
1390 // even if the function is hot based on sample profile data. This is to
1391 // prevent huge functions from being inlined.
1394 }
1395
1396 // Otherwise only use the cost from call analyzer, but overwite threshold with
1397 // Sample PGO threshold.
1398 return InlineCost::get(Cost.getCost(), SampleThreshold);
1399}
1400
1401bool SampleProfileLoader::inlineHotFunctionsWithPriority(
1402 Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
1403 // ProfAccForSymsInList is used in callsiteIsHot. The assertion makes sure
1404 // Profile symbol list is ignored when profile-sample-accurate is on.
1405 assert((!ProfAccForSymsInList ||
1407 !F.hasFnAttribute("profile-sample-accurate"))) &&
1408 "ProfAccForSymsInList should be false when profile-sample-accurate "
1409 "is enabled");
1410
1411 // Populating worklist with initial call sites from root inliner, along
1412 // with call site weights.
1413 CandidateQueue CQueue;
1414 InlineCandidate NewCandidate;
1415 for (auto &BB : F) {
1416 for (auto &I : BB) {
1417 auto *CB = dyn_cast<CallBase>(&I);
1418 if (!CB)
1419 continue;
1420 if (getInlineCandidate(&NewCandidate, CB))
1421 CQueue.push(NewCandidate);
1422 }
1423 }
1424
1425 // Cap the size growth from profile guided inlining. This is needed even
1426 // though cost of each inline candidate already accounts for callee size,
1427 // because with top-down inlining, we can grow inliner size significantly
1428 // with large number of smaller inlinees each pass the cost check.
1430 "Max inline size limit should not be smaller than min inline size "
1431 "limit.");
1432 unsigned SizeLimit = F.getInstructionCount() * ProfileInlineGrowthLimit;
1433 SizeLimit = std::min(SizeLimit, (unsigned)ProfileInlineLimitMax);
1434 SizeLimit = std::max(SizeLimit, (unsigned)ProfileInlineLimitMin);
1435 if (ExternalInlineAdvisor)
1436 SizeLimit = std::numeric_limits<unsigned>::max();
1437
1438 MapVector<CallBase *, const FunctionSamples *> LocalNotInlinedCallSites;
1439
1440 // Perform iterative BFS call site prioritized inlining
1441 bool Changed = false;
1442 while (!CQueue.empty() && F.getInstructionCount() < SizeLimit) {
1443 InlineCandidate Candidate = CQueue.top();
1444 CQueue.pop();
1445 CallBase *I = Candidate.CallInstr;
1446 Function *CalledFunction = I->getCalledFunction();
1447
1448 if (CalledFunction == &F)
1449 continue;
1450 if (I->isIndirectCall()) {
1451 uint64_t Sum = 0;
1452 auto CalleeSamples = findIndirectCallFunctionSamples(*I, Sum);
1453 uint64_t SumOrigin = Sum;
1454 Sum *= Candidate.CallsiteDistribution;
1455 unsigned ICPCount = 0;
1456 for (const auto *FS : CalleeSamples) {
1457 // TODO: Consider disable pre-lTO ICP for MonoLTO as well
1458 if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1459 findExternalInlineCandidate(I, FS, InlinedGUIDs,
1460 PSI->getOrCompHotCountThreshold());
1461 continue;
1462 }
1463 uint64_t EntryCountDistributed =
1464 FS->getHeadSamplesEstimate() * Candidate.CallsiteDistribution;
1465 // In addition to regular inline cost check, we also need to make sure
1466 // ICP isn't introducing excessive speculative checks even if individual
1467 // target looks beneficial to promote and inline. That means we should
1468 // only do ICP when there's a small number dominant targets.
1469 if (ICPCount >= ProfileICPRelativeHotnessSkip &&
1470 EntryCountDistributed * 100 < SumOrigin * ProfileICPRelativeHotness)
1471 break;
1472 // TODO: Fix CallAnalyzer to handle all indirect calls.
1473 // For indirect call, we don't run CallAnalyzer to get InlineCost
1474 // before actual inlining. This is because we could see two different
1475 // types from the same definition, which makes CallAnalyzer choke as
1476 // it's expecting matching parameter type on both caller and callee
1477 // side. See example from PR18962 for the triggering cases (the bug was
1478 // fixed, but we generate different types).
1479 if (!PSI->isHotCount(EntryCountDistributed))
1480 break;
1481 SmallVector<CallBase *, 8> InlinedCallSites;
1482 // Attach function profile for promoted indirect callee, and update
1483 // call site count for the promoted inline candidate too.
1484 Candidate = {I, FS, EntryCountDistributed,
1485 Candidate.CallsiteDistribution};
1486 if (tryPromoteAndInlineCandidate(F, Candidate, SumOrigin, Sum,
1487 &InlinedCallSites)) {
1488 for (auto *CB : InlinedCallSites) {
1489 if (getInlineCandidate(&NewCandidate, CB))
1490 CQueue.emplace(NewCandidate);
1491 }
1492 ICPCount++;
1493 Changed = true;
1494 } else if (!ContextTracker) {
1495 LocalNotInlinedCallSites.insert({I, FS});
1496 }
1497 }
1498 } else if (CalledFunction && CalledFunction->getSubprogram() &&
1499 !CalledFunction->isDeclaration()) {
1500 SmallVector<CallBase *, 8> InlinedCallSites;
1501 if (tryInlineCandidate(Candidate, &InlinedCallSites)) {
1502 for (auto *CB : InlinedCallSites) {
1503 if (getInlineCandidate(&NewCandidate, CB))
1504 CQueue.emplace(NewCandidate);
1505 }
1506 Changed = true;
1507 } else if (!ContextTracker) {
1508 LocalNotInlinedCallSites.insert({I, Candidate.CalleeSamples});
1509 }
1510 } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1511 findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
1512 InlinedGUIDs,
1513 PSI->getOrCompHotCountThreshold());
1514 }
1515 }
1516
1517 if (!CQueue.empty()) {
1518 if (SizeLimit == (unsigned)ProfileInlineLimitMax)
1519 ++NumCSInlinedHitMaxLimit;
1520 else if (SizeLimit == (unsigned)ProfileInlineLimitMin)
1521 ++NumCSInlinedHitMinLimit;
1522 else
1523 ++NumCSInlinedHitGrowthLimit;
1524 }
1525
1526 // For CS profile, profile for not inlined context will be merged when
1527 // base profile is being retrieved.
1529 promoteMergeNotInlinedContextSamples(LocalNotInlinedCallSites, F);
1530 return Changed;
1531}
1532
1533void SampleProfileLoader::promoteMergeNotInlinedContextSamples(
1535 const Function &F) {
1536 // Accumulate not inlined callsite information into notInlinedSamples
1537 for (const auto &Pair : NonInlinedCallSites) {
1538 CallBase *I = Pair.first;
1539 Function *Callee = I->getCalledFunction();
1540 if (!Callee || Callee->isDeclaration())
1541 continue;
1542
1543 ORE->emit(
1544 OptimizationRemarkAnalysis(getAnnotatedRemarkPassName(), "NotInline",
1545 I->getDebugLoc(), I->getParent())
1546 << "previous inlining not repeated: '" << ore::NV("Callee", Callee)
1547 << "' into '" << ore::NV("Caller", &F) << "'");
1548
1549 ++NumCSNotInlined;
1550 const FunctionSamples *FS = Pair.second;
1551 if (FS->getTotalSamples() == 0 && FS->getHeadSamplesEstimate() == 0) {
1552 continue;
1553 }
1554
1555 // Do not merge a context that is already duplicated into the base profile.
1556 if (FS->getContext().hasAttribute(sampleprof::ContextDuplicatedIntoBase))
1557 continue;
1558
1559 if (ProfileMergeInlinee) {
1560 // A function call can be replicated by optimizations like callsite
1561 // splitting or jump threading and the replicates end up sharing the
1562 // sample nested callee profile instead of slicing the original
1563 // inlinee's profile. We want to do merge exactly once by filtering out
1564 // callee profiles with a non-zero head sample count.
1565 if (FS->getHeadSamples() == 0) {
1566 // Use entry samples as head samples during the merge, as inlinees
1567 // don't have head samples.
1568 const_cast<FunctionSamples *>(FS)->addHeadSamples(
1569 FS->getHeadSamplesEstimate());
1570
1571 // Note that we have to do the merge right after processing function.
1572 // This allows OutlineFS's profile to be used for annotation during
1573 // top-down processing of functions' annotation.
1574 FunctionSamples *OutlineFS = Reader->getSamplesFor(*Callee);
1575 // If outlined function does not exist in the profile, add it to a
1576 // separate map so that it does not rehash the original profile.
1577 if (!OutlineFS)
1578 OutlineFS = &OutlineFunctionSamples[
1580 OutlineFS->merge(*FS, 1);
1581 // Set outlined profile to be synthetic to not bias the inliner.
1582 OutlineFS->SetContextSynthetic();
1583 }
1584 } else {
1585 auto pair =
1586 notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0});
1587 pair.first->second.entryCount += FS->getHeadSamplesEstimate();
1588 }
1589 }
1590}
1591
1592/// Returns the sorted CallTargetMap \p M by count in descending order.
1596 for (const auto &I : SampleRecord::SortCallTargets(M)) {
1597 R.emplace_back(
1598 InstrProfValueData{I.first.getHashCode(), I.second});
1599 }
1600 return R;
1601}
1602
1603// Generate MD_prof metadata for every branch instruction using the
1604// edge weights computed during propagation.
1605void SampleProfileLoader::generateMDProfMetadata(Function &F) {
1606 // Generate MD_prof metadata for every branch instruction using the
1607 // edge weights computed during propagation.
1608 LLVM_DEBUG(dbgs() << "\nPropagation complete. Setting branch weights\n");
1609 LLVMContext &Ctx = F.getContext();
1610 MDBuilder MDB(Ctx);
1611 for (auto &BI : F) {
1612 BasicBlock *BB = &BI;
1613
1614 if (BlockWeights[BB]) {
1615 for (auto &I : *BB) {
1616 if (!isa<CallInst>(I) && !isa<InvokeInst>(I))
1617 continue;
1618 if (!cast<CallBase>(I).getCalledFunction()) {
1619 const DebugLoc &DLoc = I.getDebugLoc();
1620 if (!DLoc)
1621 continue;
1622 const DILocation *DIL = DLoc;
1623 const FunctionSamples *FS = findFunctionSamples(I);
1624 if (!FS)
1625 continue;
1628 FS->findCallTargetMapAt(CallSite);
1629 if (!T || T.get().empty())
1630 continue;
1632 // Prorate the callsite counts based on the pre-ICP distribution
1633 // factor to reflect what is already done to the callsite before
1634 // ICP, such as calliste cloning.
1635 if (std::optional<PseudoProbe> Probe = extractProbe(I)) {
1636 if (Probe->Factor < 1)
1637 T = SampleRecord::adjustCallTargets(T.get(), Probe->Factor);
1638 }
1639 }
1640 SmallVector<InstrProfValueData, 2> SortedCallTargets =
1642 uint64_t Sum = 0;
1643 for (const auto &C : T.get())
1644 Sum += C.second;
1645 // With CSSPGO all indirect call targets are counted torwards the
1646 // original indirect call site in the profile, including both
1647 // inlined and non-inlined targets.
1649 if (const FunctionSamplesMap *M =
1650 FS->findFunctionSamplesMapAt(CallSite)) {
1651 for (const auto &NameFS : *M)
1652 Sum += NameFS.second.getHeadSamplesEstimate();
1653 }
1654 }
1655 if (Sum)
1656 updateIDTMetaData(I, SortedCallTargets, Sum);
1657 else if (OverwriteExistingWeights)
1658 I.setMetadata(LLVMContext::MD_prof, nullptr);
1659 } else if (!isa<IntrinsicInst>(&I)) {
1660 setBranchWeights(I, {static_cast<uint32_t>(BlockWeights[BB])},
1661 /*IsExpected=*/false);
1662 }
1663 }
1665 // Set profile metadata (possibly annotated by LTO prelink) to zero or
1666 // clear it for cold code.
1667 for (auto &I : *BB) {
1668 if (isa<CallInst>(I) || isa<InvokeInst>(I)) {
1669 if (cast<CallBase>(I).isIndirectCall()) {
1670 I.setMetadata(LLVMContext::MD_prof, nullptr);
1671 } else {
1672 setBranchWeights(I, {uint32_t(0)}, /*IsExpected=*/false);
1673 }
1674 }
1675 }
1676 }
1677
1678 Instruction *TI = BB->getTerminator();
1679 if (TI->getNumSuccessors() == 1)
1680 continue;
1681 if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI) &&
1682 !isa<IndirectBrInst>(TI))
1683 continue;
1684
1685 DebugLoc BranchLoc = TI->getDebugLoc();
1686 LLVM_DEBUG(dbgs() << "\nGetting weights for branch at line "
1687 << ((BranchLoc) ? Twine(BranchLoc.getLine())
1688 : Twine("<UNKNOWN LOCATION>"))
1689 << ".\n");
1691 uint32_t MaxWeight = 0;
1692 Instruction *MaxDestInst;
1693 // Since profi treats multiple edges (multiway branches) as a single edge,
1694 // we need to distribute the computed weight among the branches. We do
1695 // this by evenly splitting the edge weight among destinations.
1697 std::vector<uint64_t> EdgeIndex;
1699 EdgeIndex.resize(TI->getNumSuccessors());
1700 for (unsigned I = 0; I < TI->getNumSuccessors(); ++I) {
1701 const BasicBlock *Succ = TI->getSuccessor(I);
1702 EdgeIndex[I] = EdgeMultiplicity[Succ];
1703 EdgeMultiplicity[Succ]++;
1704 }
1705 }
1706 for (unsigned I = 0; I < TI->getNumSuccessors(); ++I) {
1707 BasicBlock *Succ = TI->getSuccessor(I);
1708 Edge E = std::make_pair(BB, Succ);
1709 uint64_t Weight = EdgeWeights[E];
1710 LLVM_DEBUG(dbgs() << "\t"; printEdgeWeight(dbgs(), E));
1711 // Use uint32_t saturated arithmetic to adjust the incoming weights,
1712 // if needed. Sample counts in profiles are 64-bit unsigned values,
1713 // but internally branch weights are expressed as 32-bit values.
1714 if (Weight > std::numeric_limits<uint32_t>::max()) {
1715 LLVM_DEBUG(dbgs() << " (saturated due to uint32_t overflow)\n");
1716 Weight = std::numeric_limits<uint32_t>::max();
1717 }
1718 if (!SampleProfileUseProfi) {
1719 // Weight is added by one to avoid propagation errors introduced by
1720 // 0 weights.
1721 Weights.push_back(static_cast<uint32_t>(
1722 Weight == std::numeric_limits<uint32_t>::max() ? Weight
1723 : Weight + 1));
1724 } else {
1725 // Profi creates proper weights that do not require "+1" adjustments but
1726 // we evenly split the weight among branches with the same destination.
1727 uint64_t W = Weight / EdgeMultiplicity[Succ];
1728 // Rounding up, if needed, so that first branches are hotter.
1729 if (EdgeIndex[I] < Weight % EdgeMultiplicity[Succ])
1730 W++;
1731 Weights.push_back(static_cast<uint32_t>(W));
1732 }
1733 if (Weight != 0) {
1734 if (Weight > MaxWeight) {
1735 MaxWeight = Weight;
1736 MaxDestInst = Succ->getFirstNonPHIOrDbgOrLifetime();
1737 }
1738 }
1739 }
1740
1741 misexpect::checkExpectAnnotations(*TI, Weights, /*IsFrontend=*/false);
1742
1743 uint64_t TempWeight;
1744 // Only set weights if there is at least one non-zero weight.
1745 // In any other case, let the analyzer set weights.
1746 // Do not set weights if the weights are present unless under
1747 // OverwriteExistingWeights. In ThinLTO, the profile annotation is done
1748 // twice. If the first annotation already set the weights, the second pass
1749 // does not need to set it. With OverwriteExistingWeights, Blocks with zero
1750 // weight should have their existing metadata (possibly annotated by LTO
1751 // prelink) cleared.
1752 if (MaxWeight > 0 &&
1753 (!TI->extractProfTotalWeight(TempWeight) || OverwriteExistingWeights)) {
1754 LLVM_DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n");
1755 setBranchWeights(*TI, Weights, /*IsExpected=*/false);
1756 ORE->emit([&]() {
1757 return OptimizationRemark(DEBUG_TYPE, "PopularDest", MaxDestInst)
1758 << "most popular destination for conditional branches at "
1759 << ore::NV("CondBranchesLoc", BranchLoc);
1760 });
1761 } else {
1763 TI->setMetadata(LLVMContext::MD_prof, nullptr);
1764 LLVM_DEBUG(dbgs() << "CLEARED. All branch weights are zero.\n");
1765 } else {
1766 LLVM_DEBUG(dbgs() << "SKIPPED. All branch weights are zero.\n");
1767 }
1768 }
1769 }
1770}
1771
1772/// Once all the branch weights are computed, we emit the MD_prof
1773/// metadata on BB using the computed values for each of its branches.
1774///
1775/// \param F The function to query.
1776///
1777/// \returns true if \p F was modified. Returns false, otherwise.
1778bool SampleProfileLoader::emitAnnotations(Function &F) {
1779 bool Changed = false;
1780
1782 LLVM_DEBUG({
1783 if (!ProbeManager->getDesc(F))
1784 dbgs() << "Probe descriptor missing for Function " << F.getName()
1785 << "\n";
1786 });
1787
1788 if (ProbeManager->profileIsValid(F, *Samples)) {
1789 ++NumMatchedProfile;
1790 } else {
1791 ++NumMismatchedProfile;
1792 LLVM_DEBUG(
1793 dbgs() << "Profile is invalid due to CFG mismatch for Function "
1794 << F.getName() << "\n");
1796 return false;
1797 }
1798 } else {
1799 if (getFunctionLoc(F) == 0)
1800 return false;
1801
1802 LLVM_DEBUG(dbgs() << "Line number for the first instruction in "
1803 << F.getName() << ": " << getFunctionLoc(F) << "\n");
1804 }
1805
1806 DenseSet<GlobalValue::GUID> InlinedGUIDs;
1808 Changed |= inlineHotFunctionsWithPriority(F, InlinedGUIDs);
1809 else
1810 Changed |= inlineHotFunctions(F, InlinedGUIDs);
1811
1812 Changed |= computeAndPropagateWeights(F, InlinedGUIDs);
1813
1814 if (Changed)
1815 generateMDProfMetadata(F);
1816
1817 emitCoverageRemarks(F);
1818 return Changed;
1819}
1820
1821std::unique_ptr<ProfiledCallGraph>
1822SampleProfileLoader::buildProfiledCallGraph(Module &M) {
1823 std::unique_ptr<ProfiledCallGraph> ProfiledCG;
1825 ProfiledCG = std::make_unique<ProfiledCallGraph>(*ContextTracker);
1826 else
1827 ProfiledCG = std::make_unique<ProfiledCallGraph>(Reader->getProfiles());
1828
1829 // Add all functions into the profiled call graph even if they are not in
1830 // the profile. This makes sure functions missing from the profile still
1831 // gets a chance to be processed.
1832 for (Function &F : M) {
1834 continue;
1835 ProfiledCG->addProfiledFunction(
1837 }
1838
1839 return ProfiledCG;
1840}
1841
1842std::vector<Function *>
1843SampleProfileLoader::buildFunctionOrder(Module &M, LazyCallGraph &CG) {
1844 std::vector<Function *> FunctionOrderList;
1845 FunctionOrderList.reserve(M.size());
1846
1848 errs() << "WARNING: -use-profiled-call-graph ignored, should be used "
1849 "together with -sample-profile-top-down-load.\n";
1850
1851 if (!ProfileTopDownLoad) {
1852 if (ProfileMergeInlinee) {
1853 // Disable ProfileMergeInlinee if profile is not loaded in top down order,
1854 // because the profile for a function may be used for the profile
1855 // annotation of its outline copy before the profile merging of its
1856 // non-inlined inline instances, and that is not the way how
1857 // ProfileMergeInlinee is supposed to work.
1858 ProfileMergeInlinee = false;
1859 }
1860
1861 for (Function &F : M)
1863 FunctionOrderList.push_back(&F);
1864 return FunctionOrderList;
1865 }
1866
1868 !UseProfiledCallGraph.getNumOccurrences())) {
1869 // Use profiled call edges to augment the top-down order. There are cases
1870 // that the top-down order computed based on the static call graph doesn't
1871 // reflect real execution order. For example
1872 //
1873 // 1. Incomplete static call graph due to unknown indirect call targets.
1874 // Adjusting the order by considering indirect call edges from the
1875 // profile can enable the inlining of indirect call targets by allowing
1876 // the caller processed before them.
1877 // 2. Mutual call edges in an SCC. The static processing order computed for
1878 // an SCC may not reflect the call contexts in the context-sensitive
1879 // profile, thus may cause potential inlining to be overlooked. The
1880 // function order in one SCC is being adjusted to a top-down order based
1881 // on the profile to favor more inlining. This is only a problem with CS
1882 // profile.
1883 // 3. Transitive indirect call edges due to inlining. When a callee function
1884 // (say B) is inlined into a caller function (say A) in LTO prelink,
1885 // every call edge originated from the callee B will be transferred to
1886 // the caller A. If any transferred edge (say A->C) is indirect, the
1887 // original profiled indirect edge B->C, even if considered, would not
1888 // enforce a top-down order from the caller A to the potential indirect
1889 // call target C in LTO postlink since the inlined callee B is gone from
1890 // the static call graph.
1891 // 4. #3 can happen even for direct call targets, due to functions defined
1892 // in header files. A header function (say A), when included into source
1893 // files, is defined multiple times but only one definition survives due
1894 // to ODR. Therefore, the LTO prelink inlining done on those dropped
1895 // definitions can be useless based on a local file scope. More
1896 // importantly, the inlinee (say B), once fully inlined to a
1897 // to-be-dropped A, will have no profile to consume when its outlined
1898 // version is compiled. This can lead to a profile-less prelink
1899 // compilation for the outlined version of B which may be called from
1900 // external modules. while this isn't easy to fix, we rely on the
1901 // postlink AutoFDO pipeline to optimize B. Since the survived copy of
1902 // the A can be inlined in its local scope in prelink, it may not exist
1903 // in the merged IR in postlink, and we'll need the profiled call edges
1904 // to enforce a top-down order for the rest of the functions.
1905 //
1906 // Considering those cases, a profiled call graph completely independent of
1907 // the static call graph is constructed based on profile data, where
1908 // function objects are not even needed to handle case #3 and case 4.
1909 //
1910 // Note that static callgraph edges are completely ignored since they
1911 // can be conflicting with profiled edges for cyclic SCCs and may result in
1912 // an SCC order incompatible with profile-defined one. Using strictly
1913 // profile order ensures a maximum inlining experience. On the other hand,
1914 // static call edges are not so important when they don't correspond to a
1915 // context in the profile.
1916
1917 std::unique_ptr<ProfiledCallGraph> ProfiledCG = buildProfiledCallGraph(M);
1918 scc_iterator<ProfiledCallGraph *> CGI = scc_begin(ProfiledCG.get());
1919 while (!CGI.isAtEnd()) {
1920 auto Range = *CGI;
1921 if (SortProfiledSCC) {
1922 // Sort nodes in one SCC based on callsite hotness.
1924 Range = *SI;
1925 }
1926 for (auto *Node : Range) {
1927 Function *F = SymbolMap.lookup(Node->Name);
1928 if (F && !skipProfileForFunction(*F))
1929 FunctionOrderList.push_back(F);
1930 }
1931 ++CGI;
1932 }
1933 } else {
1934 CG.buildRefSCCs();
1935 for (LazyCallGraph::RefSCC &RC : CG.postorder_ref_sccs()) {
1936 for (LazyCallGraph::SCC &C : RC) {
1937 for (LazyCallGraph::Node &N : C) {
1938 Function &F = N.getFunction();
1940 FunctionOrderList.push_back(&F);
1941 }
1942 }
1943 }
1944 }
1945
1946 std::reverse(FunctionOrderList.begin(), FunctionOrderList.end());
1947
1948 LLVM_DEBUG({
1949 dbgs() << "Function processing order:\n";
1950 for (auto F : FunctionOrderList) {
1951 dbgs() << F->getName() << "\n";
1952 }
1953 });
1954
1955 return FunctionOrderList;
1956}
1957
1958bool SampleProfileLoader::doInitialization(Module &M,
1960 auto &Ctx = M.getContext();
1961
1962 auto ReaderOrErr = SampleProfileReader::create(
1963 Filename, Ctx, *FS, FSDiscriminatorPass::Base, RemappingFilename);
1964 if (std::error_code EC = ReaderOrErr.getError()) {
1965 std::string Msg = "Could not open profile: " + EC.message();
1966 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
1967 return false;
1968 }
1969 Reader = std::move(ReaderOrErr.get());
1970 Reader->setSkipFlatProf(LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink);
1971 // set module before reading the profile so reader may be able to only
1972 // read the function profiles which are used by the current module.
1973 Reader->setModule(&M);
1974 if (std::error_code EC = Reader->read()) {
1975 std::string Msg = "profile reading failed: " + EC.message();
1976 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
1977 return false;
1978 }
1979
1980 PSL = Reader->getProfileSymbolList();
1981
1982 // While profile-sample-accurate is on, ignore symbol list.
1983 ProfAccForSymsInList =
1985 if (ProfAccForSymsInList) {
1986 NamesInProfile.clear();
1987 GUIDsInProfile.clear();
1988 if (auto NameTable = Reader->getNameTable()) {
1990 for (auto Name : *NameTable)
1991 GUIDsInProfile.insert(Name.getHashCode());
1992 } else {
1993 for (auto Name : *NameTable)
1994 NamesInProfile.insert(Name.stringRef());
1995 }
1996 }
1997 CoverageTracker.setProfAccForSymsInList(true);
1998 }
1999
2000 if (FAM && !ProfileInlineReplayFile.empty()) {
2001 ExternalInlineAdvisor = getReplayInlineAdvisor(
2002 M, *FAM, Ctx, /*OriginalAdvisor=*/nullptr,
2007 /*EmitRemarks=*/false, InlineContext{LTOPhase, InlinePass::ReplaySampleProfileInliner});
2008 }
2009
2010 // Apply tweaks if context-sensitive or probe-based profile is available.
2011 if (Reader->profileIsCS() || Reader->profileIsPreInlined() ||
2012 Reader->profileIsProbeBased()) {
2013 if (!UseIterativeBFIInference.getNumOccurrences())
2015 if (!SampleProfileUseProfi.getNumOccurrences())
2016 SampleProfileUseProfi = true;
2017 if (!EnableExtTspBlockPlacement.getNumOccurrences())
2019 // Enable priority-base inliner and size inline by default for CSSPGO.
2020 if (!ProfileSizeInline.getNumOccurrences())
2021 ProfileSizeInline = true;
2022 if (!CallsitePrioritizedInline.getNumOccurrences())
2024 // For CSSPGO, we also allow recursive inline to best use context profile.
2025 if (!AllowRecursiveInline.getNumOccurrences())
2026 AllowRecursiveInline = true;
2027
2028 if (Reader->profileIsPreInlined()) {
2029 if (!UsePreInlinerDecision.getNumOccurrences())
2030 UsePreInlinerDecision = true;
2031 }
2032
2033 // Enable stale profile matching by default for probe-based profile.
2034 // Currently the matching relies on if the checksum mismatch is detected,
2035 // which is currently only available for pseudo-probe mode. Removing the
2036 // checksum check could cause regressions for some cases, so further tuning
2037 // might be needed if we want to enable it for all cases.
2038 if (Reader->profileIsProbeBased() &&
2039 !SalvageStaleProfile.getNumOccurrences()) {
2040 SalvageStaleProfile = true;
2041 }
2042
2043 if (!Reader->profileIsCS()) {
2044 // Non-CS profile should be fine without a function size budget for the
2045 // inliner since the contexts in the profile are either all from inlining
2046 // in the prevoius build or pre-computed by the preinliner with a size
2047 // cap, thus they are bounded.
2048 if (!ProfileInlineLimitMin.getNumOccurrences())
2049 ProfileInlineLimitMin = std::numeric_limits<unsigned>::max();
2050 if (!ProfileInlineLimitMax.getNumOccurrences())
2051 ProfileInlineLimitMax = std::numeric_limits<unsigned>::max();
2052 }
2053 }
2054
2055 if (Reader->profileIsCS()) {
2056 // Tracker for profiles under different context
2057 ContextTracker = std::make_unique<SampleContextTracker>(
2058 Reader->getProfiles(), &GUIDToFuncNameMap);
2059 }
2060
2061 // Load pseudo probe descriptors for probe-based function samples.
2062 if (Reader->profileIsProbeBased()) {
2063 ProbeManager = std::make_unique<PseudoProbeManager>(M);
2064 if (!ProbeManager->moduleIsProbed(M)) {
2065 const char *Msg =
2066 "Pseudo-probe-based profile requires SampleProfileProbePass";
2067 Ctx.diagnose(DiagnosticInfoSampleProfile(M.getModuleIdentifier(), Msg,
2068 DS_Warning));
2069 return false;
2070 }
2071 }
2072
2075 MatchingManager = std::make_unique<SampleProfileMatcher>(
2076 M, *Reader, ProbeManager.get(), LTOPhase);
2077 }
2078
2079 return true;
2080}
2081
2082// Note that this is a module-level check. Even if one module is errored out,
2083// the entire build will be errored out. However, the user could make big
2084// changes to functions in single module but those changes might not be
2085// performance significant to the whole binary. Therefore, to avoid those false
2086// positives, we select a reasonable big set of hot functions that are supposed
2087// to be globally performance significant, only compute and check the mismatch
2088// within those functions. The function selection is based on two criteria:
2089// 1) The function is hot enough, which is tuned by a hotness-based
2090// flag(HotFuncCutoffForStalenessError). 2) The num of function is large enough
2091// which is tuned by the MinfuncsForStalenessError flag.
2092bool SampleProfileLoader::rejectHighStalenessProfile(
2093 Module &M, ProfileSummaryInfo *PSI, const SampleProfileMap &Profiles) {
2095 "Only support for probe-based profile");
2096 uint64_t TotalHotFunc = 0;
2097 uint64_t NumMismatchedFunc = 0;
2098 for (const auto &I : Profiles) {
2099 const auto &FS = I.second;
2100 const auto *FuncDesc = ProbeManager->getDesc(FS.getGUID());
2101 if (!FuncDesc)
2102 continue;
2103
2104 // Use a hotness-based threshold to control the function selection.
2106 FS.getTotalSamples()))
2107 continue;
2108
2109 TotalHotFunc++;
2110 if (ProbeManager->profileIsHashMismatched(*FuncDesc, FS))
2111 NumMismatchedFunc++;
2112 }
2113 // Make sure that the num of selected function is not too small to distinguish
2114 // from the user's benign changes.
2115 if (TotalHotFunc < MinfuncsForStalenessError)
2116 return false;
2117
2118 // Finally check the mismatch percentage against the threshold.
2119 if (NumMismatchedFunc * 100 >=
2120 TotalHotFunc * PrecentMismatchForStalenessError) {
2121 auto &Ctx = M.getContext();
2122 const char *Msg =
2123 "The input profile significantly mismatches current source code. "
2124 "Please recollect profile to avoid performance regression.";
2125 Ctx.diagnose(DiagnosticInfoSampleProfile(M.getModuleIdentifier(), Msg));
2126 return true;
2127 }
2128 return false;
2129}
2130
2131void SampleProfileLoader::removePseudoProbeInsts(Module &M) {
2132 for (auto &F : M) {
2133 std::vector<Instruction *> InstsToDel;
2134 for (auto &BB : F) {
2135 for (auto &I : BB) {
2136 if (isa<PseudoProbeInst>(&I))
2137 InstsToDel.push_back(&I);
2138 }
2139 }
2140 for (auto *I : InstsToDel)
2141 I->eraseFromParent();
2142 }
2143}
2144
2145bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
2146 ProfileSummaryInfo *_PSI,
2147 LazyCallGraph &CG) {
2148 GUIDToFuncNameMapper Mapper(M, *Reader, GUIDToFuncNameMap);
2149
2150 PSI = _PSI;
2151 if (M.getProfileSummary(/* IsCS */ false) == nullptr) {
2152 M.setProfileSummary(Reader->getSummary().getMD(M.getContext()),
2154 PSI->refresh();
2155 }
2156
2158 rejectHighStalenessProfile(M, PSI, Reader->getProfiles()))
2159 return false;
2160
2161 // Compute the total number of samples collected in this profile.
2162 for (const auto &I : Reader->getProfiles())
2163 TotalCollectedSamples += I.second.getTotalSamples();
2164
2165 auto Remapper = Reader->getRemapper();
2166 // Populate the symbol map.
2167 for (const auto &N_F : M.getValueSymbolTable()) {
2168 StringRef OrigName = N_F.getKey();
2169 Function *F = dyn_cast<Function>(N_F.getValue());
2170 if (F == nullptr || OrigName.empty())
2171 continue;
2172 SymbolMap[FunctionId(OrigName)] = F;
2174 if (OrigName != NewName && !NewName.empty()) {
2175 auto r = SymbolMap.emplace(FunctionId(NewName), F);
2176 // Failiing to insert means there is already an entry in SymbolMap,
2177 // thus there are multiple functions that are mapped to the same
2178 // stripped name. In this case of name conflicting, set the value
2179 // to nullptr to avoid confusion.
2180 if (!r.second)
2181 r.first->second = nullptr;
2182 OrigName = NewName;
2183 }
2184 // Insert the remapped names into SymbolMap.
2185 if (Remapper) {
2186 if (auto MapName = Remapper->lookUpNameInProfile(OrigName)) {
2187 if (*MapName != OrigName && !MapName->empty())
2188 SymbolMap.emplace(FunctionId(*MapName), F);
2189 }
2190 }
2191 }
2192 assert(SymbolMap.count(FunctionId()) == 0 &&
2193 "No empty StringRef should be added in SymbolMap");
2194
2197 MatchingManager->runOnModule();
2198 MatchingManager->clearMatchingData();
2199 }
2200
2201 bool retval = false;
2202 for (auto *F : buildFunctionOrder(M, CG)) {
2203 assert(!F->isDeclaration());
2204 clearFunctionData();
2205 retval |= runOnFunction(*F, AM);
2206 }
2207
2208 // Account for cold calls not inlined....
2210 for (const std::pair<Function *, NotInlinedProfileInfo> &pair :
2211 notInlinedCallInfo)
2212 updateProfileCallee(pair.first, pair.second.entryCount);
2213
2215 removePseudoProbeInsts(M);
2216
2217 return retval;
2218}
2219
2220bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) {
2221 LLVM_DEBUG(dbgs() << "\n\nProcessing Function " << F.getName() << "\n");
2222 DILocation2SampleMap.clear();
2223 // By default the entry count is initialized to -1, which will be treated
2224 // conservatively by getEntryCount as the same as unknown (None). This is
2225 // to avoid newly added code to be treated as cold. If we have samples
2226 // this will be overwritten in emitAnnotations.
2227 uint64_t initialEntryCount = -1;
2228
2229 ProfAccForSymsInList = ProfileAccurateForSymsInList && PSL;
2230 if (ProfileSampleAccurate || F.hasFnAttribute("profile-sample-accurate")) {
2231 // initialize all the function entry counts to 0. It means all the
2232 // functions without profile will be regarded as cold.
2233 initialEntryCount = 0;
2234 // profile-sample-accurate is a user assertion which has a higher precedence
2235 // than symbol list. When profile-sample-accurate is on, ignore symbol list.
2236 ProfAccForSymsInList = false;
2237 }
2238 CoverageTracker.setProfAccForSymsInList(ProfAccForSymsInList);
2239
2240 // PSL -- profile symbol list include all the symbols in sampled binary.
2241 // If ProfileAccurateForSymsInList is enabled, PSL is used to treat
2242 // old functions without samples being cold, without having to worry
2243 // about new and hot functions being mistakenly treated as cold.
2244 if (ProfAccForSymsInList) {
2245 // Initialize the entry count to 0 for functions in the list.
2246 if (PSL->contains(F.getName()))
2247 initialEntryCount = 0;
2248
2249 // Function in the symbol list but without sample will be regarded as
2250 // cold. To minimize the potential negative performance impact it could
2251 // have, we want to be a little conservative here saying if a function
2252 // shows up in the profile, no matter as outline function, inline instance
2253 // or call targets, treat the function as not being cold. This will handle
2254 // the cases such as most callsites of a function are inlined in sampled
2255 // binary but not inlined in current build (because of source code drift,
2256 // imprecise debug information, or the callsites are all cold individually
2257 // but not cold accumulatively...), so the outline function showing up as
2258 // cold in sampled binary will actually not be cold after current build.
2261 GUIDsInProfile.count(Function::getGUID(CanonName))) ||
2262 (!FunctionSamples::UseMD5 && NamesInProfile.count(CanonName)))
2263 initialEntryCount = -1;
2264 }
2265
2266 // Initialize entry count when the function has no existing entry
2267 // count value.
2268 if (!F.getEntryCount())
2269 F.setEntryCount(ProfileCount(initialEntryCount, Function::PCT_Real));
2270 std::unique_ptr<OptimizationRemarkEmitter> OwnedORE;
2271 if (AM) {
2272 auto &FAM =
2274 .getManager();
2276 } else {
2277 OwnedORE = std::make_unique<OptimizationRemarkEmitter>(&F);
2278 ORE = OwnedORE.get();
2279 }
2280
2282 Samples = ContextTracker->getBaseSamplesFor(F);
2283 else {
2284 Samples = Reader->getSamplesFor(F);
2285 // Try search in previously inlined functions that were split or duplicated
2286 // into base.
2287 if (!Samples) {
2289 auto It = OutlineFunctionSamples.find(FunctionId(CanonName));
2290 if (It != OutlineFunctionSamples.end()) {
2291 Samples = &It->second;
2292 } else if (auto Remapper = Reader->getRemapper()) {
2293 if (auto RemppedName = Remapper->lookUpNameInProfile(CanonName)) {
2294 It = OutlineFunctionSamples.find(FunctionId(*RemppedName));
2295 if (It != OutlineFunctionSamples.end())
2296 Samples = &It->second;
2297 }
2298 }
2299 }
2300 }
2301
2302 if (Samples && !Samples->empty())
2303 return emitAnnotations(F);
2304 return false;
2305}
2307 std::string File, std::string RemappingFile, ThinOrFullLTOPhase LTOPhase,
2309 : ProfileFileName(File), ProfileRemappingFileName(RemappingFile),
2310 LTOPhase(LTOPhase), FS(std::move(FS)) {}
2311
2316
2317 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
2319 };
2320 auto GetTTI = [&](Function &F) -> TargetTransformInfo & {
2322 };
2323 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
2325 };
2326
2327 if (!FS)
2329
2330 SampleProfileLoader SampleLoader(
2331 ProfileFileName.empty() ? SampleProfileFile : ProfileFileName,
2332 ProfileRemappingFileName.empty() ? SampleProfileRemappingFile
2333 : ProfileRemappingFileName,
2334 LTOPhase, FS, GetAssumptionCache, GetTTI, GetTLI);
2335
2336 if (!SampleLoader.doInitialization(M, &FAM))
2337 return PreservedAnalyses::all();
2338
2341 if (!SampleLoader.runOnModule(M, &AM, PSI, CG))
2342 return PreservedAnalyses::all();
2343
2344 return PreservedAnalyses::none();
2345}
This file defines the StringMap class.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:686
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
std::string Name
static bool runOnFunction(Function &F, bool PostInlining)
Provides ErrorOr<T> smart pointer.
static cl::opt< unsigned > SizeLimit("eif-limit", cl::init(6), cl::Hidden, cl::desc("Size limit in Hexagon early if-conversion"))
LVReader * CurrentReader
Definition: LVReader.cpp:153
Implements a lazy call graph analysis and related passes for the new pass manager.
Legalize the Machine IR a function s Machine IR
Definition: Legalizer.cpp:81
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file implements a map that provides insertion order iteration.
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
Module.h This file contains the declarations for the Module class.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
FunctionAnalysisManager FAM
This header defines various interfaces for pass management in LLVM.
This file defines the PriorityQueue class.
This file contains the declarations for profiling metadata utility functions.
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file provides the interface for context-sensitive profile tracker used by CSSPGO.
This file provides the interface for the sampled PGO profile loader base implementation.
This file provides the utility functions for the sampled PGO loader base implementation.
This file provides the interface for SampleProfileMatcher.
This file provides the interface for the pseudo probe implementation for AutoFDO.
static cl::opt< std::string > SampleProfileFile("sample-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile file loaded by -sample-profile"), cl::Hidden)
static cl::opt< unsigned > MinfuncsForStalenessError("min-functions-for-staleness-error", cl::Hidden, cl::init(50), cl::desc("Skip the check if the number of hot functions is smaller than " "the specified number."))
static cl::opt< bool > ProfileSampleBlockAccurate("profile-sample-block-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " "branches and calls as having 0 samples. Otherwise, treat " "them conservatively as unknown. "))
static cl::opt< unsigned > PrecentMismatchForStalenessError("precent-mismatch-for-staleness-error", cl::Hidden, cl::init(80), cl::desc("Reject the profile if the mismatch percent is higher than the " "given number."))
static cl::opt< bool > RemoveProbeAfterProfileAnnotation("sample-profile-remove-probe", cl::Hidden, cl::init(false), cl::desc("Remove pseudo-probe after sample profile annotation."))
static cl::opt< unsigned > MaxNumPromotions("sample-profile-icp-max-prom", cl::init(3), cl::Hidden, cl::desc("Max number of promotions for a single indirect " "call callsite in sample profile loader"))
static cl::opt< ReplayInlinerSettings::Fallback > ProfileInlineReplayFallback("sample-profile-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How sample profile inline replay treats sites that don't come " "from the replay. Original: defers to original advisor, " "AlwaysInline: inline all sites not in replay, NeverInline: " "inline no sites not in replay"), cl::Hidden)
static cl::opt< bool > OverwriteExistingWeights("overwrite-existing-weights", cl::Hidden, cl::init(false), cl::desc("Ignore existing branch weights on IR and always overwrite."))
static void updateIDTMetaData(Instruction &Inst, const SmallVectorImpl< InstrProfValueData > &CallTargets, uint64_t Sum)
Update indirect call target profile metadata for Inst.
static cl::opt< bool > AnnotateSampleProfileInlinePhase("annotate-sample-profile-inline-phase", cl::Hidden, cl::init(false), cl::desc("Annotate LTO phase (prelink / postlink), or main (no LTO) for " "sample-profile inline pass name."))
static cl::opt< std::string > ProfileInlineReplayFile("sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by inlining from sample profile loader."), cl::Hidden)
static cl::opt< bool > ProfileMergeInlinee("sample-profile-merge-inlinee", cl::Hidden, cl::init(true), cl::desc("Merge past inlinee's profile to outline version if sample " "profile loader decided not to inline a call site. It will " "only be enabled when top-down order of profile loading is " "enabled. "))
cl::opt< bool > PersistProfileStaleness("persist-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute stale profile statistical metrics and write it into the " "native object file(.llvm_stats section)."))
static bool doesHistoryAllowICP(const Instruction &Inst, StringRef Candidate)
Check whether the indirect call promotion history of Inst allows the promotion for Candidate.
static SmallVector< InstrProfValueData, 2 > GetSortedValueDataFromCallTargets(const SampleRecord::CallTargetMap &M)
Returns the sorted CallTargetMap M by count in descending order.
#define CSINLINE_DEBUG
static cl::opt< bool > UseProfiledCallGraph("use-profiled-call-graph", cl::init(true), cl::Hidden, cl::desc("Process functions in a top-down order " "defined by the profiled call graph when " "-sample-profile-top-down-load is on."))
static cl::opt< ReplayInlinerSettings::Scope > ProfileInlineReplayScope("sample-profile-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during sample profile inlining."), cl::Hidden)
static cl::opt< unsigned > ProfileICPRelativeHotness("sample-profile-icp-relative-hotness", cl::Hidden, cl::init(25), cl::desc("Relative hotness percentage threshold for indirect " "call promotion in proirity-based sample profile loader inlining."))
Function::ProfileCount ProfileCount
static cl::opt< unsigned > ProfileICPRelativeHotnessSkip("sample-profile-icp-relative-hotness-skip", cl::Hidden, cl::init(1), cl::desc("Skip relative hotness check for ICP up to given number of targets."))
cl::opt< bool > ReportProfileStaleness("report-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute and report stale profile statistical metrics."))
static cl::opt< bool > UsePreInlinerDecision("sample-profile-use-preinliner", cl::Hidden, cl::desc("Use the preinliner decisions stored in profile context."))
static cl::opt< bool > ProfileAccurateForSymsInList("profile-accurate-for-symsinlist", cl::Hidden, cl::init(true), cl::desc("For symbols in profile symbol list, regard their profiles to " "be accurate. It may be overriden by profile-sample-accurate. "))
#define DEBUG_TYPE
static cl::opt< bool > DisableSampleLoaderInlining("disable-sample-loader-inlining", cl::Hidden, cl::init(false), cl::desc("If true, artifically skip inline transformation in sample-loader " "pass, and merge (or scale) profiles (as configured by " "--sample-profile-merge-inlinee)."))
static cl::opt< bool > ProfileSizeInline("sample-profile-inline-size", cl::Hidden, cl::init(false), cl::desc("Inline cold call sites in profile loader if it's beneficial " "for code size."))
cl::opt< bool > SalvageStaleProfile("salvage-stale-profile", cl::Hidden, cl::init(false), cl::desc("Salvage stale profile by fuzzy matching and use the remapped " "location for sample profile query."))
static cl::opt< bool > ProfileTopDownLoad("sample-profile-top-down-load", cl::Hidden, cl::init(true), cl::desc("Do profile annotation and inlining for functions in top-down " "order of call graph during sample profile loading. It only " "works for new pass manager. "))
static cl::opt< bool > ProfileSampleAccurate("profile-sample-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " "callsite and function as having 0 samples. Otherwise, treat " "un-sampled callsites and functions conservatively as unknown. "))
static cl::opt< bool > AllowRecursiveInline("sample-profile-recursive-inline", cl::Hidden, cl::desc("Allow sample loader inliner to inline recursive calls."))
static cl::opt< CallSiteFormat::Format > ProfileInlineReplayFormat("sample-profile-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How sample profile inline replay file is formatted"), cl::Hidden)
static cl::opt< std::string > SampleProfileRemappingFile("sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden)
static cl::opt< unsigned > HotFuncCutoffForStalenessError("hot-func-cutoff-for-staleness-error", cl::Hidden, cl::init(800000), cl::desc("A function is considered hot for staleness error check if its " "total sample count is above the specified percentile"))
static cl::opt< bool > CallsitePrioritizedInline("sample-profile-prioritized-inline", cl::Hidden, cl::desc("Use call site prioritized inlining for sample profile loader." "Currently only CSSPGO is supported."))
This file provides the interface for the sampled PGO loader pass.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This pass exposes codegen information to IR-level passes.
Defines the virtual file system interface vfs::FileSystem.
Value * RHS
Value * LHS
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:242
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:394
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:208
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
This class represents a function call, abstracting a target machine's calling convention.
Debug location.
A debug info location.
Definition: DebugLoc.h:33
unsigned getLine() const
Definition: DebugLoc.cpp:24
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:202
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Definition: DenseMap.h:235
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Diagnostic information for the sample profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
Represents either an error or a value T.
Definition: ErrorOr.h:56
Class to represent profile counts.
Definition: Function.h:279
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1830
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:286
Represents the cost of inlining a function.
Definition: InlineCost.h:90
static InlineCost getNever(const char *Reason, std::optional< CostBenefitPair > CostBenefit=std::nullopt)
Definition: InlineCost.h:131
static InlineCost getAlways(const char *Reason, std::optional< CostBenefitPair > CostBenefit=std::nullopt)
Definition: InlineCost.h:126
static InlineCost get(int Cost, int Threshold, int StaticBonus=0)
Definition: InlineCost.h:120
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:202
InlineResult is basically true or false.
Definition: InlineCost.h:180
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:552
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:470
bool extractProfTotalWeight(uint64_t &TotalVal) const
Retrieve total raw weight values of a branch.
Definition: Metadata.cpp:1744
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1635
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
An analysis pass which computes the call graph for a module.
A node in the call graph.
A RefSCC of the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
iterator_range< postorder_ref_scc_iterator > postorder_ref_sccs()
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
VectorType::iterator erase(typename VectorType::iterator Iterator)
Remove the element given by Iterator.
Definition: MapVector.h:193
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: MapVector.h:141
ValueT lookup(const KeyT &Key) const
Definition: MapVector.h:110
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Diagnostic information for optimization analysis remarks.
Diagnostic information for applied optimization remarks.
PostDominatorTree Class - Concrete subclass of DominatorTree that is used to compute the post-dominat...
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
PriorityQueue - This class behaves like std::priority_queue and provides a few additional convenience...
Definition: PriorityQueue.h:28
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
void refresh()
If no summary is present, attempt to refresh.
bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C) const
Returns true if count C is considered hot with regard to a given hot percentile cutoff value.
Sample profile inference pass.
void computeDominanceAndLoopInfo(FunctionT &F)
virtual ErrorOr< uint64_t > getInstWeight(const InstructionT &Inst)
Get the weight for an instruction.
virtual const FunctionSamples * findFunctionSamples(const InstructionT &I) const
Get the FunctionSamples for an instruction.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
SampleProfileLoaderPass(std::string File="", std::string RemappingFile="", ThinOrFullLTOPhase LTOPhase=ThinOrFullLTOPhase::None, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:818
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
StringSet - A wrapper for StringMap that provides set-like functionality.
Definition: StringSet.h:23
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
LLVM Value Representation.
Definition: Value.h:74
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
const ParentTy * getParent() const
Definition: ilist_node.h:32
This class represents a function that is read from a sample profile.
Definition: FunctionId.h:36
Representation of the samples collected for a function.
Definition: SampleProf.h:744
void findInlinedFunctions(DenseSet< GlobalValue::GUID > &S, const HashKeyMap< std::unordered_map, FunctionId, Function * > &SymbolMap, uint64_t Threshold) const
Recursively traverses all children, if the total sample count of the corresponding function is no les...
Definition: SampleProf.h:1036
FunctionId getFunction() const
Return the function name.
Definition: SampleProf.h:1069
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes.
Definition: SampleProf.h:1085
SampleContext & getContext() const
Definition: SampleProf.h:1185
sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight=1)
Merge the samples in Other into this one.
Definition: SampleProf.h:996
static LineLocation getCallSiteIdentifier(const DILocation *DIL, bool ProfileIsFS=false)
Returns a unique call site identifier for a given debug location of a call instruction.
Definition: SampleProf.cpp:221
uint64_t getHeadSamplesEstimate() const
Return an estimate of the sample count of the function entry basic block.
Definition: SampleProf.h:947
uint64_t getGUID() const
Return the GUID of the context's name.
Definition: SampleProf.h:1204
const BodySampleMap & getBodySamples() const
Return all the samples collected in the body of the function.
Definition: SampleProf.h:971
static bool UseMD5
Whether the profile uses MD5 to represent string.
Definition: SampleProf.h:1190
This class is a wrapper to associative container MapT<KeyT, ValueT> using the hash value of the origi...
Definition: HashKeyMap.h:53
bool hasState(ContextStateMask S)
Definition: SampleProf.h:611
bool hasAttribute(ContextAttributeMask A)
Definition: SampleProf.h:607
This class provides operator overloads to the map container using MD5 as the key type,...
Definition: SampleProf.h:1306
Sample-based profile reader.
static ErrorOr< std::unique_ptr< SampleProfileReader > > create(StringRef Filename, LLVMContext &C, vfs::FileSystem &FS, FSDiscriminatorPass P=FSDiscriminatorPass::Base, StringRef RemapFilename="")
Create a sample profile reader appropriate to the file format.
std::unordered_map< FunctionId, uint64_t > CallTargetMap
Definition: SampleProf.h:338
static const SortedCallTargetSet SortCallTargets(const CallTargetMap &Targets)
Sort call targets in descending order of call frequency.
Definition: SampleProf.h:406
static const CallTargetMap adjustCallTargets(const CallTargetMap &Targets, float DistributionFactor)
Prorate call targets by a distribution factor.
Definition: SampleProf.h:415
Enumerate the SCCs of a directed graph in reverse topological order of the SCC DAG.
Definition: SCCIterator.h:49
bool isAtEnd() const
Direct loop termination test which is more efficient than comparison with end().
Definition: SCCIterator.h:113
Sort the nodes of a directed SCC in the decreasing order of the edge weights.
Definition: SCCIterator.h:253
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ FS
Definition: X86.h:206
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:711
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
void checkExpectAnnotations(Instruction &I, const ArrayRef< uint32_t > ExistingWeights, bool IsFrontend)
checkExpectAnnotations - compares PGO counters to the thresholds used for llvm.expect and warns if th...
Definition: MisExpect.cpp:203
DenseMap< SymbolStringPtr, ExecutorSymbolDef > SymbolMap
A map from symbol names (as SymbolStringPtrs) to JITSymbols (address/flags pairs).
Definition: Core.h:121
DiagnosticInfoOptimizationBase::Argument NV
CallBase & promoteIndirectCall(CallBase &CB, Function *F, uint64_t Count, uint64_t TotalCount, bool AttachProfToDirectCall, OptimizationRemarkEmitter *ORE)
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
static FunctionId getRepInFormat(StringRef Name)
Get the proper representation of a string according to whether the current Format uses MD5 to represe...
Definition: SampleProf.h:1292
std::map< FunctionId, FunctionSamples > FunctionSamplesMap
Definition: SampleProf.h:734
bool callsiteIsHot(const FunctionSamples *CallsiteFS, ProfileSummaryInfo *PSI, bool ProfAccForSymsInList)
Return true if the given callsite is hot wrt to hot cutoff threshold.
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1680
cl::opt< int > ProfileInlineLimitMin
bool succ_empty(const Instruction *I)
Definition: CFG.h:255
std::unique_ptr< InstrProfValueData[]> getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...
Definition: InstrProf.cpp:1372
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
Definition: SCCIterator.h:233
void setProbeDistributionFactor(Instruction &Inst, float Factor)
Definition: PseudoProbe.cpp:76
std::string AnnotateInlinePassName(InlineContext IC)
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
Definition: Pass.h:76
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
cl::opt< bool > SampleProfileUseProfi
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Definition: InstrProf.cpp:1273
void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1647
llvm::cl::opt< bool > UseIterativeBFIInference
std::optional< PseudoProbe > extractProbe(const Instruction &Inst)
Definition: PseudoProbe.cpp:56
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
cl::opt< int > SampleHotCallSiteThreshold
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void updateProfileCallee(Function *Callee, int64_t EntryDelta, const ValueMap< const Value *, WeakTrackingVH > *VMap=nullptr)
Updates profile information by adjusting the entry count by adding EntryDelta then scaling callsite i...
cl::opt< int > SampleColdCallSiteThreshold
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1849
@ DS_Warning
static bool skipProfileForFunction(const Function &F)
cl::opt< bool > SortProfiledSCC
cl::opt< int > ProfileInlineLimitMax
cl::opt< bool > EnableExtTspBlockPlacement
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
Definition: Metadata.h:57
cl::opt< int > ProfileInlineGrowthLimit
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
#define N
Used in the streaming interface as the general argument type.
A wrapper of binary function with basic blocks and jumps.
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Definition: InlineAdvisor.h:59
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:206
std::optional< bool > AllowRecursiveCall
Indicate whether we allow inlining for recursive call.
Definition: InlineCost.h:239
std::optional< bool > ComputeFullInlineCost
Compute inline cost even when the cost has exceeded the threshold.
Definition: InlineCost.h:233