LLVM 17.0.0git
ProfileSummaryInfo.cpp
Go to the documentation of this file.
1//===- ProfileSummaryInfo.cpp - Global profile summary information --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that provides access to the global profile summary
10// information.
11//
12//===----------------------------------------------------------------------===//
13
16#include "llvm/IR/BasicBlock.h"
18#include "llvm/IR/Module.h"
23#include <optional>
24using namespace llvm;
25
26// Knobs for profile summary based thresholds.
27namespace llvm {
34} // namespace llvm
35
37 "partial-profile", cl::Hidden, cl::init(false),
38 cl::desc("Specify the current profile is used as a partial profile."));
39
41 "scale-partial-sample-profile-working-set-size", cl::Hidden, cl::init(true),
43 "If true, scale the working set size of the partial sample profile "
44 "by the partial profile ratio to reflect the size of the program "
45 "being compiled."));
46
48 "partial-sample-profile-working-set-size-scale-factor", cl::Hidden,
49 cl::init(0.008),
50 cl::desc("The scale factor used to scale the working set size of the "
51 "partial sample profile along with the partial profile ratio. "
52 "This includes the factor of the profile counter per block "
53 "and the factor to scale the working set size to use the same "
54 "shared thresholds as PGO."));
55
56// The profile summary metadata may be attached either by the frontend or by
57// any backend passes (IR level instrumentation, for example). This method
58// checks if the Summary is null and if so checks if the summary metadata is now
59// available in the module and parses it to get the Summary object.
62 return;
63 // First try to get context sensitive ProfileSummary.
64 auto *SummaryMD = M->getProfileSummary(/* IsCS */ true);
65 if (SummaryMD)
66 Summary.reset(ProfileSummary::getFromMD(SummaryMD));
67
68 if (!hasProfileSummary()) {
69 // This will actually return PSK_Instr or PSK_Sample summary.
70 SummaryMD = M->getProfileSummary(/* IsCS */ false);
71 if (SummaryMD)
72 Summary.reset(ProfileSummary::getFromMD(SummaryMD));
73 }
74 if (!hasProfileSummary())
75 return;
76 computeThresholds();
77}
78
80 const CallBase &Call, BlockFrequencyInfo *BFI, bool AllowSynthetic) const {
81 assert((isa<CallInst>(Call) || isa<InvokeInst>(Call)) &&
82 "We can only get profile count for call/invoke instruction.");
83 if (hasSampleProfile()) {
84 // In sample PGO mode, check if there is a profile metadata on the
85 // instruction. If it is present, determine hotness solely based on that,
86 // since the sampled entry count may not be accurate. If there is no
87 // annotated on the instruction, return std::nullopt.
88 uint64_t TotalCount;
89 if (Call.extractProfTotalWeight(TotalCount))
90 return TotalCount;
91 return std::nullopt;
92 }
93 if (BFI)
94 return BFI->getBlockProfileCount(Call.getParent(), AllowSynthetic);
95 return std::nullopt;
96}
97
98/// Returns true if the function's entry is hot. If it returns false, it
99/// either means it is not hot or it is unknown whether it is hot or not (for
100/// example, no profile data is available).
102 if (!F || !hasProfileSummary())
103 return false;
104 auto FunctionCount = F->getEntryCount();
105 // FIXME: The heuristic used below for determining hotness is based on
106 // preliminary SPEC tuning for inliner. This will eventually be a
107 // convenience method that calls isHotCount.
108 return FunctionCount && isHotCount(FunctionCount->getCount());
109}
110
111/// Returns true if the function contains hot code. This can include a hot
112/// function entry count, hot basic block, or (in the case of Sample PGO)
113/// hot total call edge count.
114/// If it returns false, it either means it is not hot or it is unknown
115/// (for example, no profile data is available).
117 const Function *F, BlockFrequencyInfo &BFI) const {
118 if (!F || !hasProfileSummary())
119 return false;
120 if (auto FunctionCount = F->getEntryCount())
121 if (isHotCount(FunctionCount->getCount()))
122 return true;
123
124 if (hasSampleProfile()) {
125 uint64_t TotalCallCount = 0;
126 for (const auto &BB : *F)
127 for (const auto &I : BB)
128 if (isa<CallInst>(I) || isa<InvokeInst>(I))
129 if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr))
130 TotalCallCount += *CallCount;
131 if (isHotCount(TotalCallCount))
132 return true;
133 }
134 for (const auto &BB : *F)
135 if (isHotBlock(&BB, &BFI))
136 return true;
137 return false;
138}
139
140/// Returns true if the function only contains cold code. This means that
141/// the function entry and blocks are all cold, and (in the case of Sample PGO)
142/// the total call edge count is cold.
143/// If it returns false, it either means it is not cold or it is unknown
144/// (for example, no profile data is available).
146 const Function *F, BlockFrequencyInfo &BFI) const {
147 if (!F || !hasProfileSummary())
148 return false;
149 if (auto FunctionCount = F->getEntryCount())
150 if (!isColdCount(FunctionCount->getCount()))
151 return false;
152
153 if (hasSampleProfile()) {
154 uint64_t TotalCallCount = 0;
155 for (const auto &BB : *F)
156 for (const auto &I : BB)
157 if (isa<CallInst>(I) || isa<InvokeInst>(I))
158 if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr))
159 TotalCallCount += *CallCount;
160 if (!isColdCount(TotalCallCount))
161 return false;
162 }
163 for (const auto &BB : *F)
164 if (!isColdBlock(&BB, &BFI))
165 return false;
166 return true;
167}
168
170 assert(hasPartialSampleProfile() && "Expect partial sample profile");
171 return !F.getEntryCount();
172}
173
174template <bool isHot>
175bool ProfileSummaryInfo::isFunctionHotOrColdInCallGraphNthPercentile(
176 int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const {
177 if (!F || !hasProfileSummary())
178 return false;
179 if (auto FunctionCount = F->getEntryCount()) {
180 if (isHot &&
181 isHotCountNthPercentile(PercentileCutoff, FunctionCount->getCount()))
182 return true;
183 if (!isHot &&
184 !isColdCountNthPercentile(PercentileCutoff, FunctionCount->getCount()))
185 return false;
186 }
187 if (hasSampleProfile()) {
188 uint64_t TotalCallCount = 0;
189 for (const auto &BB : *F)
190 for (const auto &I : BB)
191 if (isa<CallInst>(I) || isa<InvokeInst>(I))
192 if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr))
193 TotalCallCount += *CallCount;
194 if (isHot && isHotCountNthPercentile(PercentileCutoff, TotalCallCount))
195 return true;
196 if (!isHot && !isColdCountNthPercentile(PercentileCutoff, TotalCallCount))
197 return false;
198 }
199 for (const auto &BB : *F) {
200 if (isHot && isHotBlockNthPercentile(PercentileCutoff, &BB, &BFI))
201 return true;
202 if (!isHot && !isColdBlockNthPercentile(PercentileCutoff, &BB, &BFI))
203 return false;
204 }
205 return !isHot;
206}
207
208// Like isFunctionHotInCallGraph but for a given cutoff.
210 int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const {
211 return isFunctionHotOrColdInCallGraphNthPercentile<true>(
212 PercentileCutoff, F, BFI);
213}
214
216 int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const {
217 return isFunctionHotOrColdInCallGraphNthPercentile<false>(
218 PercentileCutoff, F, BFI);
219}
220
221/// Returns true if the function's entry is a cold. If it returns false, it
222/// either means it is not cold or it is unknown whether it is cold or not (for
223/// example, no profile data is available).
225 if (!F)
226 return false;
227 if (F->hasFnAttribute(Attribute::Cold))
228 return true;
229 if (!hasProfileSummary())
230 return false;
231 auto FunctionCount = F->getEntryCount();
232 // FIXME: The heuristic used below for determining coldness is based on
233 // preliminary SPEC tuning for inliner. This will eventually be a
234 // convenience method that calls isHotCount.
235 return FunctionCount && isColdCount(FunctionCount->getCount());
236}
237
238/// Compute the hot and cold thresholds.
239void ProfileSummaryInfo::computeThresholds() {
240 auto &DetailedSummary = Summary->getDetailedSummary();
242 DetailedSummary, ProfileSummaryCutoffHot);
243 HotCountThreshold =
245 ColdCountThreshold =
247 assert(ColdCountThreshold <= HotCountThreshold &&
248 "Cold count threshold cannot exceed hot count threshold!");
250 HasHugeWorkingSetSize =
252 HasLargeWorkingSetSize =
254 } else {
255 // Scale the working set size of the partial sample profile to reflect the
256 // size of the program being compiled.
257 double PartialProfileRatio = Summary->getPartialProfileRatio();
258 uint64_t ScaledHotEntryNumCounts =
259 static_cast<uint64_t>(HotEntry.NumCounts * PartialProfileRatio *
261 HasHugeWorkingSetSize =
262 ScaledHotEntryNumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
263 HasLargeWorkingSetSize =
264 ScaledHotEntryNumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
265 }
266}
267
268std::optional<uint64_t>
269ProfileSummaryInfo::computeThreshold(int PercentileCutoff) const {
270 if (!hasProfileSummary())
271 return std::nullopt;
272 auto iter = ThresholdCache.find(PercentileCutoff);
273 if (iter != ThresholdCache.end()) {
274 return iter->second;
275 }
276 auto &DetailedSummary = Summary->getDetailedSummary();
277 auto &Entry = ProfileSummaryBuilder::getEntryForPercentile(DetailedSummary,
279 uint64_t CountThreshold = Entry.MinCount;
280 ThresholdCache[PercentileCutoff] = CountThreshold;
281 return CountThreshold;
282}
283
285 return HasHugeWorkingSetSize && *HasHugeWorkingSetSize;
286}
287
289 return HasLargeWorkingSetSize && *HasLargeWorkingSetSize;
290}
291
293 return HotCountThreshold && C >= *HotCountThreshold;
294}
295
297 return ColdCountThreshold && C <= *ColdCountThreshold;
298}
299
300template <bool isHot>
301bool ProfileSummaryInfo::isHotOrColdCountNthPercentile(int PercentileCutoff,
302 uint64_t C) const {
303 auto CountThreshold = computeThreshold(PercentileCutoff);
304 if (isHot)
305 return CountThreshold && C >= *CountThreshold;
306 else
307 return CountThreshold && C <= *CountThreshold;
308}
309
311 uint64_t C) const {
312 return isHotOrColdCountNthPercentile<true>(PercentileCutoff, C);
313}
314
316 uint64_t C) const {
317 return isHotOrColdCountNthPercentile<false>(PercentileCutoff, C);
318}
319
321 return HotCountThreshold.value_or(UINT64_MAX);
322}
323
325 return ColdCountThreshold.value_or(0);
326}
327
329 BlockFrequencyInfo *BFI) const {
330 auto Count = BFI->getBlockProfileCount(BB);
331 return Count && isHotCount(*Count);
332}
333
335 BlockFrequencyInfo *BFI) const {
336 auto Count = BFI->getBlockProfileCount(BB);
337 return Count && isColdCount(*Count);
338}
339
340template <bool isHot>
341bool ProfileSummaryInfo::isHotOrColdBlockNthPercentile(
342 int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const {
343 auto Count = BFI->getBlockProfileCount(BB);
344 if (isHot)
345 return Count && isHotCountNthPercentile(PercentileCutoff, *Count);
346 else
347 return Count && isColdCountNthPercentile(PercentileCutoff, *Count);
348}
349
351 int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const {
352 return isHotOrColdBlockNthPercentile<true>(PercentileCutoff, BB, BFI);
353}
354
356 int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const {
357 return isHotOrColdBlockNthPercentile<false>(PercentileCutoff, BB, BFI);
358}
359
361 BlockFrequencyInfo *BFI) const {
362 auto C = getProfileCount(CB, BFI);
363 return C && isHotCount(*C);
364}
365
367 BlockFrequencyInfo *BFI) const {
368 auto C = getProfileCount(CB, BFI);
369 if (C)
370 return isColdCount(*C);
371
372 // In SamplePGO, if the caller has been sampled, and there is no profile
373 // annotated on the callsite, we consider the callsite as cold.
374 return hasSampleProfile() && CB.getCaller()->hasProfileData();
375}
376
378 return hasProfileSummary() &&
379 Summary->getKind() == ProfileSummary::PSK_Sample &&
380 (PartialProfile || Summary->isPartialProfile());
381}
382
384 "Profile summary info", false, true)
385
387 : ImmutablePass(ID) {
389}
390
392 PSI.reset(new ProfileSummaryInfo(M));
393 return false;
394}
395
397 PSI.reset();
398 return false;
399}
400
401AnalysisKey ProfileSummaryAnalysis::Key;
404 return ProfileSummaryInfo(M);
405}
406
410
411 OS << "Functions in " << M.getName() << " with hot/cold annotations: \n";
412 for (auto &F : M) {
413 OS << F.getName();
414 if (PSI.isFunctionEntryHot(&F))
415 OS << " :hot entry ";
416 else if (PSI.isFunctionEntryCold(&F))
417 OS << " :cold entry ";
418 OS << "\n";
419 }
420 return PreservedAnalyses::all();
421}
422
static cl::opt< unsigned > CountThreshold("hexagon-cext-threshold", cl::init(3), cl::Hidden, cl::desc("Minimum number of extenders to trigger replacement"))
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
static cl::opt< unsigned > PercentileCutoff("mfs-psi-cutoff", cl::desc("Percentile profile summary cutoff used to " "determine cold blocks. Unused if set to zero."), cl::init(999950), cl::Hidden)
Module.h This file contains the declarations for the Module class.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
static cl::opt< bool > PartialProfile("partial-profile", cl::Hidden, cl::init(false), cl::desc("Specify the current profile is used as a partial profile."))
static cl::opt< double > PartialSampleProfileWorkingSetSizeScaleFactor("partial-sample-profile-working-set-size-scale-factor", cl::Hidden, cl::init(0.008), cl::desc("The scale factor used to scale the working set size of the " "partial sample profile along with the partial profile ratio. " "This includes the factor of the profile counter per block " "and the factor to scale the working set size to use the same " "shared thresholds as PGO."))
cl::opt< bool > ScalePartialSampleProfileWorkingSetSize("scale-partial-sample-profile-working-set-size", cl::Hidden, cl::init(true), cl::desc("If true, scale the working set size of the partial sample profile " "by the partial profile ratio to reflect the size of the program " "being compiled."))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1186
Function * getCaller()
Helper to get the caller (the parent function).
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
iterator end()
Definition: DenseMap.h:84
bool hasProfileData(bool IncludeSynthetic=false) const
Return true if the function is annotated with profile data.
Definition: Function.h:289
ImmutablePass class - This class is used to provide information that does not need to be run.
Definition: Pass.h:279
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Metadata * getProfileSummary(bool IsCS) const
Returns profile summary metadata.
Definition: Module.cpp:643
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Result run(Module &M, ModuleAnalysisManager &)
static const ProfileSummaryEntry & getEntryForPercentile(const SummaryEntryVector &DS, uint64_t Percentile)
Find the summary entry for a desired percentile of counts.
static uint64_t getHotCountThreshold(const SummaryEntryVector &DS)
static uint64_t getColdCountThreshold(const SummaryEntryVector &DS)
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
bool doFinalization(Module &M) override
doFinalization - Virtual method overriden by subclasses to do any necessary clean up after all passes...
bool doInitialization(Module &M) override
doInitialization - Virtual method overridden by subclasses to do any necessary initialization before ...
Analysis providing profile information.
bool isFunctionEntryHot(const Function *F) const
Returns true if F has hot function entry.
bool isHotBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const
Returns true if BasicBlock BB is considered hot with regard to a given hot percentile cutoff value.
uint64_t getOrCompColdCountThreshold() const
Returns ColdCountThreshold if set.
bool hasProfileSummary() const
Returns true if profile summary is available.
bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI) const
Returns true if BasicBlock BB is considered hot.
bool isFunctionHotnessUnknown(const Function &F) const
Returns true if the hotness of F is unknown.
void refresh()
If no summary is present, attempt to refresh.
bool isColdBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const
Returns true if BasicBlock BB is considered cold with regard to a given cold percentile cutoff value.
std::optional< uint64_t > getProfileCount(const CallBase &CallInst, BlockFrequencyInfo *BFI, bool AllowSynthetic=false) const
Returns the profile count for CallInst.
bool hasSampleProfile() const
Returns true if module M has sample profile.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool isColdCountNthPercentile(int PercentileCutoff, uint64_t C) const
Returns true if count C is considered cold with regard to a given cold percentile cutoff value.
bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C) const
Returns true if count C is considered hot with regard to a given hot percentile cutoff value.
bool isFunctionColdInCallGraphNthPercentile(int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const
Returns true if F contains cold code with regard to a given cold percentile cutoff value.
bool hasPartialSampleProfile() const
Returns true if module M has partial-profile sample profile.
bool hasLargeWorkingSetSize() const
Returns true if the working set size of the code is considered large.
bool isColdCallSite(const CallBase &CB, BlockFrequencyInfo *BFI) const
Returns true if call site CB is considered cold.
bool isFunctionHotInCallGraphNthPercentile(int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const
Returns true if F contains hot code with regard to a given hot percentile cutoff value.
bool isFunctionHotInCallGraph(const Function *F, BlockFrequencyInfo &BFI) const
Returns true if F contains hot code.
bool isHotCallSite(const CallBase &CB, BlockFrequencyInfo *BFI) const
Returns true if the call site CB is considered hot.
bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI) const
Returns true if BasicBlock BB is considered cold.
bool isFunctionColdInCallGraph(const Function *F, BlockFrequencyInfo &BFI) const
Returns true if F contains only cold code.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
bool hasHugeWorkingSetSize() const
Returns true if the working set size of the code is considered huge.
uint64_t getOrCompHotCountThreshold() const
Returns HotCountThreshold if set.
bool isFunctionEntryCold(const Function *F) const
Returns true if F has cold function entry.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
static ProfileSummary * getFromMD(Metadata *MD)
Construct profile summary from metdata.
#define UINT64_MAX
Definition: DataTypes.h:77
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
cl::opt< int > ProfileSummaryHotCount
cl::opt< int > ProfileSummaryColdCount
cl::opt< int > ProfileSummaryCutoffCold
cl::opt< unsigned > ProfileSummaryLargeWorkingSetSizeThreshold
void initializeProfileSummaryInfoWrapperPassPass(PassRegistry &)
cl::opt< int > ProfileSummaryCutoffHot
cl::opt< unsigned > ProfileSummaryHugeWorkingSetSizeThreshold
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: PassManager.h:69