LLVM  13.0.0git
SampleProf.h
Go to the documentation of this file.
1 //===- SampleProf.h - Sampling profiling format support ---------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains common definitions used in the reading and writing of
10 // sample profile data.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_PROFILEDATA_SAMPLEPROF_H
15 #define LLVM_PROFILEDATA_SAMPLEPROF_H
16 
17 #include "llvm/ADT/DenseSet.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/StringMap.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/ADT/StringSet.h"
22 #include "llvm/IR/Function.h"
23 #include "llvm/IR/GlobalValue.h"
24 #include "llvm/IR/Module.h"
25 #include "llvm/Support/Allocator.h"
26 #include "llvm/Support/Debug.h"
27 #include "llvm/Support/ErrorOr.h"
30 #include <algorithm>
31 #include <cstdint>
32 #include <map>
33 #include <set>
34 #include <string>
35 #include <system_error>
36 #include <utility>
37 
38 namespace llvm {
39 
41 
42 enum class sampleprof_error {
43  success = 0,
44  bad_magic,
46  too_large,
47  truncated,
48  malformed,
59 };
60 
61 inline std::error_code make_error_code(sampleprof_error E) {
62  return std::error_code(static_cast<int>(E), sampleprof_category());
63 }
64 
66  sampleprof_error Result) {
67  // Prefer first error encountered as later errors may be secondary effects of
68  // the initial problem.
70  Result != sampleprof_error::success)
71  Accumulator = Result;
72  return Accumulator;
73 }
74 
75 } // end namespace llvm
76 
77 namespace std {
78 
79 template <>
80 struct is_error_code_enum<llvm::sampleprof_error> : std::true_type {};
81 
82 } // end namespace std
83 
84 namespace llvm {
85 namespace sampleprof {
86 
88  SPF_None = 0,
89  SPF_Text = 0x1,
91  SPF_GCC = 0x3,
93  SPF_Binary = 0xff
94 };
95 
96 static inline uint64_t SPMagic(SampleProfileFormat Format = SPF_Binary) {
97  return uint64_t('S') << (64 - 8) | uint64_t('P') << (64 - 16) |
98  uint64_t('R') << (64 - 24) | uint64_t('O') << (64 - 32) |
99  uint64_t('F') << (64 - 40) | uint64_t('4') << (64 - 48) |
100  uint64_t('2') << (64 - 56) | uint64_t(Format);
101 }
102 
103 /// Get the proper representation of a string according to whether the
104 /// current Format uses MD5 to represent the string.
105 static inline StringRef getRepInFormat(StringRef Name, bool UseMD5,
106  std::string &GUIDBuf) {
107  if (Name.empty())
108  return Name;
110  return UseMD5 ? StringRef(GUIDBuf) : Name;
111 }
112 
113 static inline uint64_t SPVersion() { return 103; }
114 
115 // Section Type used by SampleProfileExtBinaryBaseReader and
116 // SampleProfileExtBinaryBaseWriter. Never change the existing
117 // value of enum. Only append new ones.
118 enum SecType {
125  // marker for the first type of profile.
128 };
129 
130 static inline std::string getSecName(SecType Type) {
131  switch (Type) {
132  case SecInValid:
133  return "InvalidSection";
134  case SecProfSummary:
135  return "ProfileSummarySection";
136  case SecNameTable:
137  return "NameTableSection";
139  return "ProfileSymbolListSection";
140  case SecFuncOffsetTable:
141  return "FuncOffsetTableSection";
142  case SecFuncMetadata:
143  return "FunctionMetadata";
144  case SecLBRProfile:
145  return "LBRProfileSection";
146  }
147  llvm_unreachable("A SecType has no name for output");
148 }
149 
150 // Entry type of section header table used by SampleProfileExtBinaryBaseReader
151 // and SampleProfileExtBinaryBaseWriter.
154  uint64_t Flags;
155  uint64_t Offset;
156  uint64_t Size;
157  // The index indicating the location of the current entry in
158  // SectionHdrLayout table.
160 };
161 
162 // Flags common for all sections are defined here. In SecHdrTableEntry::Flags,
163 // common flags will be saved in the lower 32bits and section specific flags
164 // will be saved in the higher 32 bits.
165 enum class SecCommonFlags : uint32_t {
166  SecFlagInValid = 0,
167  SecFlagCompress = (1 << 0),
168  // Indicate the section contains only profile without context.
169  SecFlagFlat = (1 << 1)
170 };
171 
172 // Section specific flags are defined here.
173 // !!!Note: Everytime a new enum class is created here, please add
174 // a new check in verifySecFlag.
176  SecFlagInValid = 0,
177  SecFlagMD5Name = (1 << 0),
178  // Store MD5 in fixed length instead of ULEB128 so NameTable can be
179  // accessed like an array.
180  SecFlagFixedLengthMD5 = (1 << 1),
181  // Profile contains ".__uniq." suffix name. Compiler shouldn't strip
182  // the suffix when doing profile matching when seeing the flag.
183  SecFlagUniqSuffix = (1 << 2)
184 };
186  SecFlagInValid = 0,
187  /// SecFlagPartial means the profile is for common/shared code.
188  /// The common profile is usually merged from profiles collected
189  /// from running other targets.
190  SecFlagPartial = (1 << 0),
191  /// SecFlagContext means this is context-sensitive profile for
192  /// CSSPGO
193  SecFlagFullContext = (1 << 1),
194  /// SecFlagFSDiscriminator means this profile uses flow-sensitive
195  /// discriminators.
196  SecFlagFSDiscriminator = (1 << 2)
197 };
198 
200  SecFlagInvalid = 0,
201  SecFlagIsProbeBased = (1 << 0),
202  SecFlagHasAttribute = (1 << 1)
203 };
204 
205 // Verify section specific flag is used for the correct section.
206 template <class SecFlagType>
207 static inline void verifySecFlag(SecType Type, SecFlagType Flag) {
208  // No verification is needed for common flags.
209  if (std::is_same<SecCommonFlags, SecFlagType>())
210  return;
211 
212  // Verification starts here for section specific flag.
213  bool IsFlagLegal = false;
214  switch (Type) {
215  case SecNameTable:
216  IsFlagLegal = std::is_same<SecNameTableFlags, SecFlagType>();
217  break;
218  case SecProfSummary:
219  IsFlagLegal = std::is_same<SecProfSummaryFlags, SecFlagType>();
220  break;
221  case SecFuncMetadata:
222  IsFlagLegal = std::is_same<SecFuncMetadataFlags, SecFlagType>();
223  break;
224  default:
225  break;
226  }
227  if (!IsFlagLegal)
228  llvm_unreachable("Misuse of a flag in an incompatible section");
229 }
230 
231 template <class SecFlagType>
232 static inline void addSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {
233  verifySecFlag(Entry.Type, Flag);
234  auto FVal = static_cast<uint64_t>(Flag);
235  bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
236  Entry.Flags |= IsCommon ? FVal : (FVal << 32);
237 }
238 
239 template <class SecFlagType>
240 static inline void removeSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag) {
241  verifySecFlag(Entry.Type, Flag);
242  auto FVal = static_cast<uint64_t>(Flag);
243  bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
244  Entry.Flags &= ~(IsCommon ? FVal : (FVal << 32));
245 }
246 
247 template <class SecFlagType>
248 static inline bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag) {
249  verifySecFlag(Entry.Type, Flag);
250  auto FVal = static_cast<uint64_t>(Flag);
251  bool IsCommon = std::is_same<SecCommonFlags, SecFlagType>();
252  return Entry.Flags & (IsCommon ? FVal : (FVal << 32));
253 }
254 
255 /// Represents the relative location of an instruction.
256 ///
257 /// Instruction locations are specified by the line offset from the
258 /// beginning of the function (marked by the line where the function
259 /// header is) and the discriminator value within that line.
260 ///
261 /// The discriminator value is useful to distinguish instructions
262 /// that are on the same line but belong to different basic blocks
263 /// (e.g., the two post-increment instructions in "if (p) x++; else y++;").
264 struct LineLocation {
266 
267  void print(raw_ostream &OS) const;
268  void dump() const;
269 
270  bool operator<(const LineLocation &O) const {
271  return LineOffset < O.LineOffset ||
272  (LineOffset == O.LineOffset && Discriminator < O.Discriminator);
273  }
274 
275  bool operator==(const LineLocation &O) const {
276  return LineOffset == O.LineOffset && Discriminator == O.Discriminator;
277  }
278 
279  bool operator!=(const LineLocation &O) const {
280  return LineOffset != O.LineOffset || Discriminator != O.Discriminator;
281  }
282 
285 };
286 
288 
289 /// Representation of a single sample record.
290 ///
291 /// A sample record is represented by a positive integer value, which
292 /// indicates how frequently was the associated line location executed.
293 ///
294 /// Additionally, if the associated location contains a function call,
295 /// the record will hold a list of all the possible called targets. For
296 /// direct calls, this will be the exact function being invoked. For
297 /// indirect calls (function pointers, virtual table dispatch), this
298 /// will be a list of one or more functions.
300 public:
301  using CallTarget = std::pair<StringRef, uint64_t>;
303  bool operator()(const CallTarget &LHS, const CallTarget &RHS) const {
304  if (LHS.second != RHS.second)
305  return LHS.second > RHS.second;
306 
307  return LHS.first < RHS.first;
308  }
309  };
310 
311  using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>;
313  SampleRecord() = default;
314 
315  /// Increment the number of samples for this record by \p S.
316  /// Optionally scale sample count \p S by \p Weight.
317  ///
318  /// Sample counts accumulate using saturating arithmetic, to avoid wrapping
319  /// around unsigned integers.
320  sampleprof_error addSamples(uint64_t S, uint64_t Weight = 1) {
321  bool Overflowed;
322  NumSamples = SaturatingMultiplyAdd(S, Weight, NumSamples, &Overflowed);
323  return Overflowed ? sampleprof_error::counter_overflow
325  }
326 
327  /// Add called function \p F with samples \p S.
328  /// Optionally scale sample count \p S by \p Weight.
329  ///
330  /// Sample counts accumulate using saturating arithmetic, to avoid wrapping
331  /// around unsigned integers.
333  uint64_t Weight = 1) {
334  uint64_t &TargetSamples = CallTargets[F];
335  bool Overflowed;
336  TargetSamples =
337  SaturatingMultiplyAdd(S, Weight, TargetSamples, &Overflowed);
338  return Overflowed ? sampleprof_error::counter_overflow
340  }
341 
342  /// Return true if this sample record contains function calls.
343  bool hasCalls() const { return !CallTargets.empty(); }
344 
345  uint64_t getSamples() const { return NumSamples; }
346  const CallTargetMap &getCallTargets() const { return CallTargets; }
348  return SortCallTargets(CallTargets);
349  }
350 
351  /// Sort call targets in descending order of call frequency.
352  static const SortedCallTargetSet SortCallTargets(const CallTargetMap &Targets) {
353  SortedCallTargetSet SortedTargets;
354  for (const auto &I : Targets) {
355  SortedTargets.emplace(I.first(), I.second);
356  }
357  return SortedTargets;
358  }
359 
360  /// Prorate call targets by a distribution factor.
361  static const CallTargetMap adjustCallTargets(const CallTargetMap &Targets,
362  float DistributionFactor) {
363  CallTargetMap AdjustedTargets;
364  for (const auto &I : Targets) {
365  AdjustedTargets[I.first()] = I.second * DistributionFactor;
366  }
367  return AdjustedTargets;
368  }
369 
370  /// Merge the samples in \p Other into this record.
371  /// Optionally scale sample counts by \p Weight.
372  sampleprof_error merge(const SampleRecord &Other, uint64_t Weight = 1);
373  void print(raw_ostream &OS, unsigned Indent) const;
374  void dump() const;
375 
376 private:
377  uint64_t NumSamples = 0;
378  CallTargetMap CallTargets;
379 };
380 
381 raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample);
382 
383 // State of context associated with FunctionSamples
385  UnknownContext = 0x0, // Profile without context
386  RawContext = 0x1, // Full context profile from input profile
387  SyntheticContext = 0x2, // Synthetic context created for context promotion
388  InlinedContext = 0x4, // Profile for context that is inlined into caller
389  MergedContext = 0x8 // Profile for context merged into base profile
390 };
391 
392 // Attribute of context associated with FunctionSamples
394  ContextNone = 0x0,
395  ContextWasInlined = 0x1, // Leaf of context was inlined in previous build
396  ContextShouldBeInlined = 0x2, // Leaf of context should be inlined
397 };
398 
399 // Sample context for FunctionSamples. It consists of the calling context,
400 // the function name and context state. Internally sample context is represented
401 // using StringRef, which is also the input for constructing a `SampleContext`.
402 // It can accept and represent both full context string as well as context-less
403 // function name.
404 // Example of full context string (note the wrapping `[]`):
405 // `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
406 // Example of context-less function name (same as AutoFDO):
407 // `_Z8funcLeafi`
409 public:
410  SampleContext() : State(UnknownContext), Attributes(ContextNone) {}
412  : Attributes(ContextNone) {
413  setContext(ContextStr, CState);
414  }
415 
416  // Promote context by removing top frames (represented by `ContextStrToRemove`).
417  // Note that with string representation of context, the promotion is effectively
418  // a substr operation with `ContextStrToRemove` removed from left.
419  void promoteOnPath(StringRef ContextStrToRemove) {
420  assert(FullContext.startswith(ContextStrToRemove));
421 
422  // Remove leading context and frame separator " @ ".
423  FullContext = FullContext.substr(ContextStrToRemove.size() + 3);
424  CallingContext = CallingContext.substr(ContextStrToRemove.size() + 3);
425  }
426 
427  // Split the top context frame (left-most substr) from context.
428  static std::pair<StringRef, StringRef>
430  return ContextStr.split(" @ ");
431  }
432 
433  // Decode context string for a frame to get function name and location.
434  // `ContextStr` is in the form of `FuncName:StartLine.Discriminator`.
435  static void decodeContextString(StringRef ContextStr, StringRef &FName,
436  LineLocation &LineLoc) {
437  // Get function name
438  auto EntrySplit = ContextStr.split(':');
439  FName = EntrySplit.first;
440 
441  LineLoc = {0, 0};
442  if (!EntrySplit.second.empty()) {
443  // Get line offset, use signed int for getAsInteger so string will
444  // be parsed as signed.
445  int LineOffset = 0;
446  auto LocSplit = EntrySplit.second.split('.');
447  LocSplit.first.getAsInteger(10, LineOffset);
448  LineLoc.LineOffset = LineOffset;
449 
450  // Get discriminator
451  if (!LocSplit.second.empty())
452  LocSplit.second.getAsInteger(10, LineLoc.Discriminator);
453  }
454  }
455 
456  operator StringRef() const { return FullContext; }
457  bool hasAttribute(ContextAttributeMask A) { return Attributes & (uint32_t)A; }
458  void setAttribute(ContextAttributeMask A) { Attributes |= (uint32_t)A; }
459  uint32_t getAllAttributes() { return Attributes; }
460  void setAllAttributes(uint32_t A) { Attributes = A; }
461  bool hasState(ContextStateMask S) { return State & (uint32_t)S; }
462  void setState(ContextStateMask S) { State |= (uint32_t)S; }
463  void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
464  bool hasContext() const { return State != UnknownContext; }
465  bool isBaseContext() const { return CallingContext.empty(); }
466  StringRef getNameWithoutContext() const { return Name; }
467  StringRef getCallingContext() const { return CallingContext; }
468  StringRef getNameWithContext() const { return FullContext; }
469 
470 private:
471  // Give a context string, decode and populate internal states like
472  // Function name, Calling context and context state. Example of input
473  // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
474  void setContext(StringRef ContextStr, ContextStateMask CState) {
475  assert(!ContextStr.empty());
476  // Note that `[]` wrapped input indicates a full context string, otherwise
477  // it's treated as context-less function name only.
478  bool HasContext = ContextStr.startswith("[");
479  if (!HasContext && CState == UnknownContext) {
480  State = UnknownContext;
481  Name = FullContext = ContextStr;
482  } else {
483  // Assume raw context profile if unspecified
484  if (CState == UnknownContext)
485  State = RawContext;
486  else
487  State = CState;
488 
489  // Remove encapsulating '[' and ']' if any
490  if (HasContext)
491  FullContext = ContextStr.substr(1, ContextStr.size() - 2);
492  else
493  FullContext = ContextStr;
494 
495  // Caller is to the left of callee in context string
496  auto NameContext = FullContext.rsplit(" @ ");
497  if (NameContext.second.empty()) {
498  Name = NameContext.first;
499  CallingContext = NameContext.second;
500  } else {
501  Name = NameContext.second;
502  CallingContext = NameContext.first;
503  }
504  }
505  }
506 
507  // Full context string including calling context and leaf function name
508  StringRef FullContext;
509  // Function name for the associated sample profile
510  StringRef Name;
511  // Calling context (leaf function excluded) for the associated sample profile
512  StringRef CallingContext;
513  // State of the associated sample profile
514  uint32_t State;
515  // Attribute of the associated sample profile
516  uint32_t Attributes;
517 };
518 
519 class FunctionSamples;
520 class SampleProfileReaderItaniumRemapper;
521 
522 using BodySampleMap = std::map<LineLocation, SampleRecord>;
523 // NOTE: Using a StringMap here makes parsed profiles consume around 17% more
524 // memory, which is *very* significant for large profiles.
525 using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>;
526 using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;
527 
528 /// Representation of the samples collected for a function.
529 ///
530 /// This data structure contains all the collected samples for the body
531 /// of a function. Each sample corresponds to a LineLocation instance
532 /// within the body of the function.
534 public:
535  FunctionSamples() = default;
536 
537  void print(raw_ostream &OS = dbgs(), unsigned Indent = 0) const;
538  void dump() const;
539 
540  sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight = 1) {
541  bool Overflowed;
542  TotalSamples =
543  SaturatingMultiplyAdd(Num, Weight, TotalSamples, &Overflowed);
544  return Overflowed ? sampleprof_error::counter_overflow
546  }
547 
548  void setTotalSamples(uint64_t Num) { TotalSamples = Num; }
549 
550  sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) {
551  bool Overflowed;
552  TotalHeadSamples =
553  SaturatingMultiplyAdd(Num, Weight, TotalHeadSamples, &Overflowed);
554  return Overflowed ? sampleprof_error::counter_overflow
556  }
557 
559  uint64_t Num, uint64_t Weight = 1) {
560  return BodySamples[LineLocation(LineOffset, Discriminator)].addSamples(
561  Num, Weight);
562  }
563 
565  uint32_t Discriminator,
566  StringRef FName, uint64_t Num,
567  uint64_t Weight = 1) {
568  return BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(
569  FName, Num, Weight);
570  }
571 
573  uint64_t Weight = 1) {
574  SampleRecord S;
575  S.addSamples(Num, Weight);
576  return BodySamples[LineLocation(Index, 0)].merge(S, Weight);
577  }
578 
579  /// Return the number of samples collected at the given location.
580  /// Each location is specified by \p LineOffset and \p Discriminator.
581  /// If the location is not found in profile, return error.
583  uint32_t Discriminator) const {
584  const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator));
585  if (ret == BodySamples.end()) {
586  // For CSSPGO, in order to conserve profile size, we no longer write out
587  // locations profile for those not hit during training, so we need to
588  // treat them as zero instead of error here.
590  return 0;
591  return std::error_code();
592  } else {
593  // Return error for an invalid sample count which is usually assigned to
594  // dangling probe.
596  ret->second.getSamples() == FunctionSamples::InvalidProbeCount)
597  return std::error_code();
598  return ret->second.getSamples();
599  }
600  }
601 
602  /// Returns the call target map collected at a given location.
603  /// Each location is specified by \p LineOffset and \p Discriminator.
604  /// If the location is not found in profile, return error.
606  findCallTargetMapAt(uint32_t LineOffset, uint32_t Discriminator) const {
607  const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator));
608  if (ret == BodySamples.end())
609  return std::error_code();
610  return ret->second.getCallTargets();
611  }
612 
613  /// Returns the call target map collected at a given location specified by \p
614  /// CallSite. If the location is not found in profile, return error.
616  findCallTargetMapAt(const LineLocation &CallSite) const {
617  const auto &Ret = BodySamples.find(CallSite);
618  if (Ret == BodySamples.end())
619  return std::error_code();
620  return Ret->second.getCallTargets();
621  }
622 
623  /// Return the function samples at the given callsite location.
625  return CallsiteSamples[Loc];
626  }
627 
628  /// Returns the FunctionSamplesMap at the given \p Loc.
629  const FunctionSamplesMap *
631  auto iter = CallsiteSamples.find(Loc);
632  if (iter == CallsiteSamples.end())
633  return nullptr;
634  return &iter->second;
635  }
636 
637  /// Returns a pointer to FunctionSamples at the given callsite location
638  /// \p Loc with callee \p CalleeName. If no callsite can be found, relax
639  /// the restriction to return the FunctionSamples at callsite location
640  /// \p Loc with the maximum total sample count. If \p Remapper is not
641  /// nullptr, use \p Remapper to find FunctionSamples with equivalent name
642  /// as \p CalleeName.
643  const FunctionSamples *
644  findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName,
645  SampleProfileReaderItaniumRemapper *Remapper) const;
646 
647  bool empty() const { return TotalSamples == 0; }
648 
649  /// Return the total number of samples collected inside the function.
650  uint64_t getTotalSamples() const { return TotalSamples; }
651 
652  /// Return the total number of branch samples that have the function as the
653  /// branch target. This should be equivalent to the sample of the first
654  /// instruction of the symbol. But as we directly get this info for raw
655  /// profile without referring to potentially inaccurate debug info, this
656  /// gives more accurate profile data and is preferred for standalone symbols.
657  uint64_t getHeadSamples() const { return TotalHeadSamples; }
658 
659  /// Return the sample count of the first instruction of the function.
660  /// The function can be either a standalone symbol or an inlined function.
661  uint64_t getEntrySamples() const {
663  // For CS profile, if we already have more accurate head samples
664  // counted by branch sample from caller, use them as entry samples.
665  return getHeadSamples();
666  }
667  uint64_t Count = 0;
668  // Use either BodySamples or CallsiteSamples which ever has the smaller
669  // lineno.
670  if (!BodySamples.empty() &&
671  (CallsiteSamples.empty() ||
672  BodySamples.begin()->first < CallsiteSamples.begin()->first))
673  Count = BodySamples.begin()->second.getSamples();
674  else if (!CallsiteSamples.empty()) {
675  // An indirect callsite may be promoted to several inlined direct calls.
676  // We need to get the sum of them.
677  for (const auto &N_FS : CallsiteSamples.begin()->second)
678  Count += N_FS.second.getEntrySamples();
679  }
680  // Return at least 1 if total sample is not 0.
681  return Count ? Count : TotalSamples > 0;
682  }
683 
684  /// Return all the samples collected in the body of the function.
685  const BodySampleMap &getBodySamples() const { return BodySamples; }
686 
687  /// Return all the callsite samples collected in the body of the function.
689  return CallsiteSamples;
690  }
691 
692  /// Return the maximum of sample counts in a function body including functions
693  /// inlined in it.
694  uint64_t getMaxCountInside() const {
695  uint64_t MaxCount = 0;
696  for (const auto &L : getBodySamples())
697  MaxCount = std::max(MaxCount, L.second.getSamples());
698  for (const auto &C : getCallsiteSamples())
699  for (const FunctionSamplesMap::value_type &F : C.second)
700  MaxCount = std::max(MaxCount, F.second.getMaxCountInside());
701  return MaxCount;
702  }
703 
704  /// Merge the samples in \p Other into this one.
705  /// Optionally scale samples by \p Weight.
706  sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight = 1) {
708  Name = Other.getName();
709  if (!GUIDToFuncNameMap)
710  GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
711  if (Context.getNameWithContext().empty())
712  Context = Other.getContext();
713  if (FunctionHash == 0) {
714  // Set the function hash code for the target profile.
715  FunctionHash = Other.getFunctionHash();
716  } else if (FunctionHash != Other.getFunctionHash()) {
717  // The two profiles coming with different valid hash codes indicates
718  // either:
719  // 1. They are same-named static functions from different compilation
720  // units (without using -unique-internal-linkage-names), or
721  // 2. They are really the same function but from different compilations.
722  // Let's bail out in either case for now, which means one profile is
723  // dropped.
725  }
726 
727  MergeResult(Result, addTotalSamples(Other.getTotalSamples(), Weight));
728  MergeResult(Result, addHeadSamples(Other.getHeadSamples(), Weight));
729  for (const auto &I : Other.getBodySamples()) {
730  const LineLocation &Loc = I.first;
731  const SampleRecord &Rec = I.second;
732  MergeResult(Result, BodySamples[Loc].merge(Rec, Weight));
733  }
734  for (const auto &I : Other.getCallsiteSamples()) {
735  const LineLocation &Loc = I.first;
737  for (const auto &Rec : I.second)
738  MergeResult(Result, FSMap[Rec.first].merge(Rec.second, Weight));
739  }
740  return Result;
741  }
742 
743  /// Recursively traverses all children, if the total sample count of the
744  /// corresponding function is no less than \p Threshold, add its corresponding
745  /// GUID to \p S. Also traverse the BodySamples to add hot CallTarget's GUID
746  /// to \p S.
749  uint64_t Threshold) const {
750  if (TotalSamples <= Threshold)
751  return;
752  auto isDeclaration = [](const Function *F) {
753  return !F || F->isDeclaration();
754  };
755  if (isDeclaration(SymbolMap.lookup(getFuncName()))) {
756  // Add to the import list only when it's defined out of module.
757  S.insert(getGUID(Name));
758  }
759  // Import hot CallTargets, which may not be available in IR because full
760  // profile annotation cannot be done until backend compilation in ThinLTO.
761  for (const auto &BS : BodySamples)
762  for (const auto &TS : BS.second.getCallTargets())
763  if (TS.getValue() > Threshold) {
764  const Function *Callee = SymbolMap.lookup(getFuncName(TS.getKey()));
765  if (isDeclaration(Callee))
766  S.insert(getGUID(TS.getKey()));
767  }
768  for (const auto &CS : CallsiteSamples)
769  for (const auto &NameFS : CS.second)
770  NameFS.second.findInlinedFunctions(S, SymbolMap, Threshold);
771  }
772 
773  /// Set the name of the function.
774  void setName(StringRef FunctionName) { Name = FunctionName; }
775 
776  /// Return the function name.
777  StringRef getName() const { return Name; }
778 
779  /// Return function name with context.
781  return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name;
782  }
783 
784  /// Return the original function name.
785  StringRef getFuncName() const { return getFuncName(Name); }
786 
787  void setFunctionHash(uint64_t Hash) { FunctionHash = Hash; }
788 
789  uint64_t getFunctionHash() const { return FunctionHash; }
790 
791  /// Return the canonical name for a function, taking into account
792  /// suffix elision policy attributes.
794  auto AttrName = "sample-profile-suffix-elision-policy";
795  auto Attr = F.getFnAttribute(AttrName).getValueAsString();
796  return getCanonicalFnName(F.getName(), Attr);
797  }
798 
799  /// Name suffixes which canonicalization should handle to avoid
800  /// profile mismatch.
801  static constexpr const char *LLVMSuffix = ".llvm.";
802  static constexpr const char *PartSuffix = ".part.";
803  static constexpr const char *UniqSuffix = ".__uniq.";
804 
806  StringRef Attr = "selected") {
807  // Note the sequence of the suffixes in the knownSuffixes array matters.
808  // If suffix "A" is appended after the suffix "B", "A" should be in front
809  // of "B" in knownSuffixes.
810  const char *knownSuffixes[] = {LLVMSuffix, PartSuffix, UniqSuffix};
811  if (Attr == "" || Attr == "all") {
812  return FnName.split('.').first;
813  } else if (Attr == "selected") {
814  StringRef Cand(FnName);
815  for (const auto &Suf : knownSuffixes) {
816  StringRef Suffix(Suf);
817  // If the profile contains ".__uniq." suffix, don't strip the
818  // suffix for names in the IR.
820  continue;
821  auto It = Cand.rfind(Suffix);
822  if (It == StringRef::npos)
823  continue;
824  auto Dit = Cand.rfind('.');
825  if (Dit == It + Suffix.size() - 1)
826  Cand = Cand.substr(0, It);
827  }
828  return Cand;
829  } else if (Attr == "none") {
830  return FnName;
831  } else {
832  assert(false && "internal error: unknown suffix elision policy");
833  }
834  return FnName;
835  }
836 
837  /// Translate \p Name into its original name.
838  /// When profile doesn't use MD5, \p Name needs no translation.
839  /// When profile uses MD5, \p Name in current FunctionSamples
840  /// is actually GUID of the original function name. getFuncName will
841  /// translate \p Name in current FunctionSamples into its original name
842  /// by looking up in the function map GUIDToFuncNameMap.
843  /// If the original name doesn't exist in the map, return empty StringRef.
845  if (!UseMD5)
846  return Name;
847 
848  assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be popluated first");
849  return GUIDToFuncNameMap->lookup(std::stoull(Name.data()));
850  }
851 
852  /// Returns the line offset to the start line of the subprogram.
853  /// We assume that a single function will not exceed 65535 LOC.
854  static unsigned getOffset(const DILocation *DIL);
855 
856  /// Returns a unique call site identifier for a given debug location of a call
857  /// instruction. This is wrapper of two scenarios, the probe-based profile and
858  /// regular profile, to hide implementation details from the sample loader and
859  /// the context tracker.
860  static LineLocation getCallSiteIdentifier(const DILocation *DIL);
861 
862  /// Get the FunctionSamples of the inline instance where DIL originates
863  /// from.
864  ///
865  /// The FunctionSamples of the instruction (Machine or IR) associated to
866  /// \p DIL is the inlined instance in which that instruction is coming from.
867  /// We traverse the inline stack of that instruction, and match it with the
868  /// tree nodes in the profile.
869  ///
870  /// \returns the FunctionSamples pointer to the inlined instance.
871  /// If \p Remapper is not nullptr, it will be used to find matching
872  /// FunctionSamples with not exactly the same but equivalent name.
874  const DILocation *DIL,
875  SampleProfileReaderItaniumRemapper *Remapper = nullptr) const;
876 
877  // The invalid sample count is used to represent samples collected for a
878  // dangling probe.
879  static constexpr uint64_t InvalidProbeCount = UINT64_MAX;
880 
881  static bool ProfileIsProbeBased;
882 
883  static bool ProfileIsCS;
884 
885  SampleContext &getContext() const { return Context; }
886 
887  void setContext(const SampleContext &FContext) { Context = FContext; }
888 
890 
891  /// Whether the profile uses MD5 to represent string.
892  static bool UseMD5;
893 
894  /// Whether the profile contains any ".__uniq." suffix in a name.
895  static bool HasUniqSuffix;
896 
897  /// If this profile uses flow sensitive discriminators.
898  static bool ProfileIsFS;
899 
900  /// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
901  /// all the function symbols defined or declared in current module.
903 
904  // Assume the input \p Name is a name coming from FunctionSamples itself.
905  // If UseMD5 is true, the name is already a GUID and we
906  // don't want to return the GUID of GUID.
907  static uint64_t getGUID(StringRef Name) {
908  return UseMD5 ? std::stoull(Name.data()) : Function::getGUID(Name);
909  }
910 
911  // Find all the names in the current FunctionSamples including names in
912  // all the inline instances and names of call targets.
913  void findAllNames(DenseSet<StringRef> &NameSet) const;
914 
915 private:
916  /// Mangled name of the function.
917  StringRef Name;
918 
919  /// CFG hash value for the function.
920  uint64_t FunctionHash = 0;
921 
922  /// Calling context for function profile
923  mutable SampleContext Context;
924 
925  /// Total number of samples collected inside this function.
926  ///
927  /// Samples are cumulative, they include all the samples collected
928  /// inside this function and all its inlined callees.
929  uint64_t TotalSamples = 0;
930 
931  /// Total number of samples collected at the head of the function.
932  /// This is an approximation of the number of calls made to this function
933  /// at runtime.
934  uint64_t TotalHeadSamples = 0;
935 
936  /// Map instruction locations to collected samples.
937  ///
938  /// Each entry in this map contains the number of samples
939  /// collected at the corresponding line offset. All line locations
940  /// are an offset from the start of the function.
941  BodySampleMap BodySamples;
942 
943  /// Map call sites to collected samples for the called function.
944  ///
945  /// Each entry in this map corresponds to all the samples
946  /// collected for the inlined function call at the given
947  /// location. For example, given:
948  ///
949  /// void foo() {
950  /// 1 bar();
951  /// ...
952  /// 8 baz();
953  /// }
954  ///
955  /// If the bar() and baz() calls were inlined inside foo(), this
956  /// map will contain two entries. One for all the samples collected
957  /// in the call to bar() at line offset 1, the other for all the samples
958  /// collected in the call to baz() at line offset 8.
959  CallsiteSampleMap CallsiteSamples;
960 };
961 
963 
964 /// Sort a LocationT->SampleT map by LocationT.
965 ///
966 /// It produces a sorted list of <LocationT, SampleT> records by ascending
967 /// order of LocationT.
968 template <class LocationT, class SampleT> class SampleSorter {
969 public:
970  using SamplesWithLoc = std::pair<const LocationT, SampleT>;
972 
973  SampleSorter(const std::map<LocationT, SampleT> &Samples) {
974  for (const auto &I : Samples)
975  V.push_back(&I);
976  llvm::stable_sort(V, [](const SamplesWithLoc *A, const SamplesWithLoc *B) {
977  return A->first < B->first;
978  });
979  }
980 
981  const SamplesWithLocList &get() const { return V; }
982 
983 private:
985 };
986 
987 /// SampleContextTrimmer impelements helper functions to trim, merge cold
988 /// context profiles. It also supports context profile canonicalization to make
989 /// sure ProfileMap's key is consistent with FunctionSample's name/context.
991 public:
993  : ProfileMap(Profiles){};
994  // Trim and merge cold context profile when requested.
996  bool TrimColdContext = true,
997  bool MergeColdContext = true);
998  // Canonicalize context profile name and attributes.
1000 
1001 private:
1002  StringMap<FunctionSamples> &ProfileMap;
1003 };
1004 
1005 /// ProfileSymbolList records the list of function symbols shown up
1006 /// in the binary used to generate the profile. It is useful to
1007 /// to discriminate a function being so cold as not to shown up
1008 /// in the profile and a function newly added.
1010 public:
1011  /// copy indicates whether we need to copy the underlying memory
1012  /// for the input Name.
1013  void add(StringRef Name, bool copy = false) {
1014  if (!copy) {
1015  Syms.insert(Name);
1016  return;
1017  }
1018  Syms.insert(Name.copy(Allocator));
1019  }
1020 
1021  bool contains(StringRef Name) { return Syms.count(Name); }
1022 
1023  void merge(const ProfileSymbolList &List) {
1024  for (auto Sym : List.Syms)
1025  add(Sym, true);
1026  }
1027 
1028  unsigned size() { return Syms.size(); }
1029 
1030  void setToCompress(bool TC) { ToCompress = TC; }
1031  bool toCompress() { return ToCompress; }
1032 
1033  std::error_code read(const uint8_t *Data, uint64_t ListSize);
1034  std::error_code write(raw_ostream &OS);
1035  void dump(raw_ostream &OS = dbgs()) const;
1036 
1037 private:
1038  // Determine whether or not to compress the symbol list when
1039  // writing it into profile. The variable is unused when the symbol
1040  // list is read from an existing profile.
1041  bool ToCompress = false;
1042  DenseSet<StringRef> Syms;
1043  BumpPtrAllocator Allocator;
1044 };
1045 
1046 } // end namespace sampleprof
1047 } // end namespace llvm
1048 
1049 #endif // LLVM_PROFILEDATA_SAMPLEPROF_H
llvm::sampleprof::SampleRecord::getSamples
uint64_t getSamples() const
Definition: SampleProf.h:345
llvm::sampleprof_error::uncompress_failed
@ uncompress_failed
llvm::sampleprof::FunctionSamples::getBodySamples
const BodySampleMap & getBodySamples() const
Return all the samples collected in the body of the function.
Definition: SampleProf.h:685
llvm::StringRef::startswith
LLVM_NODISCARD bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:286
llvm::sampleprof::SecHdrTableEntry::Offset
uint64_t Offset
Definition: SampleProf.h:155
MathExtras.h
llvm
Definition: AllocatorList.h:23
llvm::sampleprof::SecFuncMetadataFlags::SecFlagHasAttribute
@ SecFlagHasAttribute
llvm::sampleprof::BodySampleMap
std::map< LineLocation, SampleRecord > BodySampleMap
Definition: SampleProf.h:522
llvm::sampleprof::SecProfSummaryFlags::SecFlagFSDiscriminator
@ SecFlagFSDiscriminator
SecFlagFSDiscriminator means this profile uses flow-sensitive discriminators.
llvm::sampleprof::SampleSorter::get
const SamplesWithLocList & get() const
Definition: SampleProf.h:981
llvm::StringRef::empty
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:153
llvm::sampleprof::SampleContext::decodeContextString
static void decodeContextString(StringRef ContextStr, StringRef &FName, LineLocation &LineLoc)
Definition: SampleProf.h:435
llvm::sampleprof::FunctionSamples::ProfileIsProbeBased
static bool ProfileIsProbeBased
Definition: SampleProf.h:881
llvm::sampleprof::FunctionSamples::dump
void dump() const
Definition: SampleProf.cpp:307
llvm::sampleprof::FunctionSamples::ProfileIsCS
static bool ProfileIsCS
Definition: SampleProf.h:883
llvm::sampleprof::RawContext
@ RawContext
Definition: SampleProf.h:386
llvm::sampleprof::LineLocation::dump
void dump() const
Definition: SampleProf.cpp:139
llvm::sampleprof::SampleRecord::CallTargetComparator::operator()
bool operator()(const CallTarget &LHS, const CallTarget &RHS) const
Definition: SampleProf.h:303
llvm::sampleprof::SampleContext::hasAttribute
bool hasAttribute(ContextAttributeMask A)
Definition: SampleProf.h:457
llvm::StringRef::rfind
LLVM_NODISCARD size_t rfind(char C, size_t From=npos) const
Search for the last character C in the string.
Definition: StringRef.h:375
llvm::Function
Definition: Function.h:61
llvm::DenseMapBase::lookup
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:197
StringRef.h
llvm::sampleprof::SecHdrTableEntry
Definition: SampleProf.h:152
llvm::sampleprof::CallsiteSampleMap
std::map< LineLocation, FunctionSamplesMap > CallsiteSampleMap
Definition: SampleProf.h:526
llvm::sampleprof::LineLocation::LineLocation
LineLocation(uint32_t L, uint32_t D)
Definition: SampleProf.h:265
llvm::sampleprof::SampleSorter::SamplesWithLocList
SmallVector< const SamplesWithLoc *, 20 > SamplesWithLocList
Definition: SampleProf.h:971
llvm::sampleprof::SampleRecord::hasCalls
bool hasCalls() const
Return true if this sample record contains function calls.
Definition: SampleProf.h:343
llvm::sampleprof_error::hash_mismatch
@ hash_mismatch
llvm::sampleprof::SecNameTableFlags::SecFlagFixedLengthMD5
@ SecFlagFixedLengthMD5
llvm::StringRef::npos
static constexpr size_t npos
Definition: StringRef.h:60
llvm::sampleprof::FunctionSamples::getFuncName
StringRef getFuncName(StringRef Name) const
Translate Name into its original name.
Definition: SampleProf.h:844
llvm::SmallVector< const SamplesWithLoc *, 20 >
llvm::sampleprof::ProfileSymbolList::size
unsigned size()
Definition: SampleProf.h:1028
llvm::sampleprof::SecProfSummaryFlags::SecFlagFullContext
@ SecFlagFullContext
SecFlagContext means this is context-sensitive profile for CSSPGO.
Allocator.h
llvm::sampleprof::SampleContextTrimmer::SampleContextTrimmer
SampleContextTrimmer(StringMap< FunctionSamples > &Profiles)
Definition: SampleProf.h:992
llvm::sampleprof_error::unsupported_version
@ unsupported_version
llvm::sampleprof::hasSecFlag
static bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag)
Definition: SampleProf.h:248
llvm::sampleprof::SampleContext::getNameWithoutContext
StringRef getNameWithoutContext() const
Definition: SampleProf.h:466
llvm::DILocation
Debug location.
Definition: DebugInfoMetadata.h:1562
llvm::sampleprof::LineLocation::operator<
bool operator<(const LineLocation &O) const
Definition: SampleProf.h:270
llvm::sampleprof::SPF_GCC
@ SPF_GCC
Definition: SampleProf.h:91
llvm::sampleprof::ContextShouldBeInlined
@ ContextShouldBeInlined
Definition: SampleProf.h:396
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
Module.h
llvm::StringRef::rsplit
LLVM_NODISCARD std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Definition: StringRef.h:760
llvm::sampleprof::SecFuncOffsetTable
@ SecFuncOffsetTable
Definition: SampleProf.h:123
llvm::sampleprof::addSecFlag
static void addSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag)
Definition: SampleProf.h:232
llvm::copy
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1533
llvm::sampleprof_error::not_implemented
@ not_implemented
FMAInstKind::Accumulator
@ Accumulator
llvm::sampleprof::SecCommonFlags::SecFlagCompress
@ SecFlagCompress
ret
to esp esp setne al movzbw ax esp setg cl movzbw cx cmove cx cl jne LBB1_2 esp ret(also really horrible code on ppc). This is due to the expand code for 64-bit compares. GCC produces multiple branches
llvm::sampleprof::SampleContext::getAllAttributes
uint32_t getAllAttributes()
Definition: SampleProf.h:459
llvm::sampleprof::SecNameTableFlags::SecFlagMD5Name
@ SecFlagMD5Name
llvm::sampleprof::FunctionSamples::findInlinedFunctions
void findInlinedFunctions(DenseSet< GlobalValue::GUID > &S, const StringMap< Function * > &SymbolMap, uint64_t Threshold) const
Recursively traverses all children, if the total sample count of the corresponding function is no les...
Definition: SampleProf.h:747
llvm::MipsISD::Ret
@ Ret
Definition: MipsISelLowering.h:116
llvm::sampleprof::SampleRecord::SampleRecord
SampleRecord()=default
llvm::sampleprof::FunctionSamples::findAllNames
void findAllNames(DenseSet< StringRef > &NameSet) const
Definition: SampleProf.cpp:256
llvm::sampleprof::FunctionSamples::getName
StringRef getName() const
Return the function name.
Definition: SampleProf.h:777
llvm::sampleprof::FunctionSamplesMap
std::map< std::string, FunctionSamples, std::less<> > FunctionSamplesMap
Definition: SampleProf.h:525
llvm::sampleprof::MergedContext
@ MergedContext
Definition: SampleProf.h:389
llvm::sampleprof::SPF_None
@ SPF_None
Definition: SampleProf.h:88
llvm::sampleprof::FunctionSamples::getCanonicalFnName
static StringRef getCanonicalFnName(StringRef FnName, StringRef Attr="selected")
Definition: SampleProf.h:805
llvm::sampleprof::SecHdrTableEntry::Size
uint64_t Size
Definition: SampleProf.h:156
llvm::sampleprof::SPF_Binary
@ SPF_Binary
Definition: SampleProf.h:93
llvm::sampleprof::LineLocation::Discriminator
uint32_t Discriminator
Definition: SampleProf.h:284
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::sampleprof_error::counter_overflow
@ counter_overflow
llvm::sampleprof::LineLocation::LineOffset
uint32_t LineOffset
Definition: SampleProf.h:283
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
llvm::StringRef::substr
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:609
llvm::sampleprof::FunctionSamples::HasUniqSuffix
static bool HasUniqSuffix
Whether the profile contains any ".__uniq." suffix in a name.
Definition: SampleProf.h:895
llvm::sampleprof::removeSecFlag
static void removeSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag)
Definition: SampleProf.h:240
llvm::sampleprof::ProfileSymbolList::setToCompress
void setToCompress(bool TC)
Definition: SampleProf.h:1030
llvm::sampleprof::SecFuncMetadataFlags::SecFlagIsProbeBased
@ SecFlagIsProbeBased
llvm::sampleprof::FunctionSamples::getFuncName
StringRef getFuncName() const
Return the original function name.
Definition: SampleProf.h:785
GlobalValue.h
llvm::sampleprof_error::unsupported_writing_format
@ unsupported_writing_format
llvm::sampleprof::SampleProfileFormat
SampleProfileFormat
Definition: SampleProf.h:87
llvm::sampleprof::getRepInFormat
static StringRef getRepInFormat(StringRef Name, bool UseMD5, std::string &GUIDBuf)
Get the proper representation of a string according to whether the current Format uses MD5 to represe...
Definition: SampleProf.h:105
llvm::sampleprof::FunctionSamples::PartSuffix
static constexpr const char * PartSuffix
Definition: SampleProf.h:802
llvm::sampleprof::FunctionSamples::setTotalSamples
void setTotalSamples(uint64_t Num)
Definition: SampleProf.h:548
llvm::sampleprof::FunctionSamples::getTotalSamples
uint64_t getTotalSamples() const
Return the total number of samples collected inside the function.
Definition: SampleProf.h:650
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::sampleprof::SampleContext::hasState
bool hasState(ContextStateMask S)
Definition: SampleProf.h:461
llvm::StringRef::split
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:727
llvm::sampleprof::SampleRecord::CallTarget
std::pair< StringRef, uint64_t > CallTarget
Definition: SampleProf.h:301
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::sampleprof::FunctionSamples::FunctionSamples
FunctionSamples()=default
llvm::sampleprof::SyntheticContext
@ SyntheticContext
Definition: SampleProf.h:387
llvm::sampleprof::SecProfSummaryFlags::SecFlagPartial
@ SecFlagPartial
SecFlagPartial means the profile is for common/shared code.
UINT64_MAX
#define UINT64_MAX
Definition: DataTypes.h:77
llvm::sampleprof::FunctionSamples::findCallTargetMapAt
ErrorOr< SampleRecord::CallTargetMap > findCallTargetMapAt(uint32_t LineOffset, uint32_t Discriminator) const
Returns the call target map collected at a given location.
Definition: SampleProf.h:606
DenseSet.h
llvm::orc::SymbolMap
DenseMap< SymbolStringPtr, JITEvaluatedSymbol > SymbolMap
A map from symbol names (as SymbolStringPtrs) to JITSymbols (address/flags pairs).
Definition: Core.h:111
llvm::sampleprof::FunctionSamples::getGUID
static uint64_t getGUID(StringRef Name)
Definition: SampleProf.h:907
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::sampleprof_category
const std::error_category & sampleprof_category()
Definition: SampleProf.cpp:101
llvm::sampleprof::FunctionSamples::Format
static SampleProfileFormat Format
Definition: SampleProf.h:889
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:146
ColdCountThreshold
static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)
llvm::sampleprof::SampleSorter::SampleSorter
SampleSorter(const std::map< LocationT, SampleT > &Samples)
Definition: SampleProf.h:973
llvm::sampleprof::SPMagic
static uint64_t SPMagic(SampleProfileFormat Format=SPF_Binary)
Definition: SampleProf.h:96
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:50
llvm::sampleprof::SecNameTableFlags::SecFlagInValid
@ SecFlagInValid
llvm::sampleprof::ContextStateMask
ContextStateMask
Definition: SampleProf.h:384
StringMap.h
llvm::sampleprof::ProfileSymbolList::read
std::error_code read(const uint8_t *Data, uint64_t ListSize)
Definition: SampleProf.cpp:310
llvm::sampleprof::SampleContext
Definition: SampleProf.h:408
llvm::sampleprof::SecProfSummaryFlags::SecFlagInValid
@ SecFlagInValid
llvm::sampleprof::FunctionSamples::getHeadSamples
uint64_t getHeadSamples() const
Return the total number of branch samples that have the function as the branch target.
Definition: SampleProf.h:657
llvm::sampleprof::SampleContext::isBaseContext
bool isBaseContext() const
Definition: SampleProf.h:465
llvm::StringMap< uint64_t >
llvm::sampleprof::FunctionSamples::ProfileIsFS
static bool ProfileIsFS
If this profile uses flow sensitive discriminators.
Definition: SampleProf.h:898
llvm::sampleprof::FunctionSamples::empty
bool empty() const
Definition: SampleProf.h:647
llvm::sampleprof::SecHdrTableEntry::Type
SecType Type
Definition: SampleProf.h:153
llvm::sampleprof::LineLocation::operator!=
bool operator!=(const LineLocation &O) const
Definition: SampleProf.h:279
llvm::sampleprof::SampleRecord::getCallTargets
const CallTargetMap & getCallTargets() const
Definition: SampleProf.h:346
llvm::sampleprof::SampleRecord::SortedCallTargetSet
std::set< CallTarget, CallTargetComparator > SortedCallTargetSet
Definition: SampleProf.h:311
llvm::DenseSet< GlobalValue::GUID >
llvm::sampleprof::ProfileSymbolList::contains
bool contains(StringRef Name)
Definition: SampleProf.h:1021
llvm::sampleprof::FunctionSamples::getCallsiteSamples
const CallsiteSampleMap & getCallsiteSamples() const
Return all the callsite samples collected in the body of the function.
Definition: SampleProf.h:688
llvm::sampleprof::SecHdrTableEntry::Flags
uint64_t Flags
Definition: SampleProf.h:154
llvm::sampleprof::SampleRecord::CallTargetComparator
Definition: SampleProf.h:302
llvm::sampleprof::UnknownContext
@ UnknownContext
Definition: SampleProf.h:385
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:179
llvm::sampleprof::SecProfSummary
@ SecProfSummary
Definition: SampleProf.h:120
llvm::sampleprof::SPF_Compact_Binary
@ SPF_Compact_Binary
Definition: SampleProf.h:90
llvm::sampleprof::ProfileSymbolList
ProfileSymbolList records the list of function symbols shown up in the binary used to generate the pr...
Definition: SampleProf.h:1009
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
llvm::sampleprof::SampleContext::setAttribute
void setAttribute(ContextAttributeMask A)
Definition: SampleProf.h:458
llvm::sampleprof::FunctionSamples::addHeadSamples
sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:550
D
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
llvm::sampleprof::SampleContext::SampleContext
SampleContext()
Definition: SampleProf.h:410
llvm::sampleprof::FunctionSamples::getEntrySamples
uint64_t getEntrySamples() const
Return the sample count of the first instruction of the function.
Definition: SampleProf.h:661
llvm::sampleprof::FunctionSamples::GUIDToFuncNameMap
DenseMap< uint64_t, StringRef > * GUIDToFuncNameMap
GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for all the function symbols define...
Definition: SampleProf.h:902
llvm::sampleprof::LineLocation::operator==
bool operator==(const LineLocation &O) const
Definition: SampleProf.h:275
llvm::sampleprof::FunctionSamples
Representation of the samples collected for a function.
Definition: SampleProf.h:533
llvm::BumpPtrAllocatorImpl
Allocate memory in an ever growing pool, as if by bump-pointer.
Definition: Allocator.h:67
llvm::sampleprof::SecFuncMetadataFlags::SecFlagInvalid
@ SecFlagInvalid
llvm::sampleprof::SPF_Ext_Binary
@ SPF_Ext_Binary
Definition: SampleProf.h:92
llvm::DenseMap
Definition: DenseMap.h:714
llvm::sampleprof::FunctionSamples::functionSamplesAt
FunctionSamplesMap & functionSamplesAt(const LineLocation &Loc)
Return the function samples at the given callsite location.
Definition: SampleProf.h:624
ErrorOr.h
llvm::sampleprof::SampleContext::clearState
void clearState(ContextStateMask S)
Definition: SampleProf.h:463
llvm::sampleprof::SecLBRProfile
@ SecLBRProfile
Definition: SampleProf.h:127
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::SaturatingMultiplyAdd
std::enable_if_t< std::is_unsigned< T >::value, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
Definition: MathExtras.h:866
llvm::sampleprof::FunctionSamples::setContext
void setContext(const SampleContext &FContext)
Definition: SampleProf.h:887
llvm::sampleprof::FunctionSamples::getFunctionHash
uint64_t getFunctionHash() const
Definition: SampleProf.h:789
llvm::sampleprof::FunctionSamples::addCalledTargetSamples
sampleprof_error addCalledTargetSamples(uint32_t LineOffset, uint32_t Discriminator, StringRef FName, uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:564
llvm::sampleprof::SampleContext::promoteOnPath
void promoteOnPath(StringRef ContextStrToRemove)
Definition: SampleProf.h:419
llvm::sampleprof::SampleRecord::adjustCallTargets
static const CallTargetMap adjustCallTargets(const CallTargetMap &Targets, float DistributionFactor)
Prorate call targets by a distribution factor.
Definition: SampleProf.h:361
llvm::sampleprof::SampleContextTrimmer::trimAndMergeColdContextProfiles
void trimAndMergeColdContextProfiles(uint64_t ColdCountThreshold, bool TrimColdContext=true, bool MergeColdContext=true)
Definition: SampleProf.cpp:326
llvm::sampleprof::SampleRecord::getSortedCallTargets
const SortedCallTargetSet getSortedCallTargets() const
Definition: SampleProf.h:347
llvm::sampleprof::SampleContext::setAllAttributes
void setAllAttributes(uint32_t A)
Definition: SampleProf.h:460
llvm::sampleprof_error::malformed
@ malformed
llvm::sampleprof::SampleContext::getNameWithContext
StringRef getNameWithContext() const
Definition: SampleProf.h:468
llvm::sampleprof::FunctionSamples::getCallSiteIdentifier
static LineLocation getCallSiteIdentifier(const DILocation *DIL)
Returns a unique call site identifier for a given debug location of a call instruction.
Definition: SampleProf.cpp:215
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::sampleprof::SecCommonFlags::SecFlagInValid
@ SecFlagInValid
llvm::sampleprof::SampleContext::setState
void setState(ContextStateMask S)
Definition: SampleProf.h:462
llvm::sampleprof::operator<<
raw_ostream & operator<<(raw_ostream &OS, const LineLocation &Loc)
Definition: SampleProf.cpp:111
llvm::sampleprof::FunctionSamples::UseMD5
static bool UseMD5
Whether the profile uses MD5 to represent string.
Definition: SampleProf.h:892
llvm::sampleprof::SecHdrTableEntry::LayoutIndex
uint32_t LayoutIndex
Definition: SampleProf.h:159
llvm::sampleprof::ProfileSymbolList::merge
void merge(const ProfileSymbolList &List)
Definition: SampleProf.h:1023
llvm::sampleprof::FunctionSamples::getOffset
static unsigned getOffset(const DILocation *DIL)
Returns the line offset to the start line of the subprogram.
Definition: SampleProf.cpp:210
llvm::sampleprof::LineLocation
Represents the relative location of an instruction.
Definition: SampleProf.h:264
llvm::MergeResult
sampleprof_error MergeResult(sampleprof_error &Accumulator, sampleprof_error Result)
Definition: SampleProf.h:65
llvm::sampleprof::SecProfSummaryFlags
SecProfSummaryFlags
Definition: SampleProf.h:185
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::sampleprof::SampleSorter
Sort a LocationT->SampleT map by LocationT.
Definition: SampleProf.h:968
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::sampleprof_error::unrecognized_format
@ unrecognized_format
llvm::sampleprof::SecInValid
@ SecInValid
Definition: SampleProf.h:119
uint32_t
llvm::sampleprof::FunctionSamples::addBodySamples
sampleprof_error addBodySamples(uint32_t LineOffset, uint32_t Discriminator, uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:558
llvm::sampleprof::ProfileSymbolList::add
void add(StringRef Name, bool copy=false)
copy indicates whether we need to copy the underlying memory for the input Name.
Definition: SampleProf.h:1013
llvm::sampleprof::FunctionSamples::print
void print(raw_ostream &OS=dbgs(), unsigned Indent=0) const
Print the samples collected for a function on stream OS.
Definition: SampleProf.cpp:164
llvm::sampleprof::FunctionSamples::findFunctionSamplesMapAt
const FunctionSamplesMap * findFunctionSamplesMapAt(const LineLocation &Loc) const
Returns the FunctionSamplesMap at the given Loc.
Definition: SampleProf.h:630
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::sampleprof::FunctionSamples::findSamplesAt
ErrorOr< uint64_t > findSamplesAt(uint32_t LineOffset, uint32_t Discriminator) const
Return the number of samples collected at the given location.
Definition: SampleProf.h:582
llvm::sampleprof::SampleRecord::CallTargetMap
StringMap< uint64_t > CallTargetMap
Definition: SampleProf.h:312
llvm::sampleprof::SPF_Text
@ SPF_Text
Definition: SampleProf.h:89
llvm::sampleprof::getSecName
static std::string getSecName(SecType Type)
Definition: SampleProf.h:130
llvm::sampleprof::FunctionSamples::findFunctionSamplesAt
const FunctionSamples * findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName, SampleProfileReaderItaniumRemapper *Remapper) const
Returns a pointer to FunctionSamples at the given callsite location Loc with callee CalleeName.
Definition: SampleProf.cpp:270
llvm::sampleprof::ProfileSymbolList::toCompress
bool toCompress()
Definition: SampleProf.h:1031
llvm::make_error_code
std::error_code make_error_code(BitcodeError E)
Definition: BitcodeReader.h:270
llvm::sampleprof::InlinedContext
@ InlinedContext
Definition: SampleProf.h:388
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:206
llvm::sampleprof_error::ostream_seek_unsupported
@ ostream_seek_unsupported
StringSet.h
llvm::stable_sort
void stable_sort(R &&Range)
Definition: STLExtras.h:1617
llvm::sampleprof::ProfileSymbolList::write
std::error_code write(raw_ostream &OS)
Definition: SampleProf.cpp:405
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
std
Definition: BitVector.h:838
llvm::sampleprof::LineLocation::print
void print(raw_ostream &OS) const
Definition: SampleProf.cpp:105
llvm::sampleprof::SampleRecord::print
void print(raw_ostream &OS, unsigned Indent) const
Print the sample record to the stream OS indented by Indent.
Definition: SampleProf.cpp:143
llvm::sampleprof::SPVersion
static uint64_t SPVersion()
Definition: SampleProf.h:113
llvm::sampleprof::verifySecFlag
static void verifySecFlag(SecType Type, SecFlagType Flag)
Definition: SampleProf.h:207
llvm::GlobalValue::getGUID
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:511
llvm::sampleprof::SecProfileSymbolList
@ SecProfileSymbolList
Definition: SampleProf.h:122
Function.h
llvm::sampleprof::FunctionSamples::findCallTargetMapAt
ErrorOr< SampleRecord::CallTargetMap > findCallTargetMapAt(const LineLocation &CallSite) const
Returns the call target map collected at a given location specified by CallSite.
Definition: SampleProf.h:616
llvm::sampleprof::FunctionSamples::addTotalSamples
sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:540
llvm::sampleprof::SecType
SecType
Definition: SampleProf.h:118
llvm::sampleprof::SampleRecord
Representation of a single sample record.
Definition: SampleProf.h:299
llvm::sampleprof_error::compress_failed
@ compress_failed
llvm::sampleprof_error
sampleprof_error
Definition: SampleProf.h:42
llvm::sampleprof::FunctionSamples::merge
sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight=1)
Merge the samples in Other into this one.
Definition: SampleProf.h:706
llvm::sampleprof::FunctionSamples::LLVMSuffix
static constexpr const char * LLVMSuffix
Name suffixes which canonicalization should handle to avoid profile mismatch.
Definition: SampleProf.h:801
llvm::sampleprof::ContextNone
@ ContextNone
Definition: SampleProf.h:394
llvm::sampleprof_error::zlib_unavailable
@ zlib_unavailable
llvm::sampleprof::SampleContext::hasContext
bool hasContext() const
Definition: SampleProf.h:464
llvm::sampleprof::SecFuncProfileFirst
@ SecFuncProfileFirst
Definition: SampleProf.h:126
llvm::sampleprof_error::too_large
@ too_large
llvm::sampleprof::SampleRecord::addCalledTarget
sampleprof_error addCalledTarget(StringRef F, uint64_t S, uint64_t Weight=1)
Add called function F with samples S.
Definition: SampleProf.h:332
llvm::sampleprof::FunctionSamples::getContext
SampleContext & getContext() const
Definition: SampleProf.h:885
llvm::sampleprof_error::success
@ success
llvm::sampleprof::SecNameTableFlags::SecFlagUniqSuffix
@ SecFlagUniqSuffix
llvm::sampleprof::FunctionSamples::getMaxCountInside
uint64_t getMaxCountInside() const
Return the maximum of sample counts in a function body including functions inlined in it.
Definition: SampleProf.h:694
llvm::sampleprof::ContextAttributeMask
ContextAttributeMask
Definition: SampleProf.h:393
llvm::sampleprof::SecNameTableFlags
SecNameTableFlags
Definition: SampleProf.h:175
llvm::sampleprof::SampleContext::SampleContext
SampleContext(StringRef ContextStr, ContextStateMask CState=UnknownContext)
Definition: SampleProf.h:411
llvm::sampleprof::ContextWasInlined
@ ContextWasInlined
Definition: SampleProf.h:395
llvm::sampleprof::FunctionSamples::getCanonicalFnName
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes.
Definition: SampleProf.h:793
llvm::sampleprof::SampleContextTrimmer::canonicalizeContextProfiles
void canonicalizeContextProfiles()
Definition: SampleProf.cpp:377
llvm::StringMapImpl::empty
bool empty() const
Definition: StringMap.h:92
SmallVector.h
llvm::sampleprof::SampleRecord::dump
void dump() const
Definition: SampleProf.cpp:154
llvm::sampleprof::SampleRecord::SortCallTargets
static const SortedCallTargetSet SortCallTargets(const CallTargetMap &Targets)
Sort call targets in descending order of call frequency.
Definition: SampleProf.h:352
llvm::sampleprof::SecCommonFlags::SecFlagFlat
@ SecFlagFlat
llvm::sampleprof::SecCommonFlags
SecCommonFlags
Definition: SampleProf.h:165
List
const NodeList & List
Definition: RDFGraph.cpp:201
llvm::sampleprof::FunctionSamples::UniqSuffix
static constexpr const char * UniqSuffix
Definition: SampleProf.h:803
llvm::sampleprof::SampleRecord::merge
sampleprof_error merge(const SampleRecord &Other, uint64_t Weight=1)
Merge the samples in Other into this record.
Definition: SampleProf.cpp:119
llvm::sampleprof::FunctionSamples::InvalidProbeCount
static constexpr uint64_t InvalidProbeCount
Definition: SampleProf.h:879
llvm::ErrorOr
Represents either an error or a value T.
Definition: ErrorOr.h:56
llvm::to_string
std::string to_string(const T &Value)
Definition: ScopedPrinter.h:62
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:340
Threshold
static cl::opt< unsigned > Threshold("loop-unswitch-threshold", cl::desc("Max loop size to unswitch"), cl::init(100), cl::Hidden)
llvm::X86AS::FS
@ FS
Definition: X86.h:188
llvm::sampleprof::SecNameTable
@ SecNameTable
Definition: SampleProf.h:121
llvm::sampleprof::FunctionSamples::addBodySamplesForProbe
sampleprof_error addBodySamplesForProbe(uint32_t Index, uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:572
llvm::sampleprof::SampleProfileReaderItaniumRemapper
SampleProfileReaderItaniumRemapper remaps the profile data from a sample profile data reader,...
Definition: SampleProfReader.h:262
llvm::sampleprof::SecFuncMetadata
@ SecFuncMetadata
Definition: SampleProf.h:124
llvm::sampleprof::SampleContext::getCallingContext
StringRef getCallingContext() const
Definition: SampleProf.h:467
llvm::sampleprof_error::truncated_name_table
@ truncated_name_table
llvm::sampleprof::SampleSorter::SamplesWithLoc
std::pair< const LocationT, SampleT > SamplesWithLoc
Definition: SampleProf.h:970
llvm::sampleprof::SampleContextTrimmer
SampleContextTrimmer impelements helper functions to trim, merge cold context profiles.
Definition: SampleProf.h:990
llvm::sampleprof::ProfileSymbolList::dump
void dump(raw_ostream &OS=dbgs()) const
Definition: SampleProf.cpp:421
llvm::sampleprof::FunctionSamples::getNameWithContext
StringRef getNameWithContext() const
Return function name with context.
Definition: SampleProf.h:780
raw_ostream.h
llvm::StringRef::size
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:157
error_category
static ManagedStatic< _object_error_category > error_category
Definition: Error.cpp:76
llvm::sampleprof_error::truncated
@ truncated
Debug.h
llvm::sampleprof::SampleRecord::addSamples
sampleprof_error addSamples(uint64_t S, uint64_t Weight=1)
Increment the number of samples for this record by S.
Definition: SampleProf.h:320
llvm::sampleprof::SecFuncMetadataFlags
SecFuncMetadataFlags
Definition: SampleProf.h:199
llvm::sampleprof_error::bad_magic
@ bad_magic
llvm::sampleprof::FunctionSamples::setFunctionHash
void setFunctionHash(uint64_t Hash)
Definition: SampleProf.h:787
Other
Optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1169
llvm::sampleprof::SampleContext::splitContextString
static std::pair< StringRef, StringRef > splitContextString(StringRef ContextStr)
Definition: SampleProf.h:429
llvm::sampleprof::FunctionSamples::findFunctionSamples
const FunctionSamples * findFunctionSamples(const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper=nullptr) const
Get the FunctionSamples of the inline instance where DIL originates from.
Definition: SampleProf.cpp:229
llvm::sampleprof::FunctionSamples::setName
void setName(StringRef FunctionName)
Set the name of the function.
Definition: SampleProf.h:774