15#include "llvm/Config/llvm-config.h"
25#include <system_error>
28using namespace sampleprof;
32 cl::desc(
"Cutoff value about how many symbols in profile symbol list "
33 "will be used. This is very useful for performance debugging"));
36 "generate-merged-base-profiles",
37 cl::desc(
"When generating nested context-sensitive profiles, always "
38 "generate extra base profile for function with all its context "
39 "profiles merged into it."));
57class SampleProfErrorCategoryType :
public std::error_category {
58 const char *
name()
const noexcept
override {
return "llvm.sampleprof"; }
60 std::string message(
int IE)
const override {
63 case sampleprof_error::success:
65 case sampleprof_error::bad_magic:
66 return "Invalid sample profile data (bad magic)";
67 case sampleprof_error::unsupported_version:
68 return "Unsupported sample profile format version";
69 case sampleprof_error::too_large:
70 return "Too much profile data";
71 case sampleprof_error::truncated:
72 return "Truncated profile data";
73 case sampleprof_error::malformed:
74 return "Malformed sample profile data";
75 case sampleprof_error::unrecognized_format:
76 return "Unrecognized sample profile encoding format";
77 case sampleprof_error::unsupported_writing_format:
78 return "Profile encoding format unsupported for writing operations";
79 case sampleprof_error::truncated_name_table:
80 return "Truncated function name table";
81 case sampleprof_error::not_implemented:
82 return "Unimplemented feature";
83 case sampleprof_error::counter_overflow:
84 return "Counter overflow";
85 case sampleprof_error::ostream_seek_unsupported:
86 return "Ostream does not support seek";
87 case sampleprof_error::uncompress_failed:
88 return "Uncompress failure";
89 case sampleprof_error::zlib_unavailable:
90 return "Zlib is unavailable";
91 case sampleprof_error::hash_mismatch:
92 return "Function hash mismatch";
101 static SampleProfErrorCategoryType ErrorCategory;
102 return ErrorCategory;
123 for (
const auto &
I :
Other.getCallTargets()) {
129#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
139 OS <<
" " <<
I.first <<
":" <<
I.second;
144#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
159 OS << TotalSamples <<
", " << TotalHeadSamples <<
", " << BodySamples.size()
160 <<
" sampled lines\n";
163 if (!BodySamples.empty()) {
164 OS <<
"Samples collected in the function's body {\n";
166 for (
const auto &SI : SortedBodySamples.
get()) {
168 OS << SI->first <<
": " << SI->second;
173 OS <<
"No samples collected in the function's body\n";
177 if (!CallsiteSamples.empty()) {
178 OS <<
"Samples collected in inlined callsites {\n";
181 for (
const auto &CS : SortedCallsiteSamples.
get()) {
182 for (
const auto &FS : CS->second) {
184 OS << CS->first <<
": inlined callee: " << FS.second.getName() <<
": ";
185 FS.second.print(
OS, Indent + 4);
191 OS <<
"No inlined callsites in this function\n";
203 std::vector<NameFunctionSamples> &SortedProfiles) {
204 for (
const auto &
I : ProfileMap) {
205 SortedProfiles.push_back(std::make_pair(
I.first, &
I.second));
209 if (
A.second->getTotalSamples() ==
B.second->getTotalSamples())
210 return A.second->getContext() <
B.second->getContext();
211 return A.second->getTotalSamples() >
B.second->getTotalSamples();
216 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &
228 DIL->getDiscriminator()),
231 unsigned Discriminator =
239 uint64_t NameHash = std::hash<std::string>{}(CalleeName.
str());
242 return NameHash + (LocId << 5) + LocId;
251 for (DIL = DIL->getInlinedAt(); DIL; DIL = DIL->getInlinedAt()) {
253 StringRef Name = PrevDIL->getScope()->getSubprogram()->getLinkageName();
255 Name = PrevDIL->getScope()->getSubprogram()->getName();
265 for (
int i = S.
size() - 1; i >= 0 && FS !=
nullptr; i--) {
266 FS = FS->findFunctionSamplesAt(S[i].first, S[i].second, Remapper);
273 for (
const auto &BS : BodySamples)
274 for (
const auto &TS : BS.second.getCallTargets())
275 NameSet.
insert(TS.getKey());
277 for (
const auto &CS : CallsiteSamples) {
278 for (
const auto &NameFS : CS.second) {
279 NameSet.
insert(NameFS.first);
280 NameFS.second.findAllNames(NameSet);
290 std::string CalleeGUID;
294 if (iter == CallsiteSamples.end())
296 auto FS = iter->second.find(CalleeName);
297 if (FS != iter->second.end())
301 auto FS = iter->second.find(*NameInProfile);
302 if (FS != iter->second.end())
309 if (!CalleeName.
empty())
313 for (
const auto &NameFS : iter->second)
314 if (NameFS.second.getTotalSamples() >= MaxTotalSamples) {
315 MaxTotalSamples = NameFS.second.getTotalSamples();
321#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
327 const char *ListStart =
reinterpret_cast<const char *
>(
Data);
333 Size += Str.size() + 1;
343 uint32_t ColdContextFrameLength,
bool TrimBaseProfileOnly) {
344 if (!TrimColdContext && !MergeColdContext)
354 if (MergeColdContext)
355 TrimBaseProfileOnly =
false;
359 std::vector<std::pair<hash_code, const FunctionSamples *>> ColdProfiles;
360 for (
const auto &
I : ProfileMap) {
364 (!TrimBaseProfileOnly ||
Context.isBaseContext()))
365 ColdProfiles.emplace_back(
I.first, &
I.second);
371 for (
const auto &
I : ColdProfiles) {
372 if (MergeColdContext) {
378 MergedProfile.
merge(*
I.second);
380 ProfileMap.erase(
I.first);
384 for (
const auto &
I : MergedProfileMap) {
387 ProfileMap.find(
I.second.getContext()) == ProfileMap.end())
394 OrigProfile.
merge(
I.second);
401 std::vector<StringRef> SortedList(Syms.begin(), Syms.end());
404 std::string OutputString;
405 for (
auto &
Sym : SortedList) {
406 OutputString.append(
Sym.str());
407 OutputString.append(1,
'\0');
415 OS <<
"======== Dump profile symbol list ========\n";
416 std::vector<StringRef> SortedList(Syms.begin(), Syms.end());
419 for (
auto &
Sym : SortedList)
429 assert(It->second.FuncName == CalleeName &&
430 "Hash collision for child context node");
439 : ProfileMap(Profiles) {
440 for (
auto &FuncSample : Profiles) {
442 auto *NewNode = getOrCreateContextPath(FSamples->
getContext());
443 assert(!NewNode->FuncSamples &&
"New node cannot have sample profile");
444 NewNode->FuncSamples = FSamples;
449ProfileConverter::getOrCreateContextPath(
const SampleContext &Context) {
450 auto Node = &RootFrame;
452 for (
auto &Callsite :
Context.getContextFrames()) {
453 Node = Node->getOrCreateChildFrame(CallSiteLoc, Callsite.FuncName);
454 CallSiteLoc = Callsite.Location;
463 auto *NodeProfile =
Node.FuncSamples;
464 for (
auto &It :
Node.AllChildFrames) {
465 auto &ChildNode = It.second;
467 auto *ChildProfile = ChildNode.FuncSamples;
473 ChildProfile->getContext().setName(OrigChildContext.
getName());
476 auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
477 SamplesMap.emplace(OrigChildContext.
getName().
str(), *ChildProfile);
478 NodeProfile->addTotalSamples(ChildProfile->getTotalSamples());
481 auto Count = NodeProfile->removeCalledTargetAndBodySample(
482 ChildNode.CallSiteLoc.LineOffset, ChildNode.CallSiteLoc.Discriminator,
484 NodeProfile->removeTotalSamples(Count);
495 ProfileMap[ChildProfile->getContext()].merge(*ChildProfile);
496 NewChildProfileHash = ChildProfile->getContext().getHashCode();
498 ProfileMap[ChildProfile->getContext()].merge(*ChildProfile);
499 NewChildProfileHash = ChildProfile->getContext().getHashCode();
500 auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
501 SamplesMap[ChildProfile->getName().str()].getContext().setAttribute(
508 if (NewChildProfileHash != OrigChildContextHash)
509 ProfileMap.
erase(OrigChildContextHash);
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static cl::opt< bool > GenerateMergedBaseProfiles("generate-merged-base-profiles", cl::desc("When generating nested context-sensitive profiles, always " "generate extra base profile for function with all its context " "profiles merged into it."))
static cl::opt< uint64_t > ProfileSymbolListCutOff("profile-symbol-list-cutoff", cl::Hidden, cl::init(-1), cl::desc("Cutoff value about how many symbols in profile symbol list " "will be used. This is very useful for performance debugging"))
unsigned getBaseDiscriminator() const
Returns the base discriminator stored in the discriminator.
Implements a dense probed hash-table based set.
reference emplace_back(ArgTypes &&... Args)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::string str() const
str - Get the contents as an std::string.
constexpr bool empty() const
empty - Check if the string is empty.
std::pair< iterator, bool > insert(const ValueT &V)
An opaque object representing a hash code.
This class implements an extremely fast bulk output stream that can only output to a stream.
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
Representation of the samples collected for a function.
static bool ProfileIsPreInlined
const LineLocation & mapIRLocToProfileLoc(const LineLocation &IRLoc) const
uint64_t getFunctionHash() const
void findAllNames(DenseSet< StringRef > &NameSet) const
const FunctionSamples * findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName, SampleProfileReaderItaniumRemapper *Remapper) const
Returns a pointer to FunctionSamples at the given callsite location Loc with callee CalleeName.
static bool ProfileIsProbeBased
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes.
static uint64_t getCallSiteHash(StringRef CalleeName, const LineLocation &Callsite)
Returns a unique hash code for a combination of a callsite location and the callee function name.
static unsigned getOffset(const DILocation *DIL)
Returns the line offset to the start line of the subprogram.
static bool ProfileIsFS
If this profile uses flow sensitive discriminators.
SampleContext & getContext() const
static bool HasUniqSuffix
Whether the profile contains any ".__uniq." suffix in a name.
uint64_t getTotalSamples() const
Return the total number of samples collected inside the function.
void print(raw_ostream &OS=dbgs(), unsigned Indent=0) const
Print the samples collected for a function on stream OS.
sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight=1)
Merge the samples in Other into this one.
static LineLocation getCallSiteIdentifier(const DILocation *DIL, bool ProfileIsFS=false)
Returns a unique call site identifier for a given debug location of a call instruction.
const FunctionSamples * findFunctionSamples(const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper=nullptr) const
Get the FunctionSamples of the inline instance where DIL originates from.
StringRef getName() const
Return the function name.
static bool UseMD5
Whether the profile uses MD5 to represent string.
ProfileConverter(SampleProfileMap &Profiles)
std::error_code write(raw_ostream &OS)
void dump(raw_ostream &OS=dbgs()) const
void add(StringRef Name, bool copy=false)
copy indicates whether we need to copy the underlying memory for the input Name.
std::error_code read(const uint8_t *Data, uint64_t ListSize)
void trimAndMergeColdContextProfiles(uint64_t ColdCountThreshold, bool TrimColdContext, bool MergeColdContext, uint32_t ColdContextFrameLength, bool TrimBaseProfileOnly)
StringRef getName() const
uint64_t getHashCode() const
This class provides operator overloads to the map container using MD5 as the key type,...
mapped_type & Create(const SampleContext &Ctx)
size_t erase(const SampleContext &Ctx)
SampleProfileReaderItaniumRemapper remaps the profile data from a sample profile data reader,...
std::optional< StringRef > lookUpNameInProfile(StringRef FunctionName)
Return the equivalent name in the profile for FunctionName if it exists.
Representation of a single sample record.
bool hasCalls() const
Return true if this sample record contains function calls.
sampleprof_error merge(const SampleRecord &Other, uint64_t Weight=1)
Merge the samples in Other into this record.
sampleprof_error addSamples(uint64_t S, uint64_t Weight=1)
Increment the number of samples for this record by S.
sampleprof_error addCalledTarget(StringRef F, uint64_t S, uint64_t Weight=1)
Add called function F with samples S.
const SortedCallTargetSet getSortedCallTargets() const
void print(raw_ostream &OS, unsigned Indent) const
Print the sample record to the stream OS indented by Indent.
Sort a LocationT->SampleT map by LocationT.
const SamplesWithLocList & get() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
initializer< Ty > init(const Ty &Val)
void sortFuncProfiles(const SampleProfileMap &ProfileMap, std::vector< NameFunctionSamples > &SortedProfiles)
std::pair< hash_code, const FunctionSamples * > NameFunctionSamples
raw_ostream & operator<<(raw_ostream &OS, const LineLocation &Loc)
@ ContextDuplicatedIntoBase
static StringRef getRepInFormat(StringRef Name, bool UseMD5, std::string &GUIDBuf)
Get the proper representation of a string according to whether the current Format uses MD5 to represe...
This is an optimization pass for GlobalISel generic memory operations.
void stable_sort(R &&Range)
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
const std::error_category & sampleprof_category()
sampleprof_error MergeResult(sampleprof_error &Accumulator, sampleprof_error Result)
static uint32_t extractProbeIndex(uint32_t Value)
Represents the relative location of an instruction.
void print(raw_ostream &OS) const
FrameNode * getOrCreateChildFrame(const LineLocation &CallSite, StringRef CalleeName)
std::map< uint64_t, FrameNode > AllChildFrames