LLVM 18.0.0git
InstrProf.h
Go to the documentation of this file.
1//===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Instrumentation-based profiling data is generated by instrumented
10// binaries through library functions in compiler-rt, and read by the clang
11// frontend to feed PGO.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_PROFILEDATA_INSTRPROF_H
16#define LLVM_PROFILEDATA_INSTRPROF_H
17
18#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/StringSet.h"
23#include "llvm/IR/GlobalValue.h"
29#include "llvm/Support/Endian.h"
30#include "llvm/Support/Error.h"
32#include "llvm/Support/MD5.h"
37#include <algorithm>
38#include <cassert>
39#include <cstddef>
40#include <cstdint>
41#include <cstring>
42#include <list>
43#include <memory>
44#include <string>
45#include <system_error>
46#include <utility>
47#include <vector>
48
49namespace llvm {
50
51class Function;
52class GlobalVariable;
53struct InstrProfRecord;
54class InstrProfSymtab;
55class Instruction;
56class MDNode;
57class Module;
58
60#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
62};
63
64/// Return the max count value. We reserver a few large values for special use.
66 return std::numeric_limits<uint64_t>::max() - 2;
67}
68
69/// Return the name of the profile section corresponding to \p IPSK.
70///
71/// The name of the section depends on the object format type \p OF. If
72/// \p AddSegmentInfo is true, a segment prefix and additional linker hints may
73/// be added to the section name (this is the default).
76 bool AddSegmentInfo = true);
77
78/// Return the name profile runtime entry point to do value profiling
79/// for a given site.
81 return INSTR_PROF_VALUE_PROF_FUNC_STR;
82}
83
84/// Return the name profile runtime entry point to do memop size value
85/// profiling.
87 return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
88}
89
90/// Return the name prefix of variables containing instrumented function names.
91inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
92
93/// Return the name prefix of variables containing per-function control data.
94inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }
95
96/// Return the name prefix of profile counter variables.
97inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; }
98
99/// Return the name prefix of value profile variables.
100inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }
101
102/// Return the name of value profile node array variables:
103inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
104
105/// Return the name of the variable holding the strings (possibly compressed)
106/// of all function's PGO names.
108 return "__llvm_prf_nm";
109}
110
111/// Return the name of a covarage mapping variable (internal linkage)
112/// for each instrumented source module. Such variables are allocated
113/// in the __llvm_covmap section.
115 return "__llvm_coverage_mapping";
116}
117
118/// Return the name of the internal variable recording the array
119/// of PGO name vars referenced by the coverage mapping. The owning
120/// functions of those names are not emitted by FE (e.g, unused inline
121/// functions.)
123 return "__llvm_coverage_names";
124}
125
126/// Return the name of function that registers all the per-function control
127/// data at program startup time by calling __llvm_register_function. This
128/// function has internal linkage and is called by __llvm_profile_init
129/// runtime method. This function is not generated for these platforms:
130/// Darwin, Linux, and FreeBSD.
132 return "__llvm_profile_register_functions";
133}
134
135/// Return the name of the runtime interface that registers per-function control
136/// data for one instrumented function.
138 return "__llvm_profile_register_function";
139}
140
141/// Return the name of the runtime interface that registers the PGO name strings.
143 return "__llvm_profile_register_names_function";
144}
145
146/// Return the name of the runtime initialization method that is generated by
147/// the compiler. The function calls __llvm_profile_register_functions and
148/// __llvm_profile_override_default_filename functions if needed. This function
149/// has internal linkage and invoked at startup time via init_array.
150inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; }
151
152/// Return the name of the hook variable defined in profile runtime library.
153/// A reference to the variable causes the linker to link in the runtime
154/// initialization module (which defines the hook variable).
156 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
157}
158
159/// Return the name of the compiler generated function that references the
160/// runtime hook variable. The function is a weak global.
162 return "__llvm_profile_runtime_user";
163}
164
166 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
167}
168
169/// Return the marker used to separate PGO names during serialization.
170inline StringRef getInstrProfNameSeparator() { return "\01"; }
171
172/// Return the modified name for function \c F suitable to be
173/// used the key for profile lookup. Variable \c InLTO indicates if this
174/// is called in LTO optimization passes.
175std::string getPGOFuncName(const Function &F, bool InLTO = false,
176 uint64_t Version = INSTR_PROF_INDEX_VERSION);
177
178/// Return the modified name for a function suitable to be
179/// used the key for profile lookup. The function's original
180/// name is \c RawFuncName and has linkage of type \c Linkage.
181/// The function is defined in module \c FileName.
182std::string getPGOFuncName(StringRef RawFuncName,
184 StringRef FileName,
185 uint64_t Version = INSTR_PROF_INDEX_VERSION);
186
187/// \return the modified name for function \c F suitable to be
188/// used as the key for IRPGO profile lookup. \c InLTO indicates if this is
189/// called from LTO optimization passes.
190std::string getIRPGOFuncName(const Function &F, bool InLTO = false);
191
192/// \return the filename and the function name parsed from the output of
193/// \c getIRPGOFuncName()
194std::pair<StringRef, StringRef> getParsedIRPGOFuncName(StringRef IRPGOFuncName);
195
196/// Return the name of the global variable used to store a function
197/// name in PGO instrumentation. \c FuncName is the name of the function
198/// returned by the \c getPGOFuncName call.
199std::string getPGOFuncNameVarName(StringRef FuncName,
201
202/// Create and return the global variable for function name used in PGO
203/// instrumentation. \c FuncName is the name of the function returned
204/// by \c getPGOFuncName call.
205GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);
206
207/// Create and return the global variable for function name used in PGO
208/// instrumentation. /// \c FuncName is the name of the function
209/// returned by \c getPGOFuncName call, \c M is the owning module,
210/// and \c Linkage is the linkage of the instrumented function.
211GlobalVariable *createPGOFuncNameVar(Module &M,
213 StringRef PGOFuncName);
214
215/// Return the initializer in string of the PGO name var \c NameVar.
216StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);
217
218/// Given a PGO function name, remove the filename prefix and return
219/// the original (static) function name.
220StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
221 StringRef FileName = "<unknown>");
222
223/// Given a vector of strings (function PGO names) \c NameStrs, the
224/// method generates a combined string \c Result that is ready to be
225/// serialized. The \c Result string is comprised of three fields:
226/// The first field is the length of the uncompressed strings, and the
227/// the second field is the length of the zlib-compressed string.
228/// Both fields are encoded in ULEB128. If \c doCompress is false, the
229/// third field is the uncompressed strings; otherwise it is the
230/// compressed string. When the string compression is off, the
231/// second field will have value zero.
232Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
233 bool doCompression, std::string &Result);
234
235/// Produce \c Result string with the same format described above. The input
236/// is vector of PGO function name variables that are referenced.
237Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
238 std::string &Result, bool doCompression = true);
239
240/// \c NameStrings is a string composed of one of more sub-strings encoded in
241/// the format described above. The substrings are separated by 0 or more zero
242/// bytes. This method decodes the string and populates the \c Symtab.
243Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab);
244
245/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
246/// set in IR PGO compilation.
247bool isIRPGOFlagSet(const Module *M);
248
249/// Check if we can safely rename this Comdat function. Instances of the same
250/// comdat function may have different control flows thus can not share the
251/// same counter variable.
252bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);
253
255#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
257};
258
259/// Get the value profile data for value site \p SiteIdx from \p InstrProfR
260/// and annotate the instruction \p Inst with the value profile meta data.
261/// Annotate up to \p MaxMDCount (default 3) number of records per value site.
262void annotateValueSite(Module &M, Instruction &Inst,
263 const InstrProfRecord &InstrProfR,
264 InstrProfValueKind ValueKind, uint32_t SiteIndx,
265 uint32_t MaxMDCount = 3);
266
267/// Same as the above interface but using an ArrayRef, as well as \p Sum.
268void annotateValueSite(Module &M, Instruction &Inst,
269 ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
270 InstrProfValueKind ValueKind, uint32_t MaxMDCount);
271
272/// Extract the value profile data from \p Inst which is annotated with
273/// value profile meta data. Return false if there is no value data annotated,
274/// otherwise return true.
275bool getValueProfDataFromInst(const Instruction &Inst,
276 InstrProfValueKind ValueKind,
277 uint32_t MaxNumValueData,
278 InstrProfValueData ValueData[],
279 uint32_t &ActualNumValueData, uint64_t &TotalC,
280 bool GetNoICPValue = false);
281
282inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
283
284/// Return the PGOFuncName meta data associated with a function.
285MDNode *getPGOFuncNameMetadata(const Function &F);
286
287/// Create the PGOFuncName meta data if PGOFuncName is different from
288/// function's raw name. This should only apply to internal linkage functions
289/// declared by users only.
290void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
291
292/// Check if we can use Comdat for profile variables. This will eliminate
293/// the duplicated profile variables for Comdat functions.
294bool needsComdatForCounter(const Function &F, const Module &M);
295
296/// An enum describing the attributes of an instrumented profile.
297enum class InstrProfKind {
298 Unknown = 0x0,
299 // A frontend clang profile, incompatible with other attrs.
301 // An IR-level profile (default when -fprofile-generate is used).
302 IRInstrumentation = 0x2,
303 // A profile with entry basic block instrumentation.
305 // A context sensitive IR-level profile.
306 ContextSensitive = 0x8,
307 // Use single byte probes for coverage.
308 SingleByteCoverage = 0x10,
309 // Only instrument the function entry basic block.
310 FunctionEntryOnly = 0x20,
311 // A memory profile collected using -fprofile=memory.
312 MemProf = 0x40,
313 // A temporal profile.
314 TemporalProfile = 0x80,
316};
317
318const std::error_category &instrprof_category();
319
320enum class instrprof_error {
321 success = 0,
322 eof,
324 bad_magic,
328 too_large,
329 truncated,
330 malformed,
345};
346
347/// An ordered list of functions identified by their NameRef found in
348/// INSTR_PROF_DATA
350 std::vector<uint64_t> FunctionNameRefs;
352 TemporalProfTraceTy(std::initializer_list<uint64_t> Trace = {},
353 uint64_t Weight = 1)
355
356 /// Use a set of temporal profile traces to create a list of balanced
357 /// partitioning function nodes used by BalancedPartitioning to generate a
358 /// function order that reduces page faults during startup
359 static std::vector<BPFunctionNode>
360 createBPFunctionNodes(ArrayRef<TemporalProfTraceTy> Traces);
361};
362
363inline std::error_code make_error_code(instrprof_error E) {
364 return std::error_code(static_cast<int>(E), instrprof_category());
365}
366
367class InstrProfError : public ErrorInfo<InstrProfError> {
368public:
370 : Err(Err), Msg(ErrStr.str()) {
371 assert(Err != instrprof_error::success && "Not an error");
372 }
373
374 std::string message() const override;
375
376 void log(raw_ostream &OS) const override { OS << message(); }
377
378 std::error_code convertToErrorCode() const override {
379 return make_error_code(Err);
380 }
381
382 instrprof_error get() const { return Err; }
383 const std::string &getMessage() const { return Msg; }
384
385 /// Consume an Error and return the raw enum value contained within it, and
386 /// the optional error message. The Error must either be a success value, or
387 /// contain a single InstrProfError.
388 static std::pair<instrprof_error, std::string> take(Error E) {
389 auto Err = instrprof_error::success;
390 std::string Msg = "";
391 handleAllErrors(std::move(E), [&Err, &Msg](const InstrProfError &IPE) {
392 assert(Err == instrprof_error::success && "Multiple errors encountered");
393 Err = IPE.get();
394 Msg = IPE.getMessage();
395 });
396 return {Err, Msg};
397 }
398
399 static char ID;
400
401private:
402 instrprof_error Err;
403 std::string Msg;
404};
405
406namespace object {
407
408class SectionRef;
409
410} // end namespace object
411
412namespace IndexedInstrProf {
413
415
416} // end namespace IndexedInstrProf
417
418/// A symbol table used for function PGO name look-up with keys
419/// (such as pointers, md5hash values) to the function. A function's
420/// PGO name or name's md5hash are used in retrieving the profile
421/// data of the function. See \c getPGOFuncName() method for details
422/// on how PGO name is formed.
424public:
425 using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
426
427private:
428 StringRef Data;
429 uint64_t Address = 0;
430 // Unique name strings.
431 StringSet<> NameTab;
432 // A map from MD5 keys to function name strings.
433 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
434 // A map from MD5 keys to function define. We only populate this map
435 // when build the Symtab from a Module.
436 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
437 // A map from function runtime address to function name MD5 hash.
438 // This map is only populated and used by raw instr profile reader.
439 AddrHashMap AddrToMD5Map;
440 bool Sorted = false;
441
442 static StringRef getExternalSymbol() {
443 return "** External Symbol **";
444 }
445
446 Error addFuncWithName(Function &F, StringRef PGOFuncName);
447
448 // If the symtab is created by a series of calls to \c addFuncName, \c
449 // finalizeSymtab needs to be called before looking up function names.
450 // This is required because the underlying map is a vector (for space
451 // efficiency) which needs to be sorted.
452 inline void finalizeSymtab();
453
454public:
455 InstrProfSymtab() = default;
456
457 /// Create InstrProfSymtab from an object file section which
458 /// contains function PGO names. When section may contain raw
459 /// string data or string data in compressed form. This method
460 /// only initialize the symtab with reference to the data and
461 /// the section base address. The decompression will be delayed
462 /// until before it is used. See also \c create(StringRef) method.
464
465 /// This interface is used by reader of CoverageMapping test
466 /// format.
467 inline Error create(StringRef D, uint64_t BaseAddr);
468
469 /// \c NameStrings is a string composed of one of more sub-strings
470 /// encoded in the format described in \c collectPGOFuncNameStrings.
471 /// This method is a wrapper to \c readPGOFuncNameStrings method.
472 inline Error create(StringRef NameStrings);
473
474 /// A wrapper interface to populate the PGO symtab with functions
475 /// decls from module \c M. This interface is used by transformation
476 /// passes such as indirect function call promotion. Variable \c InLTO
477 /// indicates if this is called from LTO optimization passes.
478 Error create(Module &M, bool InLTO = false);
479
480 /// Create InstrProfSymtab from a set of names iteratable from
481 /// \p IterRange. This interface is used by IndexedProfReader.
482 template <typename NameIterRange> Error create(const NameIterRange &IterRange);
483
484 /// Update the symtab by adding \p FuncName to the table. This interface
485 /// is used by the raw and text profile readers.
487 if (FuncName.empty())
488 return make_error<InstrProfError>(instrprof_error::malformed,
489 "function name is empty");
490 auto Ins = NameTab.insert(FuncName);
491 if (Ins.second) {
492 MD5NameMap.push_back(std::make_pair(
493 IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey()));
494 Sorted = false;
495 }
496 return Error::success();
497 }
498
499 /// Map a function address to its name's MD5 hash. This interface
500 /// is only used by the raw profiler reader.
502 AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
503 }
504
505 /// Return a function's hash, or 0, if the function isn't in this SymTab.
507
508 /// Return function's PGO name from the function name's symbol
509 /// address in the object file. If an error occurs, return
510 /// an empty string.
511 StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
512
513 /// Return function's PGO name from the name's md5 hash value.
514 /// If not found, return an empty string.
515 inline StringRef getFuncName(uint64_t FuncMD5Hash);
516
517 /// Just like getFuncName, except that it will return a non-empty StringRef
518 /// if the function is external to this symbol table. All such cases
519 /// will be represented using the same StringRef value.
521
522 /// True if Symbol is the value used to represent external symbols.
523 static bool isExternalSymbol(const StringRef &Symbol) {
524 return Symbol == InstrProfSymtab::getExternalSymbol();
525 }
526
527 /// Return function from the name's md5 hash. Return nullptr if not found.
528 inline Function *getFunction(uint64_t FuncMD5Hash);
529
530 /// Return the name section data.
531 inline StringRef getNameData() const { return Data; }
532
533 /// Dump the symbols in this table.
534 void dumpNames(raw_ostream &OS) const;
535};
536
538 Data = D;
539 Address = BaseAddr;
540 return Error::success();
541}
542
544 return readPGOFuncNameStrings(NameStrings, *this);
545}
546
547template <typename NameIterRange>
548Error InstrProfSymtab::create(const NameIterRange &IterRange) {
549 for (auto Name : IterRange)
550 if (Error E = addFuncName(Name))
551 return E;
552
553 finalizeSymtab();
554 return Error::success();
555}
556
557void InstrProfSymtab::finalizeSymtab() {
558 if (Sorted)
559 return;
560 llvm::sort(MD5NameMap, less_first());
561 llvm::sort(MD5FuncMap, less_first());
562 llvm::sort(AddrToMD5Map, less_first());
563 AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
564 AddrToMD5Map.end());
565 Sorted = true;
566}
567
569 StringRef ret = getFuncName(FuncMD5Hash);
570 if (ret.empty())
571 return InstrProfSymtab::getExternalSymbol();
572 return ret;
573}
574
576 finalizeSymtab();
577 auto Result = llvm::lower_bound(MD5NameMap, FuncMD5Hash,
578 [](const std::pair<uint64_t, StringRef> &LHS,
579 uint64_t RHS) { return LHS.first < RHS; });
580 if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash)
581 return Result->second;
582 return StringRef();
583}
584
586 finalizeSymtab();
587 auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash,
588 [](const std::pair<uint64_t, Function *> &LHS,
589 uint64_t RHS) { return LHS.first < RHS; });
590 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
591 return Result->second;
592 return nullptr;
593}
594
595// To store the sums of profile count values, or the percentage of
596// the sums of the total count values.
599 double CountSum;
600 double ValueCounts[IPVK_Last - IPVK_First + 1];
602 void reset() {
603 NumEntries = 0;
604 CountSum = 0.0f;
605 for (double &VC : ValueCounts)
606 VC = 0.0f;
607 }
608};
609
610// Function level or program level overlap information.
613 // Sum of the total count values for the base profile.
615 // Sum of the total count values for the test profile.
617 // Overlap lap score. Should be in range of [0.0f to 1.0f].
622 const std::string *BaseFilename;
623 const std::string *TestFilename;
626 bool Valid;
627
629 : Level(L), BaseFilename(nullptr), TestFilename(nullptr), FuncHash(0),
630 Valid(false) {}
631
632 void dump(raw_fd_ostream &OS) const;
633
635 FuncName = Name;
636 FuncHash = Hash;
637 }
638
639 Error accumulateCounts(const std::string &BaseFilename,
640 const std::string &TestFilename, bool IsCS);
641 void addOneMismatch(const CountSumOrPercent &MismatchFunc);
642 void addOneUnique(const CountSumOrPercent &UniqueFunc);
643
644 static inline double score(uint64_t Val1, uint64_t Val2, double Sum1,
645 double Sum2) {
646 if (Sum1 < 1.0f || Sum2 < 1.0f)
647 return 0.0f;
648 return std::min(Val1 / Sum1, Val2 / Sum2);
649 }
650};
651
652// This is used to filter the functions whose overlap information
653// to be output.
656 const std::string NameFilter;
657};
658
660 /// Value profiling data pairs at a given value site.
661 std::list<InstrProfValueData> ValueData;
662
664 template <class InputIterator>
665 InstrProfValueSiteRecord(InputIterator F, InputIterator L)
666 : ValueData(F, L) {}
667
668 /// Sort ValueData ascending by Value
670 ValueData.sort(
671 [](const InstrProfValueData &left, const InstrProfValueData &right) {
672 return left.Value < right.Value;
673 });
674 }
675 /// Sort ValueData Descending by Count
676 inline void sortByCount();
677
678 /// Merge data from another InstrProfValueSiteRecord
679 /// Optionally scale merged counts by \p Weight.
680 void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
681 function_ref<void(instrprof_error)> Warn);
682 /// Scale up value profile data counts by N (Numerator) / D (Denominator).
684
685 /// Compute the overlap b/w this record and Input record.
686 void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind,
687 OverlapStats &Overlap, OverlapStats &FuncLevelOverlap);
688};
689
690/// Profiling information for a single function.
692 std::vector<uint64_t> Counts;
693
694 InstrProfRecord() = default;
695 InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {}
698 : Counts(RHS.Counts),
699 ValueData(RHS.ValueData
700 ? std::make_unique<ValueProfData>(*RHS.ValueData)
701 : nullptr) {}
704 Counts = RHS.Counts;
705 if (!RHS.ValueData) {
706 ValueData = nullptr;
707 return *this;
708 }
709 if (!ValueData)
710 ValueData = std::make_unique<ValueProfData>(*RHS.ValueData);
711 else
712 *ValueData = *RHS.ValueData;
713 return *this;
714 }
715
716 /// Return the number of value profile kinds with non-zero number
717 /// of profile sites.
718 inline uint32_t getNumValueKinds() const;
719 /// Return the number of instrumented sites for ValueKind.
720 inline uint32_t getNumValueSites(uint32_t ValueKind) const;
721
722 /// Return the total number of ValueData for ValueKind.
723 inline uint32_t getNumValueData(uint32_t ValueKind) const;
724
725 /// Return the number of value data collected for ValueKind at profiling
726 /// site: Site.
728 uint32_t Site) const;
729
730 /// Return the array of profiled values at \p Site. If \p TotalC
731 /// is not null, the total count of all target values at this site
732 /// will be stored in \c *TotalC.
733 inline std::unique_ptr<InstrProfValueData[]>
734 getValueForSite(uint32_t ValueKind, uint32_t Site,
735 uint64_t *TotalC = nullptr) const;
736
737 /// Get the target value/counts of kind \p ValueKind collected at site
738 /// \p Site and store the result in array \p Dest. Return the total
739 /// counts of all target values at this site.
740 inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind,
741 uint32_t Site) const;
742
743 /// Reserve space for NumValueSites sites.
744 inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites);
745
746 /// Add ValueData for ValueKind at value Site.
747 void addValueData(uint32_t ValueKind, uint32_t Site,
748 InstrProfValueData *VData, uint32_t N,
749 InstrProfSymtab *SymTab);
750
751 /// Merge the counts in \p Other into this one.
752 /// Optionally scale merged counts by \p Weight.
753 void merge(InstrProfRecord &Other, uint64_t Weight,
754 function_ref<void(instrprof_error)> Warn);
755
756 /// Scale up profile counts (including value profile data) by
757 /// a factor of (N / D).
759
760 /// Sort value profile data (per site) by count.
762 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
763 for (auto &SR : getValueSitesForKind(Kind))
764 SR.sortByCount();
765 }
766
767 /// Clear value data entries and edge counters.
768 void Clear() {
769 Counts.clear();
771 }
772
773 /// Clear value data entries
774 void clearValueData() { ValueData = nullptr; }
775
776 /// Compute the sums of all counts and store in Sum.
777 void accumulateCounts(CountSumOrPercent &Sum) const;
778
779 /// Compute the overlap b/w this IntrprofRecord and Other.
781 OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff);
782
783 /// Compute the overlap of value profile counts.
784 void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
785 OverlapStats &Overlap,
786 OverlapStats &FuncLevelOverlap);
787
792 };
796 };
798 uint64_t FirstCount = Counts[0];
799 if (FirstCount == (uint64_t)HotFunctionVal)
800 return PseudoHot;
801 if (FirstCount == (uint64_t)WarmFunctionVal)
802 return PseudoWarm;
803 return NotPseudo;
804 }
806 if (Kind == PseudoHot)
808 else if (Kind == PseudoWarm)
810 }
811
812private:
813 struct ValueProfData {
814 std::vector<InstrProfValueSiteRecord> IndirectCallSites;
815 std::vector<InstrProfValueSiteRecord> MemOPSizes;
816 };
817 std::unique_ptr<ValueProfData> ValueData;
818
819 MutableArrayRef<InstrProfValueSiteRecord>
820 getValueSitesForKind(uint32_t ValueKind) {
821 // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever
822 // implemented in LLVM) to call the const overload of this function, then
823 // cast away the constness from the result.
824 auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
825 ValueKind);
826 return MutableArrayRef(
827 const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size());
828 }
829 ArrayRef<InstrProfValueSiteRecord>
830 getValueSitesForKind(uint32_t ValueKind) const {
831 if (!ValueData)
832 return std::nullopt;
833 switch (ValueKind) {
834 case IPVK_IndirectCallTarget:
835 return ValueData->IndirectCallSites;
836 case IPVK_MemOPSize:
837 return ValueData->MemOPSizes;
838 default:
839 llvm_unreachable("Unknown value kind!");
840 }
841 }
842
843 std::vector<InstrProfValueSiteRecord> &
844 getOrCreateValueSitesForKind(uint32_t ValueKind) {
845 if (!ValueData)
846 ValueData = std::make_unique<ValueProfData>();
847 switch (ValueKind) {
848 case IPVK_IndirectCallTarget:
849 return ValueData->IndirectCallSites;
850 case IPVK_MemOPSize:
851 return ValueData->MemOPSizes;
852 default:
853 llvm_unreachable("Unknown value kind!");
854 }
855 }
856
857 // Map indirect call target name hash to name string.
858 uint64_t remapValue(uint64_t Value, uint32_t ValueKind,
859 InstrProfSymtab *SymTab);
860
861 // Merge Value Profile data from Src record to this record for ValueKind.
862 // Scale merged value counts by \p Weight.
863 void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src,
864 uint64_t Weight,
865 function_ref<void(instrprof_error)> Warn);
866
867 // Scale up value profile data count by N (Numerator) / D (Denominator).
868 void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D,
869 function_ref<void(instrprof_error)> Warn);
870};
871
875
876 // We reserve this bit as the flag for context sensitive profile record.
877 static const int CS_FLAG_IN_FUNC_HASH = 60;
878
881 std::vector<uint64_t> Counts)
883
884 static bool hasCSFlagInHash(uint64_t FuncHash) {
885 return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1);
886 }
887 static void setCSFlagInHash(uint64_t &FuncHash) {
888 FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH);
889 }
890};
891
893 uint32_t NumValueKinds = 0;
894 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
895 NumValueKinds += !(getValueSitesForKind(Kind).empty());
896 return NumValueKinds;
897}
898
900 uint32_t N = 0;
901 for (const auto &SR : getValueSitesForKind(ValueKind))
902 N += SR.ValueData.size();
903 return N;
904}
905
907 return getValueSitesForKind(ValueKind).size();
908}
909
911 uint32_t Site) const {
912 return getValueSitesForKind(ValueKind)[Site].ValueData.size();
913}
914
915std::unique_ptr<InstrProfValueData[]>
917 uint64_t *TotalC) const {
918 uint64_t Dummy = 0;
919 uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC);
920 uint32_t N = getNumValueDataForSite(ValueKind, Site);
921 if (N == 0) {
922 TotalCount = 0;
923 return std::unique_ptr<InstrProfValueData[]>(nullptr);
924 }
925
926 auto VD = std::make_unique<InstrProfValueData[]>(N);
927 TotalCount = getValueForSite(VD.get(), ValueKind, Site);
928
929 return VD;
930}
931
933 uint32_t ValueKind,
934 uint32_t Site) const {
935 uint32_t I = 0;
936 uint64_t TotalCount = 0;
937 for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
938 Dest[I].Value = V.Value;
939 Dest[I].Count = V.Count;
940 TotalCount = SaturatingAdd(TotalCount, V.Count);
941 I++;
942 }
943 return TotalCount;
944}
945
946void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) {
947 if (!NumValueSites)
948 return;
949 getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
950}
951
954}
955
956// Include definitions for value profile data
957#define INSTR_PROF_VALUE_PROF_DATA
959
961 ValueData.sort(
962 [](const InstrProfValueData &left, const InstrProfValueData &right) {
963 return left.Count > right.Count;
964 });
965 // Now truncate
966 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
967 if (ValueData.size() > max_s)
968 ValueData.resize(max_s);
969}
970
971namespace IndexedInstrProf {
972
973enum class HashT : uint32_t {
974 MD5,
975 Last = MD5
976};
977
979 switch (Type) {
980 case HashT::MD5:
981 return MD5Hash(K);
982 }
983 llvm_unreachable("Unhandled hash type");
984}
985
986const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81"
987
989 // Version 1 is the first version. In this version, the value of
990 // a key/value pair can only include profile data of a single function.
991 // Due to this restriction, the number of block counters for a given
992 // function is not recorded but derived from the length of the value.
994 // The version 2 format supports recording profile data of multiple
995 // functions which share the same key in one value field. To support this,
996 // the number block counters is recorded as an uint64_t field right after the
997 // function structural hash.
999 // Version 3 supports value profile data. The value profile data is expected
1000 // to follow the block counter profile data.
1002 // In this version, profile summary data \c IndexedInstrProf::Summary is
1003 // stored after the profile header.
1005 // In this version, the frontend PGO stable hash algorithm defaults to V2.
1007 // In this version, the frontend PGO stable hash algorithm got fixed and
1008 // may produce hashes different from Version5.
1010 // An additional counter is added around logical operators.
1012 // An additional (optional) memory profile type is added.
1014 // Binary ids are added.
1016 // An additional (optional) temporal profile traces section is added.
1018 // The current version is 10.
1019 CurrentVersion = INSTR_PROF_INDEX_VERSION
1022
1024
1026
1027// This structure defines the file header of the LLVM profile
1028// data file in indexed-format.
1029struct Header {
1032 uint64_t Unused; // Becomes unused since version 4
1038 // New fields should only be added at the end to ensure that the size
1039 // computation is correct. The methods below need to be updated to ensure that
1040 // the new field is read correctly.
1041
1042 // Reads a header struct from the buffer.
1043 static Expected<Header> readFromBuffer(const unsigned char *Buffer);
1044
1045 // Returns the size of the header in bytes for all valid fields based on the
1046 // version. I.e a older version header will return a smaller size.
1047 size_t size() const;
1048
1049 // Returns the format version in little endian. The header retains the version
1050 // in native endian of the compiler runtime.
1051 uint64_t formatVersion() const;
1052};
1053
1054// Profile summary data recorded in the profile data file in indexed
1055// format. It is introduced in version 4. The summary data follows
1056// right after the profile file header.
1057struct Summary {
1058 struct Entry {
1059 uint64_t Cutoff; ///< The required percentile of total execution count.
1060 uint64_t
1061 MinBlockCount; ///< The minimum execution count for this percentile.
1062 uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count.
1063 };
1064 // The field kind enumerator to assigned value mapping should remain
1065 // unchanged when a new kind is added or an old kind gets deleted in
1066 // the future.
1068 /// The total number of functions instrumented.
1070 /// Total number of instrumented blocks/edges.
1072 /// The maximal execution count among all functions.
1073 /// This field does not exist for profile data from IR based
1074 /// instrumentation.
1076 /// Max block count of the program.
1078 /// Max internal block count of the program (excluding entry blocks).
1080 /// The sum of all instrumented block counts.
1084
1085 // The number of summmary fields following the summary header.
1087 // The number of Cutoff Entries (Summary::Entry) following summary fields.
1089
1090 Summary() = delete;
1091 Summary(uint32_t Size) { memset(this, 0, Size); }
1092
1093 void operator delete(void *ptr) { ::operator delete(ptr); }
1094
1096 return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) +
1097 NumSumFields * sizeof(uint64_t);
1098 }
1099
1101 return reinterpret_cast<const uint64_t *>(this + 1);
1102 }
1103
1105 return reinterpret_cast<uint64_t *>(this + 1);
1106 }
1107
1108 const Entry *getCutoffEntryBase() const {
1109 return reinterpret_cast<const Entry *>(
1111 }
1112
1114 return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]);
1115 }
1116
1118 return getSummaryDataBase()[K];
1119 }
1120
1122 getSummaryDataBase()[K] = V;
1123 }
1124
1125 const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; }
1126
1128 Entry &ER = getCutoffEntryBase()[I];
1129 ER.Cutoff = E.Cutoff;
1130 ER.MinBlockCount = E.MinCount;
1131 ER.NumBlocks = E.NumCounts;
1132 }
1133};
1134
1135inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) {
1136 return std::unique_ptr<Summary>(new (::operator new(TotalSize))
1137 Summary(TotalSize));
1138}
1139
1140} // end namespace IndexedInstrProf
1141
1142namespace RawInstrProf {
1143
1144// Version 1: First version
1145// Version 2: Added value profile data section. Per-function control data
1146// struct has more fields to describe value profile information.
1147// Version 3: Compressed name section support. Function PGO name reference
1148// from control data struct is changed from raw pointer to Name's MD5 value.
1149// Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
1150// raw header.
1151// Version 5: Bit 60 of FuncHash is reserved for the flag for the context
1152// sensitive records.
1153// Version 6: Added binary id.
1154// Version 7: Reorder binary id and include version in signature.
1155// Version 8: Use relative counter pointer.
1156const uint64_t Version = INSTR_PROF_RAW_VERSION;
1157
1158template <class IntPtrT> inline uint64_t getMagic();
1159template <> inline uint64_t getMagic<uint64_t>() {
1160 return INSTR_PROF_RAW_MAGIC_64;
1161}
1162
1163template <> inline uint64_t getMagic<uint32_t>() {
1164 return INSTR_PROF_RAW_MAGIC_32;
1165}
1166
1167// Per-function profile data header/control structure.
1168// The definition should match the structure defined in
1169// compiler-rt/lib/profile/InstrProfiling.h.
1170// It should also match the synthesized type in
1171// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
1172template <class IntPtrT> struct alignas(8) ProfileData {
1173 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
1175};
1176
1177// File header structure of the LLVM profile data in raw format.
1178// The definition should match the header referenced in
1179// compiler-rt/lib/profile/InstrProfilingFile.c and
1180// InstrProfilingBuffer.c.
1181struct Header {
1182#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
1184};
1185
1186} // end namespace RawInstrProf
1187
1188// Create the variable for the profile file name.
1189void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
1190
1191// Whether to compress function names in profile records, and filenames in
1192// code coverage mappings. Used by the Instrumentation library and unit tests.
1193extern cl::opt<bool> DoInstrProfNameCompression;
1194
1195} // end namespace llvm
1196#endif // LLVM_PROFILEDATA_INSTRPROF_H
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
uint64_t Addr
std::string Name
uint64_t Size
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Machine Check Debug Module
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
StringSet - A set-like wrapper for the StringMap.
Value * RHS
Value * LHS
Base class for user error types.
Definition: Error.h:352
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
Tagged union holding either a T or a Error.
Definition: Error.h:474
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: InstrProf.h:376
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
Definition: InstrProf.h:388
const std::string & getMessage() const
Definition: InstrProf.h:383
static char ID
Definition: InstrProf.h:399
instrprof_error get() const
Definition: InstrProf.h:382
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: InstrProf.h:378
std::string message() const override
Return the error message as a string.
Definition: InstrProf.cpp:237
InstrProfError(instrprof_error Err, const Twine &ErrStr=Twine())
Definition: InstrProf.h:369
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:423
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
Definition: InstrProf.h:523
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Definition: InstrProf.cpp:476
std::vector< std::pair< uint64_t, uint64_t > > AddrHashMap
Definition: InstrProf.h:425
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Definition: InstrProf.h:501
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Definition: InstrProf.cpp:490
Function * getFunction(uint64_t FuncMD5Hash)
Return function from the name's md5 hash. Return nullptr if not found.
Definition: InstrProf.h:585
StringRef getNameData() const
Return the name section data.
Definition: InstrProf.h:531
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:486
StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash)
Just like getFuncName, except that it will return a non-empty StringRef if the function is external t...
Definition: InstrProf.h:568
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
Definition: MD5.h:41
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
StringSet - A wrapper for StringMap that provides set-like functionality.
Definition: StringSet.h:23
std::pair< typename Base::iterator, bool > insert(StringRef key)
Definition: StringSet.h:34
ObjectFormatType
Definition: Triple.h:279
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
An efficient, type-erasing, non-owning reference to a callable.
This is a value type class that represents a single section in the list of sections in the object fil...
Definition: ObjectFile.h:81
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:454
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1135
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1025
const uint64_t Version
Definition: InstrProf.h:1021
const uint64_t Magic
Definition: InstrProf.h:986
const HashT HashType
Definition: InstrProf.h:1023
uint64_t getMagic()
const uint64_t Version
Definition: InstrProf.h:1156
uint64_t getMagic< uint32_t >()
Definition: InstrProf.h:1163
uint64_t getMagic< uint64_t >()
Definition: InstrProf.h:1159
constexpr size_t NameSize
Definition: XCOFF.h:29
static const bool IsLittleEndianHost
Definition: SwapByteOrder.h:57
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
Definition: InstrProf.cpp:1163
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:91
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
Definition: InstrProf.cpp:552
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
Definition: InstrProf.cpp:342
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:155
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
Definition: InstrProf.cpp:1223
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
Definition: InstrProf.cpp:313
std::error_code make_error_code(BitcodeError E)
StringRef getPGOFuncNameMetadataName()
Definition: InstrProf.h:282
StringRef getCoverageMappingVarName()
Return the name of a covarage mapping variable (internal linkage) for each instrumented source module...
Definition: InstrProf.h:114
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
Definition: Error.h:970
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
Definition: InstrProf.cpp:367
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
Definition: InstrProf.cpp:1219
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:94
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:122
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:218
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
Definition: InstrProf.h:65
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:150
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:100
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:165
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
Definition: InstrProf.cpp:423
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Definition: InstrProf.cpp:1118
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:161
std::pair< StringRef, StringRef > getParsedIRPGOFuncName(StringRef IRPGOFuncName)
Definition: InstrProf.cpp:360
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:131
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1652
InstrProfSectKind
Definition: InstrProf.h:59
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:97
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:535
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
Definition: InstrProf.h:170
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:86
support::endianness getHostEndianness()
Definition: InstrProf.h:952
@ Other
Any other memory.
instrprof_error
Definition: InstrProf.h:320
InstrProfValueKind
Definition: InstrProf.h:254
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:142
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1946
const std::error_category & instrprof_category()
Definition: InstrProf.cpp:185
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1235
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1283
uint64_t MD5Hash(StringRef Str)
Helper to compute and return lower 64 bits of the given string's MD5 hash.
Definition: MD5.h:109
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1306
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1854
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:497
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:80
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:137
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
Definition: InstrProf.cpp:378
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)
Add two unsigned integers, X and Y, of type T.
Definition: MathExtras.h:475
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:107
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1261
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:103
InstrProfKind
An enum describing the attributes of an instrumented profile.
Definition: InstrProf.h:297
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
#define N
double ValueCounts[IPVK_Last - IPVK_First+1]
Definition: InstrProf.h:600
uint64_t formatVersion() const
Definition: InstrProf.cpp:1451
static Expected< Header > readFromBuffer(const unsigned char *Buffer)
Definition: InstrProf.cpp:1456
uint64_t Cutoff
The required percentile of total execution count.
Definition: InstrProf.h:1059
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
Definition: InstrProf.h:1062
uint64_t MinBlockCount
The minimum execution count for this percentile.
Definition: InstrProf.h:1061
const Entry * getCutoffEntryBase() const
Definition: InstrProf.h:1108
uint64_t get(SummaryFieldKind K) const
Definition: InstrProf.h:1117
void set(SummaryFieldKind K, uint64_t V)
Definition: InstrProf.h:1121
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
Definition: InstrProf.h:1127
@ TotalNumFunctions
The total number of functions instrumented.
Definition: InstrProf.h:1069
@ TotalNumBlocks
Total number of instrumented blocks/edges.
Definition: InstrProf.h:1071
@ MaxFunctionCount
The maximal execution count among all functions.
Definition: InstrProf.h:1075
@ TotalBlockCount
The sum of all instrumented block counts.
Definition: InstrProf.h:1081
@ MaxBlockCount
Max block count of the program.
Definition: InstrProf.h:1077
@ MaxInternalBlockCount
Max internal block count of the program (excluding entry blocks).
Definition: InstrProf.h:1079
const uint64_t * getSummaryDataBase() const
Definition: InstrProf.h:1100
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
Definition: InstrProf.h:1095
const Entry & getEntry(uint32_t I) const
Definition: InstrProf.h:1125
Profiling information for a single function.
Definition: InstrProf.h:691
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
Definition: InstrProf.cpp:645
std::vector< uint64_t > Counts
Definition: InstrProf.h:692
CountPseudoKind getCountPseudoKind() const
Definition: InstrProf.h:797
InstrProfRecord(std::vector< uint64_t > Counts)
Definition: InstrProf.h:695
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
Definition: InstrProf.cpp:594
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
Definition: InstrProf.h:906
uint32_t getNumValueKinds() const
Return the number of value profile kinds with non-zero number of profile sites.
Definition: InstrProf.h:892
void setPseudoCount(CountPseudoKind Kind)
Definition: InstrProf.h:805
InstrProfRecord(InstrProfRecord &&)=default
uint32_t getNumValueData(uint32_t ValueKind) const
Return the total number of ValueData for ValueKind.
Definition: InstrProf.h:899
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:767
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
Definition: InstrProf.cpp:847
InstrProfRecord & operator=(const InstrProfRecord &RHS)
Definition: InstrProf.h:703
void clearValueData()
Clear value data entries.
Definition: InstrProf.h:774
InstrProfRecord(const InstrProfRecord &RHS)
Definition: InstrProf.h:697
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
Definition: InstrProf.h:910
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
Definition: InstrProf.h:946
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:663
void sortValueData()
Sort value profile data (per site) by count.
Definition: InstrProf.h:761
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
Definition: InstrProf.h:916
InstrProfRecord & operator=(InstrProfRecord &&)=default
void Clear()
Clear value data entries and edge counters.
Definition: InstrProf.h:768
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:818
InstrProfValueSiteRecord(InputIterator F, InputIterator L)
Definition: InstrProf.h:665
void sortByCount()
Sort ValueData Descending by Count.
Definition: InstrProf.h:960
void sortByTargetValues()
Sort ValueData ascending by Value.
Definition: InstrProf.h:669
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
Definition: InstrProf.cpp:714
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
Definition: InstrProf.cpp:614
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
Definition: InstrProf.h:661
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
Definition: InstrProf.cpp:736
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:884
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts)
Definition: InstrProf.h:880
static void setCSFlagInHash(uint64_t &FuncHash)
Definition: InstrProf.h:887
static const int CS_FLAG_IN_FUNC_HASH
Definition: InstrProf.h:877
const std::string NameFilter
Definition: InstrProf.h:656
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1351
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Definition: InstrProf.h:644
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
Definition: InstrProf.cpp:1323
void dump(raw_fd_ostream &OS) const
Definition: InstrProf.cpp:1370
CountSumOrPercent Overlap
Definition: InstrProf.h:618
CountSumOrPercent Base
Definition: InstrProf.h:614
uint64_t FuncHash
Definition: InstrProf.h:625
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1361
const std::string * BaseFilename
Definition: InstrProf.h:622
const std::string * TestFilename
Definition: InstrProf.h:623
void setFuncInfo(StringRef Name, uint64_t Hash)
Definition: InstrProf.h:634
CountSumOrPercent Unique
Definition: InstrProf.h:620
CountSumOrPercent Mismatch
Definition: InstrProf.h:619
StringRef FuncName
Definition: InstrProf.h:624
OverlapStatsLevel Level
Definition: InstrProf.h:621
OverlapStats(OverlapStatsLevel L=ProgramLevel)
Definition: InstrProf.h:628
CountSumOrPercent Test
Definition: InstrProf.h:616
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
Definition: InstrProf.h:349
static std::vector< BPFunctionNode > createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...
Definition: InstrProf.cpp:861
std::vector< uint64_t > FunctionNameRefs
Definition: InstrProf.h:350
TemporalProfTraceTy(std::initializer_list< uint64_t > Trace={}, uint64_t Weight=1)
Definition: InstrProf.h:352
Function object to check whether the first component of a container supported by std::get (like std::...
Definition: STLExtras.h:1455