15#ifndef LLVM_PROFILEDATA_INSTRPROF_H
16#define LLVM_PROFILEDATA_INSTRPROF_H
45#include <system_error>
53struct InstrProfRecord;
60#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
66 return std::numeric_limits<uint64_t>::max() - 2;
76 bool AddSegmentInfo =
true);
81 return INSTR_PROF_VALUE_PROF_FUNC_STR;
87 return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
108 return "__llvm_prf_nm";
115 return "__llvm_coverage_mapping";
123 return "__llvm_coverage_names";
132 return "__llvm_profile_register_functions";
138 return "__llvm_profile_register_function";
143 return "__llvm_profile_register_names_function";
156 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
162 return "__llvm_profile_runtime_user";
166 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
176 uint64_t Version = INSTR_PROF_INDEX_VERSION);
185 uint64_t Version = INSTR_PROF_INDEX_VERSION);
213 StringRef PGOFuncName);
221 StringRef FileName =
"<unknown>");
233 bool doCompression, std::string &Result);
238 std::string &Result,
bool doCompression =
true);
255#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
263 const InstrProfRecord &InstrProfR,
269 ArrayRef<InstrProfValueData> VDs,
uint64_t Sum,
278 InstrProfValueData ValueData[],
280 bool GetNoICPValue =
false);
359 static std::vector<BPFunctionNode>
370 : Err(Err), Msg(ErrStr.str()) {
374 std::string
message()
const override;
388 static std::pair<instrprof_error, std::string>
take(
Error E) {
390 std::string Msg =
"";
412namespace IndexedInstrProf {
433 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
436 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
443 return "** External Symbol **";
452 inline void finalizeSymtab();
482 template <
typename NameIterRange>
Error create(
const NameIterRange &IterRange);
487 if (FuncName.
empty())
489 "function name is empty");
490 auto Ins = NameTab.
insert(FuncName);
492 MD5NameMap.push_back(std::make_pair(
502 AddrToMD5Map.push_back(std::make_pair(
Addr, MD5Val));
524 return Symbol == InstrProfSymtab::getExternalSymbol();
547template <
typename NameIterRange>
549 for (
auto Name : IterRange)
557void InstrProfSymtab::finalizeSymtab() {
563 AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
571 return InstrProfSymtab::getExternalSymbol();
578 [](
const std::pair<uint64_t, StringRef> &
LHS,
580 if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash)
581 return Result->second;
588 [](
const std::pair<uint64_t, Function *> &
LHS,
590 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
591 return Result->second;
646 if (Sum1 < 1.0f || Sum2 < 1.0f)
648 return std::min(Val1 / Sum1, Val2 / Sum2);
664 template <
class InputIterator>
671 [](
const InstrProfValueData &left,
const InstrProfValueData &right) {
672 return left.Value < right.Value;
699 ValueData(
RHS.ValueData
700 ?
std::make_unique<ValueProfData>(*
RHS.ValueData)
705 if (!
RHS.ValueData) {
710 ValueData = std::make_unique<ValueProfData>(*
RHS.ValueData);
712 *ValueData = *
RHS.ValueData;
733 inline std::unique_ptr<InstrProfValueData[]>
762 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
763 for (
auto &SR : getValueSitesForKind(Kind))
813 struct ValueProfData {
814 std::vector<InstrProfValueSiteRecord> IndirectCallSites;
815 std::vector<InstrProfValueSiteRecord> MemOPSizes;
817 std::unique_ptr<ValueProfData> ValueData;
819 MutableArrayRef<InstrProfValueSiteRecord>
820 getValueSitesForKind(
uint32_t ValueKind) {
824 auto AR =
const_cast<const InstrProfRecord *
>(
this)->getValueSitesForKind(
826 return MutableArrayRef(
827 const_cast<InstrProfValueSiteRecord *
>(AR.data()), AR.size());
829 ArrayRef<InstrProfValueSiteRecord>
830 getValueSitesForKind(
uint32_t ValueKind)
const {
834 case IPVK_IndirectCallTarget:
835 return ValueData->IndirectCallSites;
837 return ValueData->MemOPSizes;
843 std::vector<InstrProfValueSiteRecord> &
844 getOrCreateValueSitesForKind(
uint32_t ValueKind) {
846 ValueData = std::make_unique<ValueProfData>();
848 case IPVK_IndirectCallTarget:
849 return ValueData->IndirectCallSites;
851 return ValueData->MemOPSizes;
859 InstrProfSymtab *SymTab);
881 std::vector<uint64_t>
Counts)
894 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
895 NumValueKinds += !(getValueSitesForKind(Kind).empty());
896 return NumValueKinds;
901 for (
const auto &SR : getValueSitesForKind(ValueKind))
902 N += SR.ValueData.size();
907 return getValueSitesForKind(ValueKind).size();
912 return getValueSitesForKind(ValueKind)[Site].ValueData.size();
915std::unique_ptr<InstrProfValueData[]>
919 uint64_t &TotalCount = (TotalC ==
nullptr ? Dummy : *TotalC);
923 return std::unique_ptr<InstrProfValueData[]>(
nullptr);
926 auto VD = std::make_unique<InstrProfValueData[]>(
N);
937 for (
auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
938 Dest[
I].Value = V.Value;
939 Dest[
I].Count = V.Count;
949 getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
957#define INSTR_PROF_VALUE_PROF_DATA
962 [](
const InstrProfValueData &left,
const InstrProfValueData &right) {
963 return left.Count > right.Count;
966 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
971namespace IndexedInstrProf {
1047 size_t size()
const;
1093 void operator delete(
void *ptr) { ::operator
delete(ptr); }
1101 return reinterpret_cast<const uint64_t *
>(
this + 1);
1105 return reinterpret_cast<uint64_t *
>(
this + 1);
1109 return reinterpret_cast<const Entry *
>(
1136 return std::unique_ptr<Summary>(
new (::operator
new(TotalSize))
1142namespace RawInstrProf {
1160 return INSTR_PROF_RAW_MAGIC_64;
1164 return INSTR_PROF_RAW_MAGIC_32;
1173 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
1182#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Machine Check Debug Module
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
StringSet - A set-like wrapper for the StringMap.
Base class for user error types.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
LinkageTypes
An enumeration for the kinds of linkage for global values.
void log(raw_ostream &OS) const override
Print an error message to an output stream.
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
const std::string & getMessage() const
instrprof_error get() const
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
std::string message() const override
Return the error message as a string.
InstrProfError(instrprof_error Err, const Twine &ErrStr=Twine())
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
std::vector< std::pair< uint64_t, uint64_t > > AddrHashMap
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Function * getFunction(uint64_t FuncMD5Hash)
Return function from the name's md5 hash. Return nullptr if not found.
StringRef getNameData() const
Return the name section data.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
InstrProfSymtab()=default
StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash)
Just like getFuncName, except that it will return a non-empty StringRef if the function is external t...
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
A Module instance is used to store all the information related to an LLVM module.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
StringSet - A wrapper for StringMap that provides set-like functionality.
std::pair< typename Base::iterator, bool > insert(StringRef key)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
An efficient, type-erasing, non-owning reference to a callable.
This is a value type class that represents a single section in the list of sections in the object fil...
A raw_ostream that writes to a file descriptor.
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
uint64_t ComputeHash(StringRef K)
uint64_t getMagic< uint32_t >()
uint64_t getMagic< uint64_t >()
constexpr size_t NameSize
static const bool IsLittleEndianHost
This is an optimization pass for GlobalISel generic memory operations.
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab)
NameStrings is a string composed of one of more sub-strings encoded in the format described above.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
std::error_code make_error_code(BitcodeError E)
StringRef getPGOFuncNameMetadataName()
StringRef getCoverageMappingVarName()
Return the name of a covarage mapping variable (internal linkage) for each instrumented source module...
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
StringRef getInstrProfCounterBiasVarName()
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
std::pair< StringRef, StringRef > getParsedIRPGOFuncName(StringRef IRPGOFuncName)
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
void sort(IteratorTy Start, IteratorTy End)
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
support::endianness getHostEndianness()
@ LLVM_MARK_AS_BITMASK_ENUM
@ unable_to_correlate_profile
@ raw_profile_version_mismatch
@ missing_debug_info_for_correlation
@ value_site_count_mismatch
@ unexpected_debug_info_for_correlation
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
const std::error_category & instrprof_category()
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
uint64_t MD5Hash(StringRef Str)
Helper to compute and return lower 64 bits of the given string's MD5 hash.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)
Add two unsigned integers, X and Y, of type T.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
InstrProfKind
An enum describing the attributes of an instrumented profile.
@ FunctionEntryInstrumentation
@ FrontendInstrumentation
Implement std::hash so that hash_code can be used in STL containers.
double ValueCounts[IPVK_Last - IPVK_First+1]
uint64_t Cutoff
The required percentile of total execution count.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
uint64_t MinBlockCount
The minimum execution count for this percentile.
const Entry * getCutoffEntryBase() const
uint64_t get(SummaryFieldKind K) const
void set(SummaryFieldKind K, uint64_t V)
Entry * getCutoffEntryBase()
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
uint64_t * getSummaryDataBase()
@ TotalNumFunctions
The total number of functions instrumented.
@ TotalNumBlocks
Total number of instrumented blocks/edges.
@ MaxFunctionCount
The maximal execution count among all functions.
@ TotalBlockCount
The sum of all instrumented block counts.
@ MaxBlockCount
Max block count of the program.
@ MaxInternalBlockCount
Max internal block count of the program (excluding entry blocks).
const uint64_t * getSummaryDataBase() const
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
uint64_t NumSummaryFields
const Entry & getEntry(uint32_t I) const
uint64_t NumCutoffEntries
Profiling information for a single function.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
InstrProfRecord()=default
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
InstrProfRecord(std::vector< uint64_t > Counts)
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
uint32_t getNumValueKinds() const
Return the number of value profile kinds with non-zero number of profile sites.
void setPseudoCount(CountPseudoKind Kind)
InstrProfRecord(InstrProfRecord &&)=default
uint32_t getNumValueData(uint32_t ValueKind) const
Return the total number of ValueData for ValueKind.
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
InstrProfRecord & operator=(const InstrProfRecord &RHS)
void clearValueData()
Clear value data entries.
InstrProfRecord(const InstrProfRecord &RHS)
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
void sortValueData()
Sort value profile data (per site) by count.
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
InstrProfRecord & operator=(InstrProfRecord &&)=default
void Clear()
Clear value data entries and edge counters.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
InstrProfValueSiteRecord(InputIterator F, InputIterator L)
void sortByCount()
Sort ValueData Descending by Count.
void sortByTargetValues()
Sort ValueData ascending by Value.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
InstrProfValueSiteRecord()
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
static bool hasCSFlagInHash(uint64_t FuncHash)
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts)
static void setCSFlagInHash(uint64_t &FuncHash)
static const int CS_FLAG_IN_FUNC_HASH
NamedInstrProfRecord()=default
const std::string NameFilter
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
const std::string * TestFilename
void setFuncInfo(StringRef Name, uint64_t Hash)
CountSumOrPercent Mismatch
OverlapStats(OverlapStatsLevel L=ProgramLevel)
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
static std::vector< BPFunctionNode > createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...
std::vector< uint64_t > FunctionNameRefs
TemporalProfTraceTy(std::initializer_list< uint64_t > Trace={}, uint64_t Weight=1)
Function object to check whether the first component of a container supported by std::get (like std::...