20#include "llvm/Config/config.h"
53#include <system_error>
62 cl::desc(
"Use full module build paths in the profile counter names for "
63 "static functions."));
75 cl::desc(
"Strip specified level of directory name from source path in "
76 "the profile counter name for static functions."));
79 const std::string &ErrMsg =
"") {
84 case instrprof_error::success:
87 case instrprof_error::eof:
90 case instrprof_error::unrecognized_format:
91 OS <<
"unrecognized instrumentation profile encoding format";
93 case instrprof_error::bad_magic:
94 OS <<
"invalid instrumentation profile data (bad magic)";
96 case instrprof_error::bad_header:
97 OS <<
"invalid instrumentation profile data (file header is corrupt)";
99 case instrprof_error::unsupported_version:
100 OS <<
"unsupported instrumentation profile format version";
102 case instrprof_error::unsupported_hash_type:
103 OS <<
"unsupported instrumentation profile hash type";
105 case instrprof_error::too_large:
106 OS <<
"too much profile data";
108 case instrprof_error::truncated:
109 OS <<
"truncated profile data";
111 case instrprof_error::malformed:
112 OS <<
"malformed instrumentation profile data";
114 case instrprof_error::missing_correlation_info:
115 OS <<
"debug info/binary for correlation is required";
117 case instrprof_error::unexpected_correlation_info:
118 OS <<
"debug info/binary for correlation is not necessary";
120 case instrprof_error::unable_to_correlate_profile:
121 OS <<
"unable to correlate profile";
123 case instrprof_error::invalid_prof:
124 OS <<
"invalid profile created. Please file a bug "
125 "at: " BUG_REPORT_URL
126 " and include the profraw files that caused this error.";
128 case instrprof_error::unknown_function:
129 OS <<
"no profile data available for function";
131 case instrprof_error::hash_mismatch:
132 OS <<
"function control flow change detected (hash mismatch)";
134 case instrprof_error::count_mismatch:
135 OS <<
"function basic block count change detected (counter mismatch)";
137 case instrprof_error::bitmap_mismatch:
138 OS <<
"function bitmap size change detected (bitmap size mismatch)";
140 case instrprof_error::counter_overflow:
141 OS <<
"counter overflow";
143 case instrprof_error::value_site_count_mismatch:
144 OS <<
"function value site count change detected (counter mismatch)";
146 case instrprof_error::compress_failed:
147 OS <<
"failed to compress data (zlib)";
149 case instrprof_error::uncompress_failed:
150 OS <<
"failed to uncompress data (zlib)";
152 case instrprof_error::empty_raw_profile:
153 OS <<
"empty raw profile file";
155 case instrprof_error::zlib_unavailable:
156 OS <<
"profile uses zlib compression but the profile reader was built "
157 "without zlib support";
159 case instrprof_error::raw_profile_version_mismatch:
160 OS <<
"raw profile version mismatch";
162 case instrprof_error::counter_value_too_large:
163 OS <<
"excessively large counter value suggests corrupted profile data";
169 OS <<
": " << ErrMsg;
179class InstrProfErrorCategoryType :
public std::error_category {
180 const char *
name()
const noexcept
override {
return "llvm.instrprof"; }
182 std::string message(
int IE)
const override {
190 static InstrProfErrorCategoryType ErrorCategory;
191 return ErrorCategory;
196const char *InstrProfSectNameCommon[] = {
197#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
202const char *InstrProfSectNameCoff[] = {
203#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
208const char *InstrProfSectNamePrefix[] = {
209#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
219 "enable-name-compression",
224 bool AddSegmentInfo) {
225 std::string SectName;
228 SectName = InstrProfSectNamePrefix[IPSK];
231 SectName += InstrProfSectNameCoff[IPSK];
233 SectName += InstrProfSectNameCommon[IPSK];
235 if (OF ==
Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
236 SectName +=
",regular,live_support";
256 std::string NewName = std::string(
Name);
262 if (FileName.
empty())
263 NewName = NewName.insert(0,
"<unknown>:");
265 NewName = NewName.insert(0, FileName.
str() +
":");
276 for (
auto & CI : PathNameStr) {
285 return PathNameStr.
substr(LastPos);
336 MDNode *PGONameMetadata) {
344 return *IRPGOFuncName;
384 if (MangledName.empty())
385 return std::make_pair(
StringRef(), IRPGOName);
386 return std::make_pair(FileName, MangledName);
390 if (FileName.
empty())
410 const char InvalidChars[] =
"-:;<>/\"'";
411 size_t found = VarName.find_first_of(InvalidChars);
412 while (found != std::string::npos) {
413 VarName[found] =
'_';
414 found = VarName.find_first_of(InvalidChars, found + 1);
474 const uint8_t *EndP = NameStrings.
bytes_end();
481 bool isCompressed = (CompressedSize != 0);
489 UncompressedNameStrings,
495 NameStrings =
toStringRef(UncompressedNameStrings);
498 StringRef(
reinterpret_cast<const char *
>(
P), UncompressedSize);
499 P += UncompressedSize;
508 while (
P < EndP && *
P == 0)
530 const std::string UniqSuffix =
".__uniq.";
531 size_t pos = PGOName.
find(UniqSuffix);
533 pos += UniqSuffix.length();
539 pos = PGOName.
find(
'.', pos);
541 return PGOName.
substr(0, pos);
553 if (
Error E = mapName(PGOFuncName))
556 StringRef CanonicalFuncName = getCanonicalName(PGOFuncName);
557 if (CanonicalFuncName != PGOFuncName)
558 return mapName(CanonicalFuncName);
565 auto It =
partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t>
A) {
566 return A.first < Address;
572 if (It != AddrToMD5Map.end() && It->first == Address)
585 bool doCompression, std::string &Result) {
586 assert(!NameStrs.
empty() &&
"No name data to emit");
588 uint8_t Header[20], *
P = Header;
589 std::string UncompressedNameStrings =
594 "PGO name is invalid (contains separator token)");
596 unsigned EncLen =
encodeULEB128(UncompressedNameStrings.length(),
P);
599 auto WriteStringToResult = [&](
size_t CompressedLen,
StringRef InputStr) {
602 char *HeaderStr =
reinterpret_cast<char *
>(&Header[0]);
603 unsigned HeaderLen =
P - &Header[0];
604 Result.append(HeaderStr, HeaderLen);
609 if (!doCompression) {
610 return WriteStringToResult(0, UncompressedNameStrings);
615 CompressedNameStrings,
618 return WriteStringToResult(CompressedNameStrings.
size(),
625 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
630 std::string &Result,
bool doCompression) {
631 std::vector<std::string> NameStrs;
632 for (
auto *NameVar : NameVars) {
646 for (
uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
649 for (
size_t I = 0;
I < NumValueSites; ++
I) {
653 KindSum += VD[V].Count;
665 double Score = 0.0f, FuncLevelScore = 0.0f;
670 while (
I != IE && J != JE) {
671 if (
I->Value == J->Value) {
679 }
else if (
I->Value < J->Value) {
695 assert(ThisNumValueSites ==
Other.getNumValueSites(ValueKind));
696 if (!ThisNumValueSites)
699 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
700 getOrCreateValueSitesForKind(ValueKind);
702 Other.getValueSitesForKind(ValueKind);
704 ThisSiteRecords[
I].
overlap(OtherSiteRecords[
I], ValueKind, Overlap,
714 bool Mismatch = (
Counts.size() !=
Other.Counts.size());
718 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
721 if (ThisNumValueSites != OtherNumValueSites) {
733 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
739 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I) {
742 MaxCount = std::max(
Other.Counts[
I], MaxCount);
747 if (MaxCount >= ValueCutoff) {
748 double FuncScore = 0.0;
749 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I)
755 FuncLevelOverlap.
Valid =
true;
766 for (
const InstrProfValueData &J : Input.
ValueData) {
767 while (
I != IE &&
I->Value < J.Value)
769 if (
I != IE &&
I->Value == J.Value) {
793void InstrProfRecord::mergeValueProfData(
797 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
798 if (ThisNumValueSites != OtherNumValueSites) {
802 if (!ThisNumValueSites)
804 std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
805 getOrCreateValueSitesForKind(ValueKind);
807 Src.getValueSitesForKind(ValueKind);
809 ThisSiteRecords[
I].
merge(OtherSiteRecords[
I], Weight, Warn);
839 for (
size_t I = 0, E =
Other.Counts.size();
I < E; ++
I) {
860 for (
size_t I = 0, E =
Other.BitmapBytes.size();
I < E; ++
I) {
864 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
865 mergeValueProfData(Kind,
Other, Weight, Warn);
868void InstrProfRecord::scaleValueProfData(
871 for (
auto &R : getValueSitesForKind(ValueKind))
877 assert(
D != 0 &&
"D cannot be 0");
878 for (
auto &Count : this->
Counts) {
888 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
889 scaleValueProfData(Kind,
N,
D, Warn);
898 if (ValueKind == IPVK_IndirectCallTarget)
910 std::vector<InstrProfValueSiteRecord> &ValueSites =
911 getOrCreateValueSitesForKind(ValueKind);
913 ValueSites.emplace_back();
915 ValueSites.emplace_back(VData, VData +
N);
925 size_t LargestTraceSize = 0;
926 for (
auto &
Trace : Traces)
928 std::max(LargestTraceSize,
Trace.FunctionNameRefs.
size());
930 for (
auto &
Trace : Traces)
934 const int N =
Log2_64(LargestTraceSize) + 1;
939 for (
size_t TraceIdx = 0; TraceIdx < Traces.
size(); TraceIdx++) {
940 auto &
Trace = Traces[TraceIdx].FunctionNameRefs;
944 UtilityNodeT GroupId = TraceIdx *
N +
I;
950 std::vector<BPFunctionNode> Nodes;
951 for (
auto Id : FunctionIds) {
952 auto &UNs = FuncGroups[Id];
954 UNs.erase(std::unique(UNs.begin(), UNs.end()), UNs.end());
955 Nodes.emplace_back(Id, UNs);
960#define INSTR_PROF_COMMON_API_IMPL
974 ->getNumValueSites(VKind);
979 ->getNumValueData(VKind);
985 ->getNumValueDataForSite(VK, S);
990 reinterpret_cast<const InstrProfRecord *
>(R)->getValueForSite(Dst, K, S);
995 (ValueProfData *)(
new (::operator
new(TotalSizeInBytes)) ValueProfData());
996 memset(VD, 0, TotalSizeInBytes);
1013 Closure.Record = &
Record;
1014 return getValueProfDataSize(&Closure);
1018std::unique_ptr<ValueProfData>
1022 std::unique_ptr<ValueProfData> VPD(
1029 Record.reserveSites(Kind, NumValueSites);
1031 InstrProfValueData *ValueData = getValueProfRecordValueData(
this);
1032 for (
uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
1033 uint8_t ValueDataCount = this->SiteCountArray[VSite];
1034 Record.addValueData(Kind, VSite, ValueData, ValueDataCount, SymTab);
1035 ValueData += ValueDataCount;
1043 using namespace support;
1049 sys::swapByteOrder<uint32_t>(NumValueSites);
1050 sys::swapByteOrder<uint32_t>(Kind);
1052 uint32_t ND = getValueProfRecordNumValueData(
this);
1053 InstrProfValueData *VD = getValueProfRecordValueData(
this);
1057 sys::swapByteOrder<uint64_t>(VD[
I].
Value);
1058 sys::swapByteOrder<uint64_t>(VD[
I].Count);
1061 sys::swapByteOrder<uint32_t>(NumValueSites);
1062 sys::swapByteOrder<uint32_t>(Kind);
1068 if (NumValueKinds == 0)
1071 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1072 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1073 VR->deserializeTo(
Record, SymTab);
1074 VR = getValueProfRecordNext(VR);
1080 using namespace support;
1083 return endian::readNext<T, llvm::endianness::little, unaligned>(
D);
1085 return endian::readNext<T, llvm::endianness::big, unaligned>(
D);
1089 return std::unique_ptr<ValueProfData>(
new (::operator
new(TotalSize))
1093Error ValueProfData::checkIntegrity() {
1094 if (NumValueKinds > IPVK_Last + 1)
1095 return make_error<InstrProfError>(
1099 return make_error<InstrProfError>(
1102 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1103 for (
uint32_t K = 0; K < this->NumValueKinds; K++) {
1104 if (VR->Kind > IPVK_Last)
1106 "value kind is invalid");
1107 VR = getValueProfRecordNext(VR);
1108 if ((
char *)VR - (
char *)
this > (
ptrdiff_t)TotalSize)
1109 return make_error<InstrProfError>(
1111 "value profile address is greater than total size");
1117ValueProfData::getValueProfData(
const unsigned char *
D,
1118 const unsigned char *
const BufferEnd,
1120 using namespace support;
1122 if (
D +
sizeof(ValueProfData) > BufferEnd)
1125 const unsigned char *Header =
D;
1126 uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
1127 if (
D + TotalSize > BufferEnd)
1131 memcpy(VPD.get(),
D, TotalSize);
1133 VPD->swapBytesToHost(Endianness);
1135 Error E = VPD->checkIntegrity();
1137 return std::move(E);
1139 return std::move(VPD);
1143 using namespace support;
1148 sys::swapByteOrder<uint32_t>(TotalSize);
1149 sys::swapByteOrder<uint32_t>(NumValueKinds);
1151 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1152 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1154 VR = getValueProfRecordNext(VR);
1159 using namespace support;
1164 ValueProfRecord *VR = getFirstValueProfRecord(
this);
1165 for (
uint32_t K = 0;
K < NumValueKinds;
K++) {
1166 ValueProfRecord *NVR = getValueProfRecordNext(VR);
1170 sys::swapByteOrder<uint32_t>(TotalSize);
1171 sys::swapByteOrder<uint32_t>(NumValueKinds);
1183 std::unique_ptr<InstrProfValueData[]> VD =
1208 for (
auto &VD : VDs) {
1222 InstrProfValueData ValueData[],
1224 bool GetNoICPValue) {
1239 if (!
Tag->getString().equals(
"VP"))
1255 ActualNumValueData = 0;
1257 for (
unsigned I = 3;
I < NOps;
I += 2) {
1258 if (ActualNumValueData >= MaxNumValueData)
1262 mdconst::dyn_extract<ConstantInt>(MD->
getOperand(
I + 1));
1263 if (!
Value || !Count)
1268 ValueData[ActualNumValueData].Value =
Value->getZExtValue();
1269 ValueData[ActualNumValueData].Count = CntValue;
1270 ActualNumValueData++;
1281 if (PGOFuncName ==
F.getName())
1319 M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1320 if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1325 if (IRInstrVar->isDeclaration())
1329 if (!IRInstrVar->hasInitializer())
1332 auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1335 return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1340 if (
F.getName().empty())
1346 if (CheckAddressTaken &&
F.hasAddressTaken())
1354 if (!
F.hasComdat()) {
1363 if (InstrProfileOutput.
empty())
1369 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1371 Triple TT(M.getTargetTriple());
1372 if (TT.supportsCOMDAT()) {
1374 ProfileNameVar->
setComdat(M.getOrInsertComdat(
1375 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1380 const std::string &TestFilename,
1382 auto getProfileSum = [IsCS](
const std::string &Filename,
1388 if (
Error E = ReaderOrErr.takeError()) {
1391 auto Reader = std::move(ReaderOrErr.get());
1392 Reader->accumulateCounts(Sum, IsCS);
1410 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1420 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1430 const char *EntryName =
1431 (Level ==
ProgramLevel ?
"functions" :
"edge counters");
1433 OS <<
"Profile overlap infomation for base_profile: " << *
BaseFilename
1434 <<
" and test_profile: " << *
TestFilename <<
"\nProgram level:\n";
1436 OS <<
"Function level:\n"
1445 OS <<
" # of " << EntryName
1451 OS <<
" Mismatched count percentage (Edge): "
1454 OS <<
" Percentage of Edge profile only in test_profile: "
1461 for (
unsigned I = 0;
I < IPVK_Last - IPVK_First + 1;
I++) {
1464 char ProfileKindName[20];
1466 case IPVK_IndirectCallTarget:
1467 strncpy(ProfileKindName,
"IndirectCall", 19);
1469 case IPVK_MemOPSize:
1470 strncpy(ProfileKindName,
"MemOP", 19);
1473 snprintf(ProfileKindName, 19,
"VP[%d]",
I);
1476 OS <<
" " << ProfileKindName
1480 OS <<
" Mismatched count percentage (" << ProfileKindName
1483 OS <<
" Percentage of " << ProfileKindName
1484 <<
" profile only in test_profile: "
1486 OS <<
" " << ProfileKindName
1489 <<
" " << ProfileKindName
1495namespace IndexedInstrProf {
1497template <
typename T1,
typename T2>
1499 constexpr T2 Object{};
1500 return size_t(&(Object.*Member)) - size_t(&Object);
1508 using namespace support;
1509 return endian::byte_swap<uint64_t, llvm::endianness::little>(
Version);
1513 using namespace support;
1514 static_assert(std::is_standard_layout_v<Header>,
1515 "The header should be standard layout type since we use offset "
1516 "of fields to read.");
1522 endian::byte_swap<uint64_t, llvm::endianness::little>(
H.Magic);
1528 if (GET_VERSION(
H.formatVersion()) >
1532 switch (GET_VERSION(
H.formatVersion())) {
1537 "Please update the reading code below if a new field has been added, "
1538 "if not add a case statement to fall through to the latest version.");
1545 H.TemporalProfTracesOffset =
1568 "Please update the size computation below if a new field has "
1569 "been added to the header, if not add a case statement to "
1570 "fall through to the latest version.");
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_ATTRIBUTE_UNUSED
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))
static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))
static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
void setComdat(Comdat *C)
static bool isLocalLinkage(LinkageTypes Linkage)
LinkageTypes getLinkage() const
void setLinkage(LinkageTypes LT)
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Module * getParent()
Get the module that this global value is contained inside of...
bool isDiscardableIfUnused() const
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ LinkOnceAnyLinkage
Keep one copy of function when linking (inline)
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ ExternalWeakLinkage
ExternalWeak linkage description.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
std::string message() const override
Return the error message as a string.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
This is an important class for using LLVM in a threaded context.
ConstantAsMetadata * createConstant(Constant *C)
Return the given constant as metadata.
MDString * createString(StringRef Str)
Return the given string as metadata.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDString * get(LLVMContext &Context, StringRef Str)
A Module instance is used to store all the information related to an LLVM module.
const std::string & getSourceFileName() const
Get the module's original source file name.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
A vector that has set insertion semantics.
bool insert(const value_type &X)
Insert a new element into the SetVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
iterator_range< StringMapKeyIterator< ValueTy > > keys() const
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
std::string str() const
str - Get the contents as an std::string.
const unsigned char * bytes_end() const
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
static constexpr size_t npos
const unsigned char * bytes_begin() const
Triple - Helper class for working with autoconf configuration names.
bool supportsCOMDAT() const
Tests whether the target supports comdat.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
StringRef getName() const
Return a constant reference to the value's name.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
This class represents a function that is read from a sample profile.
@ C
The default llvm calling convention, compatible with C.
static uint64_t read(const unsigned char *Buffer, size_t Offset)
size_t constexpr offsetOf(T1 T2::*Member)
initializer< Ty > init(const Ty &Val)
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
constexpr int BestSizeCompression
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
bool is_separator(char value, Style style=Style::native)
Check whether the given char is a path separator on the host OS.
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
StringRef getPGOFuncNameMetadataName()
void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
auto partition_point(R &&Range, Predicate P)
Binary search for the first iterator in a range where a predicate is false.
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
std::pair< StringRef, StringRef > getParsedIRPGOName(StringRef IRPGOName)
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
constexpr char kGlobalIdentifierDelimiter
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
void sort(IteratorTy Start, IteratorTy End)
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.
static T swapToHostOrder(const unsigned char *&D, llvm::endianness Orig)
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, MDNode *PGONameMetadata)
@ value_site_count_mismatch
std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)
Multiply two unsigned integers, X and Y, of type T.
const std::error_category & instrprof_category()
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
static StringRef getStrippedSourceFileName(const GlobalObject &GO)
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Error collectGlobalObjectNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (names of global objects like functions or, virtual tables) NameStrs,...
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)
static ValueProfRecordClosure InstrProfRecordClosure
static Error readAndDecodeStrings(StringRef NameStrings, std::function< Error(StringRef)> NameCallback)
NameStrings is a string composed of one of more possibly encoded sub-strings.
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)
static std::optional< std::string > lookupPGONameFromMetadata(MDNode *MD)
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
void consumeError(Error Err)
Consume a Error without doing anything.
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
uint32_t getNumValueKindsInstrProf(const void *Record)
ValueProfRecordClosure Interface implementation for InstrProfRecord class.
ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)
uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)
static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName)
double ValueCounts[IPVK_Last - IPVK_First+1]
Profiling information for a single function.
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
void setPseudoCount(CountPseudoKind Kind)
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
std::vector< uint8_t > BitmapBytes
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site, uint64_t *TotalC=nullptr) const
Return the array of profiled values at Site.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
void sortByTargetValues()
Sort ValueData ascending by Value.
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
std::list< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
const std::string * TestFilename
CountSumOrPercent Mismatch
static std::vector< BPFunctionNode > createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...