64 using namespace support;
69 for (
const auto &K :
P) {
70 FDOStream.
seek(K.Pos);
71 for (
int I = 0;
I < K.N;
I++)
77 FDOStream.
seek(LastPos);
80 std::string &
Data = SOStream.
str();
81 for (
const auto &K :
P) {
82 for (
int I = 0;
I < K.N;
I++) {
84 endian::byte_swap<uint64_t, llvm::endianness::little>(K.D[
I]);
86 (
const char *)&Bytes,
sizeof(
uint64_t));
120 static std::pair<offset_type, offset_type>
122 using namespace support;
130 for (
const auto &ProfileData : *V) {
139 M += ValueProfData::getSize(ProfileData.second);
143 return std::make_pair(
N, M);
151 using namespace support;
154 for (
const auto &ProfileData : *V) {
161 LE.write<
uint64_t>(ProfileData.first);
171 std::unique_ptr<ValueProfData> VDataPtr =
172 ValueProfData::serializeFrom(ProfileData.second);
175 Out.
write((
const char *)VDataPtr.get(), S);
183 uint64_t TemporalProfTraceReservoirSize,
184 uint64_t MaxTemporalProfTraceLength,
185 bool WritePrevVersion)
186 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
187 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
189 WritePrevVersion(WritePrevVersion) {}
199 this->Sparse = Sparse;
214 auto Hash =
Other.Hash;
215 Other.accumulateCounts(FuncLevelOverlap.
Test);
216 if (!FunctionData.contains(
Name)) {
224 auto &ProfileDataMap = FunctionData[
Name];
227 std::tie(Where, NewFunc) =
239 Dest.
overlap(
Other, Overlap, FuncLevelOverlap, ValueCutoff);
245 auto &ProfileDataMap = FunctionData[
Name];
249 std::tie(Where, NewFunc) =
254 Warn(make_error<InstrProfError>(
E));
261 Dest.
scale(Weight, 1, MapWarn);
264 Dest.
merge(
I, Weight, MapWarn);
272 auto Result = MemProfRecordData.insert({Id,
Record});
284 auto Result = MemProfFrameData.insert({Id, Frame});
289 if (!Result.second && Result.first->second != Frame) {
291 "frame to id mapping mismatch"));
302 if (
Trace.FunctionNameRefs.
size() > MaxTemporalProfTraceLength)
303 Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);
307 if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
309 TemporalProfTraces.push_back(std::move(
Trace));
312 std::uniform_int_distribution<uint64_t> Distribution(
313 0, TemporalProfTraceStreamSize);
314 uint64_t RandomIndex = Distribution(RNG);
315 if (RandomIndex < TemporalProfTraces.size())
316 TemporalProfTraces[RandomIndex] = std::move(
Trace);
318 ++TemporalProfTraceStreamSize;
326 (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize);
327 bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize);
328 if (!IsDestSampled && IsSrcSampled) {
330 std::swap(TemporalProfTraces, SrcTraces);
331 std::swap(TemporalProfTraceStreamSize, SrcStreamSize);
336 for (
auto &
Trace : SrcTraces)
337 addTemporalProfileTrace(std::move(
Trace));
344 std::uniform_int_distribution<uint64_t> Distribution(
345 0, TemporalProfTraceStreamSize);
346 uint64_t RandomIndex = Distribution(RNG);
347 if (RandomIndex < TemporalProfTraces.size())
348 IndicesToReplace.
insert(RandomIndex);
349 ++TemporalProfTraceStreamSize;
359 for (
auto &
I : IPW.FunctionData)
360 for (
auto &Func :
I.getValue())
361 addRecord(
I.getKey(), Func.first, std::move(Func.second), 1, Warn);
363 BinaryIds.reserve(BinaryIds.size() + IPW.BinaryIds.size());
364 for (
auto &
I : IPW.BinaryIds)
368 IPW.TemporalProfTraceStreamSize);
370 MemProfFrameData.reserve(IPW.MemProfFrameData.size());
371 for (
auto &
I : IPW.MemProfFrameData) {
378 MemProfRecordData.reserve(IPW.MemProfRecordData.size());
379 for (
auto &
I : IPW.MemProfRecordData) {
384bool InstrProfWriter::shouldEncodeData(
const ProfilingData &PD) {
387 for (
const auto &Func : PD) {
399 using namespace IndexedInstrProf;
410 for (
unsigned I = 0;
I < Res.size();
I++)
415 using namespace IndexedInstrProf;
416 using namespace support;
427 for (
const auto &
I : FunctionData)
428 if (shouldEncodeData(
I.getValue()))
431 for (
const auto &
I : OrderedData)
432 Generator.insert(
I.first,
I.second);
437 Header.Version = WritePrevVersion
445 Header.Version |= VARIANT_MASK_IR_PROF;
447 Header.Version |= VARIANT_MASK_CSIR_PROF;
448 if (
static_cast<bool>(ProfileKind &
450 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
452 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
454 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
456 Header.Version |= VARIANT_MASK_MEMPROF;
458 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
462 Header.HashOffset = 0;
463 Header.MemProfOffset = 0;
464 Header.BinaryIdOffset = 0;
465 Header.TemporalProfTracesOffset = 0;
466 Header.VTableNamesOffset = 0;
470 for (
int I = 0;
I < 4;
I++)
495 if (!WritePrevVersion)
500 uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
503 for (
unsigned I = 0;
I < SummarySize /
sizeof(
uint64_t);
I++)
508 CSSummaryOffset =
OS.
tell();
509 CSSummarySize = SummarySize /
sizeof(
uint64_t);
510 for (
unsigned I = 0;
I < CSSummarySize;
I++)
531 MemProfSectionStart =
OS.
tell();
538 for (
const auto Id : Schema) {
542 auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
543 RecordWriter->Schema = &Schema;
545 RecordTableGenerator;
546 for (
auto &
I : MemProfRecordData) {
548 RecordTableGenerator.insert(
I.first,
I.second);
551 MemProfRecordData.clear();
557 RecordTableGenerator.
Emit(
OS.OS, *RecordWriter);
561 auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
564 for (
auto &
I : MemProfFrameData) {
566 FrameTableGenerator.insert(
I.first,
I.second);
569 MemProfFrameData.clear();
571 uint64_t FrameTableOffset = FrameTableGenerator.
Emit(
OS.OS, *FrameWriter);
574 {MemProfSectionStart, &RecordTableOffset, 1},
575 {MemProfSectionStart +
sizeof(
uint64_t), &FramePayloadOffset, 1},
576 {MemProfSectionStart + 2 *
sizeof(
uint64_t), &FrameTableOffset, 1},
578 OS.patch(PatchItems);
593 BinaryIds.erase(std::unique(BinaryIds.begin(), BinaryIds.end()),
596 for (
auto BI : BinaryIds) {
598 BinaryIdsSectionSize +=
sizeof(
uint64_t);
603 OS.
write(BinaryIdsSectionSize);
605 for (
auto BI : BinaryIds) {
610 for (
unsigned K = 0;
K < BILen;
K++)
614 for (
unsigned K = 0;
K < PaddingSize;
K++)
620 if (!WritePrevVersion) {
625 std::string CompressedVTableNames =
"VTableNames";
627 uint64_t CompressedStringLen = CompressedVTableNames.length();
633 for (
auto &c : CompressedVTableNames)
634 OS.writeByte(
static_cast<uint8_t
>(c));
640 for (
uint64_t K = CompressedStringLen;
K < PaddedLength;
K++) {
645 uint64_t TemporalProfTracesSectionStart = 0;
647 TemporalProfTracesSectionStart =
OS.
tell();
648 OS.
write(TemporalProfTraces.size());
649 OS.
write(TemporalProfTraceStreamSize);
650 for (
auto &
Trace : TemporalProfTraces) {
653 for (
auto &NameRef :
Trace.FunctionNameRefs)
659 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
663 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
668 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary =
nullptr;
671 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
676 if (!WritePrevVersion) {
680 {HashTableStartFieldOffset, &HashTableStart, 1},
683 {MemProfSectionOffset, &MemProfSectionStart, 1},
685 {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
688 {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
689 {VTableNamesOffset, &VTableNamesSectionStart, 1},
691 {SummaryOffset,
reinterpret_cast<uint64_t *
>(TheSummary.get()),
692 (
int)(SummarySize /
sizeof(
uint64_t))},
693 {CSSummaryOffset,
reinterpret_cast<uint64_t *
>(TheCSSummary.get()),
694 (
int)CSSummarySize}};
696 OS.patch(PatchItems);
701 {HashTableStartFieldOffset, &HashTableStart, 1},
704 {MemProfSectionOffset, &MemProfSectionStart, 1},
706 {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
709 {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
711 {SummaryOffset,
reinterpret_cast<uint64_t *
>(TheSummary.get()),
712 (
int)(SummarySize /
sizeof(
uint64_t))},
713 {CSSummaryOffset,
reinterpret_cast<uint64_t *
>(TheCSSummary.get()),
714 (
int)CSSummarySize}};
716 OS.patch(PatchItems);
719 for (
const auto &
I : FunctionData)
720 for (
const auto &
F :
I.getValue())
730 return writeImpl(POS);
735 return writeImpl(POS);
749#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
754 for (
uint32_t VK = 0; VK <= IPVK_Last; VK++) {
755 uint32_t NS = Func.getNumValueSites(VK);
759 uint32_t ND = Func.getNumValueDataForSite(VK, S);
760 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
763 if ((VK != IPVK_IndirectCallTarget && VK != IPVK_VTableTarget) &&
764 !SeenValues.
insert(VD[
I].Value).second)
777 OS <<
"# Func Hash:\n" << Hash <<
"\n";
778 OS <<
"# Num Counters:\n" << Func.Counts.size() <<
"\n";
779 OS <<
"# Counter Values:\n";
783 if (Func.BitmapBytes.size() > 0) {
784 OS <<
"# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() <<
"\n";
785 OS <<
"# Bitmap Byte Values:\n";
786 for (uint8_t Byte : Func.BitmapBytes) {
794 uint32_t NumValueKinds = Func.getNumValueKinds();
795 if (!NumValueKinds) {
800 OS <<
"# Num Value Kinds:\n" << Func.getNumValueKinds() <<
"\n";
801 for (
uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
802 uint32_t NS = Func.getNumValueSites(VK);
806 OS <<
"# NumValueSites:\n" << NS <<
"\n";
808 uint32_t ND = Func.getNumValueDataForSite(VK, S);
810 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
812 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
814 << VD[
I].Count <<
"\n";
816 OS << VD[
I].Value <<
":" << VD[
I].Count <<
"\n";
827 OS <<
"# CSIR level Instrumentation Flag\n:csir\n";
829 OS <<
"# IR level Instrumentation Flag\n:ir\n";
831 if (
static_cast<bool>(ProfileKind &
833 OS <<
"# Always instrument the function entry block\n:entry_first\n";
835 OS <<
"# Instrument block coverage\n:single_byte_coverage\n";
839 using RecordType = std::pair<StringRef, FuncPair>;
842 for (
const auto &
I : FunctionData) {
843 if (shouldEncodeData(
I.getValue())) {
846 for (
const auto &Func :
I.getValue())
847 OrderedFuncData.
push_back(std::make_pair(
I.getKey(), Func));
855 return std::tie(
A.first,
A.second.first) <
856 std::tie(
B.first,
B.second.first);
859 for (
const auto &record : OrderedFuncData) {
861 const FuncPair &Func = record.second;
865 for (
const auto &record : OrderedFuncData) {
866 const FuncPair &Func = record.second;
876 OS <<
":temporal_prof_traces\n";
877 OS <<
"# Num Temporal Profile Traces:\n" << TemporalProfTraces.
size() <<
"\n";
878 OS <<
"# Temporal Profile Trace Stream Size:\n"
879 << TemporalProfTraceStreamSize <<
"\n";
880 for (
auto &
Trace : TemporalProfTraces) {
881 OS <<
"# Weight:\n" <<
Trace.Weight <<
"\n";
882 for (
auto &NameRef :
Trace.FunctionNameRefs)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static void setSummary(IndexedInstrProf::Summary *TheSummary, ProfileSummary &PS)
static const char * ValueProfKindStr[]
Defines facilities for reading and writing on-disk hash tables.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
Implements a dense probed hash-table based set.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
static std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
const InstrProfWriter::ProfilingData *const data_type_ref
InstrProfSummaryBuilder * SummaryBuilder
llvm::endianness ValueProfDataEndianness
static hash_value_type ComputeHash(key_type_ref K)
InstrProfSummaryBuilder * CSSummaryBuilder
InstrProfRecordWriterTrait()=default
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N)
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
const InstrProfWriter::ProfilingData *const data_type
void addRecord(const InstrProfRecord &)
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
StringRef getFuncOrVarName(uint64_t ValMD5Hash)
Return name of functions or global variables from the name's md5 hash value.
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash)
Just like getFuncOrVarName, except that it will return literal string 'External Symbol' if the functi...
Error write(raw_fd_ostream &OS)
Write the profile to OS.
void addTemporalProfileTraces(SmallVectorImpl< TemporalProfTraceTy > &SrcTraces, uint64_t SrcStreamSize)
Add SrcTraces using reservoir sampling where SrcStreamSize is the total number of temporal profiling ...
void overlapRecord(NamedInstrProfRecord &&Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, const OverlapFuncFilters &FuncFilter)
Error writeText(raw_fd_ostream &OS)
Write the profile in text format to OS.
InstrProfWriter(bool Sparse=false, uint64_t TemporalProfTraceReservoirSize=0, uint64_t MaxTemporalProfTraceLength=0, bool WritePrevVersion=false)
void addBinaryIds(ArrayRef< llvm::object::BuildID > BIs)
void addMemProfRecord(const GlobalValue::GUID Id, const memprof::IndexedMemProfRecord &Record)
Add a memprof record for a function identified by its Id.
static void writeRecordInText(StringRef Name, uint64_t Hash, const InstrProfRecord &Counters, InstrProfSymtab &Symtab, raw_fd_ostream &OS)
Write Record in text format to OS.
void setValueProfDataEndianness(llvm::endianness Endianness)
void addRecord(NamedInstrProfRecord &&I, uint64_t Weight, function_ref< void(Error)> Warn)
Add function counts for the given function.
void mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref< void(Error)> Warn)
Merge existing function counts from the given writer.
void writeTextTemporalProfTraceData(raw_fd_ostream &OS, InstrProfSymtab &Symtab)
Write temporal profile trace data to the header in text format to OS.
std::unique_ptr< MemoryBuffer > writeBuffer()
Write the profile, returning the raw data. For testing.
bool addMemProfFrame(const memprof::FrameId, const memprof::Frame &F, function_ref< void(Error)> Warn)
Add a memprof frame identified by the hash of the contents of the frame in FrameId.
void setOutputSparse(bool Sparse)
Error validateRecord(const InstrProfRecord &Func)
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it.
Generates an on disk hash table.
offset_type Emit(raw_ostream &Out)
Emit the table to Out, which must not be at offset 0.
void writeByte(uint8_t V)
void patch(ArrayRef< PatchItem > P)
ProfOStream(raw_string_ostream &STR)
support::endian::Writer LE
ProfOStream(raw_fd_ostream &FD)
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
uint64_t getTotalCount() const
uint64_t getMaxCount() const
const SummaryEntryVector & getDetailedSummary()
uint32_t getNumCounts() const
uint64_t getMaxInternalCount() const
uint64_t getMaxFunctionCount() const
uint32_t getNumFunctions() const
bool insert(const value_type &X)
Insert a new element into the SetVector.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
uint64_t seek(uint64_t off)
Flushes the stream and repositions the underlying file descriptor position to the offset specified fr...
This class implements an extremely fast bulk output stream that can only output to a stream.
uint64_t tell() const
tell - Return the current offset with the file.
raw_ostream & write_hex(unsigned long long N)
Output N in hexadecimal, without any prefix or padding.
raw_ostream & write(unsigned char C)
A raw_ostream that writes to an std::string.
std::string & str()
Returns the string's reference.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
uint64_t ComputeHash(StringRef K)
This is an optimization pass for GlobalISel generic memory operations.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
void shuffle(Iterator first, Iterator last, RNG &&g)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void sort(IteratorTy Start, IteratorTy End)
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
@ FunctionEntryInstrumentation
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
void set(SummaryFieldKind K, uint64_t V)
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
uint64_t NumSummaryFields
uint64_t NumCutoffEntries
Profiling information for a single function.
std::vector< uint64_t > Counts
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
void sortValueData()
Sort value profile data (per site) by count.
std::vector< uint8_t > BitmapBytes
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
static bool hasCSFlagInHash(uint64_t FuncHash)
const std::string NameFilter
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
Function object to check whether the first component of a container supported by std::get (like std::...
void merge(const IndexedMemProfRecord &Other)
static MemProfSchema getSchema()
Adapter to write values to a stream in a particular byte order.
void write(ArrayRef< value_type > Val)