64 using namespace support;
69 for (
int K = 0; K < NItems; K++) {
70 FDOStream.
seek(
P[K].Pos);
71 for (
int I = 0;
I <
P[K].N;
I++)
77 FDOStream.
seek(LastPos);
80 std::string &
Data = SOStream.
str();
81 for (
int K = 0; K < NItems; K++) {
82 for (
int I = 0;
I <
P[K].N;
I++) {
83 uint64_t Bytes = endian::byte_swap<uint64_t, little>(
P[K].
D[
I]);
85 (
const char *)&Bytes,
sizeof(
uint64_t));
119 static std::pair<offset_type, offset_type>
121 using namespace support;
129 for (
const auto &ProfileData : *V) {
136 M += ValueProfData::getSize(ProfileData.second);
140 return std::make_pair(
N, M);
148 using namespace support;
151 for (
const auto &ProfileData : *V) {
158 LE.write<
uint64_t>(ProfileData.first);
164 std::unique_ptr<ValueProfData> VDataPtr =
165 ValueProfData::serializeFrom(ProfileData.second);
168 Out.
write((
const char *)VDataPtr.get(), S);
176 uint64_t TemporalProfTraceReservoirSize,
177 uint64_t MaxTemporalProfTraceLength)
178 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
179 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
191 this->Sparse = Sparse;
206 auto Hash =
Other.Hash;
207 Other.accumulateCounts(FuncLevelOverlap.
Test);
208 if (!FunctionData.contains(
Name)) {
216 auto &ProfileDataMap = FunctionData[
Name];
219 std::tie(Where, NewFunc) =
231 Dest.
overlap(
Other, Overlap, FuncLevelOverlap, ValueCutoff);
237 auto &ProfileDataMap = FunctionData[
Name];
241 std::tie(Where, NewFunc) =
246 Warn(make_error<InstrProfError>(
E));
253 Dest.
scale(Weight, 1, MapWarn);
256 Dest.
merge(
I, Weight, MapWarn);
264 auto Result = MemProfRecordData.insert({Id,
Record});
276 auto Result = MemProfFrameData.insert({Id, Frame});
281 if (!Result.second && Result.first->second != Frame) {
283 "frame to id mapping mismatch"));
294 if (
Trace.FunctionNameRefs.
size() > MaxTemporalProfTraceLength)
295 Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);
299 if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
301 TemporalProfTraces.push_back(std::move(
Trace));
304 std::uniform_int_distribution<uint64_t> Distribution(
305 0, TemporalProfTraceStreamSize);
306 uint64_t RandomIndex = Distribution(RNG);
307 if (RandomIndex < TemporalProfTraces.size())
308 TemporalProfTraces[RandomIndex] = std::move(
Trace);
310 ++TemporalProfTraceStreamSize;
318 (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize);
319 bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize);
320 if (!IsDestSampled && IsSrcSampled) {
322 std::swap(TemporalProfTraces, SrcTraces);
323 std::swap(TemporalProfTraceStreamSize, SrcStreamSize);
328 for (
auto &
Trace : SrcTraces)
329 addTemporalProfileTrace(std::move(
Trace));
336 std::uniform_int_distribution<uint64_t> Distribution(
337 0, TemporalProfTraceStreamSize);
338 uint64_t RandomIndex = Distribution(RNG);
339 if (RandomIndex < TemporalProfTraces.size())
340 IndicesToReplace.
insert(RandomIndex);
341 ++TemporalProfTraceStreamSize;
351 for (
auto &
I : IPW.FunctionData)
352 for (
auto &Func :
I.getValue())
353 addRecord(
I.getKey(), Func.first, std::move(Func.second), 1, Warn);
355 BinaryIds.reserve(BinaryIds.size() + IPW.BinaryIds.size());
356 for (
auto &
I : IPW.BinaryIds)
360 IPW.TemporalProfTraceStreamSize);
362 MemProfFrameData.reserve(IPW.MemProfFrameData.size());
363 for (
auto &
I : IPW.MemProfFrameData) {
370 MemProfRecordData.reserve(IPW.MemProfRecordData.size());
371 for (
auto &
I : IPW.MemProfRecordData) {
376bool InstrProfWriter::shouldEncodeData(
const ProfilingData &PD) {
379 for (
const auto &Func : PD) {
389 using namespace IndexedInstrProf;
400 for (
unsigned I = 0;
I < Res.size();
I++)
405 using namespace IndexedInstrProf;
406 using namespace support;
416 for (
const auto &
I : FunctionData)
417 if (shouldEncodeData(
I.getValue()))
418 Generator.insert(
I.getKey(), &
I.getValue());
425 Header.Version |= VARIANT_MASK_IR_PROF;
427 Header.Version |= VARIANT_MASK_CSIR_PROF;
428 if (
static_cast<bool>(ProfileKind &
430 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
432 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
434 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
436 Header.Version |= VARIANT_MASK_MEMPROF;
438 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
442 Header.HashOffset = 0;
443 Header.MemProfOffset = 0;
444 Header.BinaryIdOffset = 0;
445 Header.TemporalProfTracesOffset = 0;
451 for (
int I = 0;
I <
N - 4;
I++)
477 uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
480 for (
unsigned I = 0;
I < SummarySize /
sizeof(
uint64_t);
I++)
485 CSSummaryOffset =
OS.
tell();
486 CSSummarySize = SummarySize /
sizeof(
uint64_t);
487 for (
unsigned I = 0;
I < CSSummarySize;
I++)
508 MemProfSectionStart =
OS.
tell();
515 for (
const auto Id : Schema) {
519 auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
520 RecordWriter->Schema = &Schema;
522 RecordTableGenerator;
523 for (
auto &
I : MemProfRecordData) {
525 RecordTableGenerator.insert(
I.first,
I.second);
529 RecordTableGenerator.
Emit(
OS.OS, *RecordWriter);
533 auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
536 for (
auto &
I : MemProfFrameData) {
538 FrameTableGenerator.insert(
I.first,
I.second);
541 uint64_t FrameTableOffset = FrameTableGenerator.
Emit(
OS.OS, *FrameWriter);
544 {MemProfSectionStart, &RecordTableOffset, 1},
545 {MemProfSectionStart +
sizeof(
uint64_t), &FramePayloadOffset, 1},
546 {MemProfSectionStart + 2 *
sizeof(
uint64_t), &FrameTableOffset, 1},
548 OS.patch(PatchItems, 3);
563 BinaryIds.erase(std::unique(BinaryIds.begin(), BinaryIds.end()),
566 for (
auto BI : BinaryIds) {
568 BinaryIdsSectionSize +=
sizeof(
uint64_t);
573 OS.
write(BinaryIdsSectionSize);
575 for (
auto BI : BinaryIds) {
580 for (
unsigned K = 0;
K < BILen;
K++)
584 for (
unsigned K = 0;
K < PaddingSize;
K++)
588 uint64_t TemporalProfTracesSectionStart = 0;
590 TemporalProfTracesSectionStart =
OS.
tell();
591 OS.
write(TemporalProfTraces.size());
592 OS.
write(TemporalProfTraceStreamSize);
593 for (
auto &
Trace : TemporalProfTraces) {
596 for (
auto &NameRef :
Trace.FunctionNameRefs)
602 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
606 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
611 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary =
nullptr;
614 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
622 {HashTableStartFieldOffset, &HashTableStart, 1},
625 {MemProfSectionOffset, &MemProfSectionStart, 1},
627 {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
630 {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
632 {SummaryOffset,
reinterpret_cast<uint64_t *
>(TheSummary.get()),
633 (
int)(SummarySize /
sizeof(
uint64_t))},
634 {CSSummaryOffset,
reinterpret_cast<uint64_t *
>(TheCSSummary.get()),
635 (
int)CSSummarySize}};
637 OS.patch(PatchItems, std::size(PatchItems));
639 for (
const auto &
I : FunctionData)
640 for (
const auto &
F :
I.getValue())
650 return writeImpl(POS);
658 if (
Error E = writeImpl(POS))
665#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
670 for (
uint32_t VK = 0; VK <= IPVK_Last; VK++) {
671 uint32_t NS = Func.getNumValueSites(VK);
675 uint32_t ND = Func.getNumValueDataForSite(VK, S);
676 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
679 if ((VK != IPVK_IndirectCallTarget) && !SeenValues.
insert(VD[
I].Value).second)
692 OS <<
"# Func Hash:\n" << Hash <<
"\n";
693 OS <<
"# Num Counters:\n" << Func.Counts.size() <<
"\n";
694 OS <<
"# Counter Values:\n";
698 uint32_t NumValueKinds = Func.getNumValueKinds();
699 if (!NumValueKinds) {
704 OS <<
"# Num Value Kinds:\n" << Func.getNumValueKinds() <<
"\n";
705 for (
uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
706 uint32_t NS = Func.getNumValueSites(VK);
710 OS <<
"# NumValueSites:\n" << NS <<
"\n";
712 uint32_t ND = Func.getNumValueDataForSite(VK, S);
714 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
716 if (VK == IPVK_IndirectCallTarget)
718 << VD[
I].Count <<
"\n";
720 OS << VD[
I].Value <<
":" << VD[
I].Count <<
"\n";
731 OS <<
"# CSIR level Instrumentation Flag\n:csir\n";
733 OS <<
"# IR level Instrumentation Flag\n:ir\n";
735 if (
static_cast<bool>(ProfileKind &
737 OS <<
"# Always instrument the function entry block\n:entry_first\n";
741 using RecordType = std::pair<StringRef, FuncPair>;
744 for (
const auto &
I : FunctionData) {
745 if (shouldEncodeData(
I.getValue())) {
748 for (
const auto &Func :
I.getValue())
749 OrderedFuncData.
push_back(std::make_pair(
I.getKey(), Func));
757 return std::tie(
A.first,
A.second.first) <
758 std::tie(
B.first,
B.second.first);
761 for (
const auto &record : OrderedFuncData) {
763 const FuncPair &Func = record.second;
767 for (
const auto &record : OrderedFuncData) {
768 const FuncPair &Func = record.second;
778 OS <<
":temporal_prof_traces\n";
779 OS <<
"# Num Temporal Profile Traces:\n" << TemporalProfTraces.
size() <<
"\n";
780 OS <<
"# Temporal Profile Trace Stream Size:\n"
781 << TemporalProfTraceStreamSize <<
"\n";
782 for (
auto &
Trace : TemporalProfTraces) {
783 OS <<
"# Weight:\n" <<
Trace.Weight <<
"\n";
784 for (
auto &NameRef :
Trace.FunctionNameRefs)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static void setSummary(IndexedInstrProf::Summary *TheSummary, ProfileSummary &PS)
static const char * ValueProfKindStr[]
Defines facilities for reading and writing on-disk hash tables.
This file implements a set that has insertion order iteration characteristics.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
Implements a dense probed hash-table based set.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
static std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
const InstrProfWriter::ProfilingData *const data_type_ref
InstrProfSummaryBuilder * SummaryBuilder
static hash_value_type ComputeHash(key_type_ref K)
support::endianness ValueProfDataEndianness
InstrProfSummaryBuilder * CSSummaryBuilder
InstrProfRecordWriterTrait()=default
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N)
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
const InstrProfWriter::ProfilingData *const data_type
void addRecord(const InstrProfRecord &)
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash)
Just like getFuncName, except that it will return a non-empty StringRef if the function is external t...
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
void setValueProfDataEndianness(support::endianness Endianness)
Error write(raw_fd_ostream &OS)
Write the profile to OS.
void addTemporalProfileTraces(SmallVectorImpl< TemporalProfTraceTy > &SrcTraces, uint64_t SrcStreamSize)
Add SrcTraces using reservoir sampling where SrcStreamSize is the total number of temporal profiling ...
void overlapRecord(NamedInstrProfRecord &&Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, const OverlapFuncFilters &FuncFilter)
Error writeText(raw_fd_ostream &OS)
Write the profile in text format to OS.
InstrProfWriter(bool Sparse=false, uint64_t TemporalProfTraceReservoirSize=0, uint64_t MaxTemporalProfTraceLength=0)
void addBinaryIds(ArrayRef< llvm::object::BuildID > BIs)
void addMemProfRecord(const GlobalValue::GUID Id, const memprof::IndexedMemProfRecord &Record)
Add a memprof record for a function identified by its Id.
static void writeRecordInText(StringRef Name, uint64_t Hash, const InstrProfRecord &Counters, InstrProfSymtab &Symtab, raw_fd_ostream &OS)
Write Record in text format to OS.
void addRecord(NamedInstrProfRecord &&I, uint64_t Weight, function_ref< void(Error)> Warn)
Add function counts for the given function.
void mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref< void(Error)> Warn)
Merge existing function counts from the given writer.
void writeTextTemporalProfTraceData(raw_fd_ostream &OS, InstrProfSymtab &Symtab)
Write temporal profile trace data to the header in text format to OS.
std::unique_ptr< MemoryBuffer > writeBuffer()
Write the profile, returning the raw data. For testing.
bool addMemProfFrame(const memprof::FrameId, const memprof::Frame &F, function_ref< void(Error)> Warn)
Add a memprof frame identified by the hash of the contents of the frame in FrameId.
void setOutputSparse(bool Sparse)
Error validateRecord(const InstrProfRecord &Func)
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it.
Generates an on disk hash table.
offset_type Emit(raw_ostream &Out)
Emit the table to Out, which must not be at offset 0.
void patch(PatchItem *P, int NItems)
void writeByte(uint8_t V)
ProfOStream(raw_string_ostream &STR)
support::endian::Writer LE
ProfOStream(raw_fd_ostream &FD)
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
uint64_t getTotalCount() const
uint64_t getMaxCount() const
const SummaryEntryVector & getDetailedSummary()
uint32_t getNumCounts() const
uint64_t getMaxInternalCount() const
uint64_t getMaxFunctionCount() const
uint32_t getNumFunctions() const
bool insert(const value_type &X)
Insert a new element into the SetVector.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
uint64_t seek(uint64_t off)
Flushes the stream and repositions the underlying file descriptor position to the offset specified fr...
This class implements an extremely fast bulk output stream that can only output to a stream.
uint64_t tell() const
tell - Return the current offset with the file.
raw_ostream & write(unsigned char C)
A raw_ostream that writes to an std::string.
std::string & str()
Returns the string's reference.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
uint64_t ComputeHash(StringRef K)
This is an optimization pass for GlobalISel generic memory operations.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
void shuffle(Iterator first, Iterator last, RNG &&g)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void sort(IteratorTy Start, IteratorTy End)
@ FunctionEntryInstrumentation
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
void set(SummaryFieldKind K, uint64_t V)
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
uint64_t NumSummaryFields
uint64_t NumCutoffEntries
Profiling information for a single function.
std::vector< uint64_t > Counts
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
void sortValueData()
Sort value profile data (per site) by count.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
static bool hasCSFlagInHash(uint64_t FuncHash)
const std::string NameFilter
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
void merge(const IndexedMemProfRecord &Other)
static MemProfSchema getSchema()
Adapter to write values to a stream in a particular byte order.
void write(ArrayRef< value_type > Val)