63 using namespace support;
68 for (
int K = 0; K < NItems; K++) {
69 FDOStream.
seek(
P[K].Pos);
70 for (
int I = 0;
I <
P[K].N;
I++)
76 FDOStream.
seek(LastPos);
79 std::string &
Data = SOStream.
str();
80 for (
int K = 0; K < NItems; K++) {
81 for (
int I = 0;
I <
P[K].N;
I++) {
82 uint64_t Bytes = endian::byte_swap<uint64_t, little>(
P[K].
D[
I]);
84 (
const char *)&Bytes,
sizeof(
uint64_t));
118 static std::pair<offset_type, offset_type>
120 using namespace support;
128 for (
const auto &ProfileData : *V) {
135 M += ValueProfData::getSize(ProfileData.second);
139 return std::make_pair(
N, M);
147 using namespace support;
150 for (
const auto &ProfileData : *V) {
157 LE.write<
uint64_t>(ProfileData.first);
163 std::unique_ptr<ValueProfData> VDataPtr =
164 ValueProfData::serializeFrom(ProfileData.second);
167 Out.
write((
const char *)VDataPtr.get(), S);
186 this->Sparse = Sparse;
201 auto Hash =
Other.Hash;
202 Other.accumulateCounts(FuncLevelOverlap.
Test);
203 if (FunctionData.find(
Name) == FunctionData.end()) {
211 auto &ProfileDataMap = FunctionData[
Name];
214 std::tie(Where, NewFunc) =
226 Dest.
overlap(
Other, Overlap, FuncLevelOverlap, ValueCutoff);
232 auto &ProfileDataMap = FunctionData[
Name];
236 std::tie(Where, NewFunc) =
241 Warn(make_error<InstrProfError>(
E));
248 Dest.
scale(Weight, 1, MapWarn);
251 Dest.
merge(
I, Weight, MapWarn);
259 auto Result = MemProfRecordData.insert({Id,
Record});
271 auto Result = MemProfFrameData.insert({Id, Frame});
276 if (!Result.second && Result.first->second != Frame) {
278 "frame to id mapping mismatch"));
290 for (
auto &
I : IPW.FunctionData)
291 for (
auto &Func :
I.getValue())
292 addRecord(
I.getKey(), Func.first, std::move(Func.second), 1, Warn);
294 BinaryIds.reserve(BinaryIds.size() + IPW.BinaryIds.size());
295 for (
auto &
I : IPW.BinaryIds)
298 MemProfFrameData.reserve(IPW.MemProfFrameData.size());
299 for (
auto &
I : IPW.MemProfFrameData) {
306 MemProfRecordData.reserve(IPW.MemProfRecordData.size());
307 for (
auto &
I : IPW.MemProfRecordData) {
312bool InstrProfWriter::shouldEncodeData(
const ProfilingData &PD) {
315 for (
const auto &Func : PD) {
325 using namespace IndexedInstrProf;
336 for (
unsigned I = 0;
I < Res.size();
I++)
341 using namespace IndexedInstrProf;
342 using namespace support;
352 for (
const auto &
I : FunctionData)
353 if (shouldEncodeData(
I.getValue()))
354 Generator.insert(
I.getKey(), &
I.getValue());
361 Header.Version |= VARIANT_MASK_IR_PROF;
363 Header.Version |= VARIANT_MASK_CSIR_PROF;
364 if (
static_cast<bool>(ProfileKind &
366 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
368 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
370 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
372 Header.Version |= VARIANT_MASK_MEMPROF;
376 Header.HashOffset = 0;
377 Header.MemProfOffset = 0;
378 Header.BinaryIdOffset = 0;
384 for (
int I = 0;
I <
N - 3;
I++)
407 uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
410 for (
unsigned I = 0;
I < SummarySize /
sizeof(
uint64_t);
I++)
415 CSSummaryOffset = OS.
tell();
416 CSSummarySize = SummarySize /
sizeof(
uint64_t);
417 for (
unsigned I = 0;
I < CSSummarySize;
I++)
438 MemProfSectionStart = OS.
tell();
445 for (
const auto Id : Schema) {
449 auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
450 RecordWriter->Schema = &Schema;
452 RecordTableGenerator;
453 for (
auto &
I : MemProfRecordData) {
455 RecordTableGenerator.insert(
I.first,
I.second);
459 RecordTableGenerator.
Emit(OS.
OS, *RecordWriter);
463 auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
466 for (
auto &
I : MemProfFrameData) {
468 FrameTableGenerator.insert(
I.first,
I.second);
471 uint64_t FrameTableOffset = FrameTableGenerator.
Emit(OS.
OS, *FrameWriter);
474 {MemProfSectionStart, &RecordTableOffset, 1},
475 {MemProfSectionStart +
sizeof(
uint64_t), &FramePayloadOffset, 1},
476 {MemProfSectionStart + 2 *
sizeof(
uint64_t), &FrameTableOffset, 1},
478 OS.
patch(PatchItems, 3);
493 BinaryIds.erase(std::unique(BinaryIds.begin(), BinaryIds.end()),
496 for (
auto BI : BinaryIds) {
498 BinaryIdsSectionSize +=
sizeof(
uint64_t);
503 OS.
write(BinaryIdsSectionSize);
505 for (
auto BI : BinaryIds) {
510 for (
unsigned K = 0; K < BILen; K++)
514 for (
unsigned K = 0; K < PaddingSize; K++)
519 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
523 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
528 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary =
nullptr;
531 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
539 {HashTableStartFieldOffset, &HashTableStart, 1},
542 {MemProfSectionOffset, &MemProfSectionStart, 1},
544 {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
546 {SummaryOffset,
reinterpret_cast<uint64_t *
>(TheSummary.get()),
547 (
int)(SummarySize /
sizeof(
uint64_t))},
548 {CSSummaryOffset,
reinterpret_cast<uint64_t *
>(TheCSSummary.get()),
549 (
int)CSSummarySize}};
551 OS.
patch(PatchItems, std::size(PatchItems));
553 for (
const auto &
I : FunctionData)
554 for (
const auto &
F :
I.getValue())
564 return writeImpl(POS);
572 if (
Error E = writeImpl(POS))
579#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
584 for (
uint32_t VK = 0; VK <= IPVK_Last; VK++) {
585 uint32_t NS = Func.getNumValueSites(VK);
589 uint32_t ND = Func.getNumValueDataForSite(VK, S);
590 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
593 if ((VK != IPVK_IndirectCallTarget) && !SeenValues.
insert(VD[
I].Value).second)
606 OS <<
"# Func Hash:\n" << Hash <<
"\n";
607 OS <<
"# Num Counters:\n" << Func.Counts.size() <<
"\n";
608 OS <<
"# Counter Values:\n";
612 uint32_t NumValueKinds = Func.getNumValueKinds();
613 if (!NumValueKinds) {
618 OS <<
"# Num Value Kinds:\n" << Func.getNumValueKinds() <<
"\n";
619 for (
uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
620 uint32_t NS = Func.getNumValueSites(VK);
624 OS <<
"# NumValueSites:\n" << NS <<
"\n";
626 uint32_t ND = Func.getNumValueDataForSite(VK, S);
628 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
630 if (VK == IPVK_IndirectCallTarget)
632 << VD[
I].Count <<
"\n";
634 OS << VD[
I].Value <<
":" << VD[
I].Count <<
"\n";
645 OS <<
"# CSIR level Instrumentation Flag\n:csir\n";
647 OS <<
"# IR level Instrumentation Flag\n:ir\n";
649 if (
static_cast<bool>(ProfileKind &
651 OS <<
"# Always instrument the function entry block\n:entry_first\n";
655 using RecordType = std::pair<StringRef, FuncPair>;
658 for (
const auto &
I : FunctionData) {
659 if (shouldEncodeData(
I.getValue())) {
662 for (
const auto &Func :
I.getValue())
663 OrderedFuncData.
push_back(std::make_pair(
I.getKey(), Func));
668 return std::tie(
A.first,
A.second.first) <
669 std::tie(
B.first,
B.second.first);
672 for (
const auto &record : OrderedFuncData) {
674 const FuncPair &Func = record.second;
678 for (
const auto &record : OrderedFuncData) {
679 const FuncPair &Func = record.second;
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static void setSummary(IndexedInstrProf::Summary *TheSummary, ProfileSummary &PS)
static const char * ValueProfKindStr[]
Defines facilities for reading and writing on-disk hash tables.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
Implements a dense probed hash-table based set.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
static std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
const InstrProfWriter::ProfilingData *const data_type_ref
InstrProfSummaryBuilder * SummaryBuilder
static hash_value_type ComputeHash(key_type_ref K)
support::endianness ValueProfDataEndianness
InstrProfSummaryBuilder * CSSummaryBuilder
InstrProfRecordWriterTrait()=default
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N)
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
const InstrProfWriter::ProfilingData *const data_type
void addRecord(const InstrProfRecord &)
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash)
Just like getFuncName, except that it will return a non-empty StringRef if the function is external t...
InstrProfWriter(bool Sparse=false)
void setValueProfDataEndianness(support::endianness Endianness)
Error write(raw_fd_ostream &OS)
Write the profile to OS.
void overlapRecord(NamedInstrProfRecord &&Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, const OverlapFuncFilters &FuncFilter)
Error writeText(raw_fd_ostream &OS)
Write the profile in text format to OS.
void addBinaryIds(ArrayRef< llvm::object::BuildID > BIs)
void addMemProfRecord(const GlobalValue::GUID Id, const memprof::IndexedMemProfRecord &Record)
Add a memprof record for a function identified by its Id.
static void writeRecordInText(StringRef Name, uint64_t Hash, const InstrProfRecord &Counters, InstrProfSymtab &Symtab, raw_fd_ostream &OS)
Write Record in text format to OS.
void addRecord(NamedInstrProfRecord &&I, uint64_t Weight, function_ref< void(Error)> Warn)
Add function counts for the given function.
void mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref< void(Error)> Warn)
Merge existing function counts from the given writer.
std::unique_ptr< MemoryBuffer > writeBuffer()
Write the profile, returning the raw data. For testing.
bool addMemProfFrame(const memprof::FrameId, const memprof::Frame &F, function_ref< void(Error)> Warn)
Add a memprof frame identified by the hash of the contents of the frame in FrameId.
void setOutputSparse(bool Sparse)
Error validateRecord(const InstrProfRecord &Func)
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it.
Generates an on disk hash table.
offset_type Emit(raw_ostream &Out)
Emit the table to Out, which must not be at offset 0.
void patch(PatchItem *P, int NItems)
void writeByte(uint8_t V)
ProfOStream(raw_string_ostream &STR)
support::endian::Writer LE
ProfOStream(raw_fd_ostream &FD)
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
uint64_t getTotalCount() const
uint64_t getMaxCount() const
const SummaryEntryVector & getDetailedSummary()
uint32_t getNumCounts() const
uint64_t getMaxInternalCount() const
uint64_t getMaxFunctionCount() const
uint32_t getNumFunctions() const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
uint64_t seek(uint64_t off)
Flushes the stream and repositions the underlying file descriptor position to the offset specified fr...
This class implements an extremely fast bulk output stream that can only output to a stream.
uint64_t tell() const
tell - Return the current offset with the file.
raw_ostream & write(unsigned char C)
A raw_ostream that writes to an std::string.
std::string & str()
Returns the string's reference.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
uint64_t ComputeHash(StringRef K)
This is an optimization pass for GlobalISel generic memory operations.
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void sort(IteratorTy Start, IteratorTy End)
@ FunctionEntryInstrumentation
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
void set(SummaryFieldKind K, uint64_t V)
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
uint64_t NumSummaryFields
uint64_t NumCutoffEntries
Profiling information for a single function.
std::vector< uint64_t > Counts
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
void sortValueData()
Sort value profile data (per site) by count.
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
static bool hasCSFlagInHash(uint64_t FuncHash)
const std::string NameFilter
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
CountSumOrPercent Overlap
void addOneUnique(const CountSumOrPercent &UniqueFunc)
void merge(const IndexedMemProfRecord &Other)
static MemProfSchema getSchema()
Adapter to write values to a stream in a particular byte order.
void write(ArrayRef< value_type > Val)