LLVM 19.0.0git
InstrProfReader.cpp
Go to the documentation of this file.
1//===- InstrProfReader.cpp - Instrumented profiling reader ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for reading profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/DenseMap.h"
18#include "llvm/ADT/StringRef.h"
24#include "llvm/Support/Endian.h"
25#include "llvm/Support/Error.h"
31#include <algorithm>
32#include <cstddef>
33#include <cstdint>
34#include <limits>
35#include <memory>
36#include <optional>
37#include <system_error>
38#include <utility>
39#include <vector>
40
41using namespace llvm;
42
43// Extracts the variant information from the top 32 bits in the version and
44// returns an enum specifying the variants present.
46 InstrProfKind ProfileKind = InstrProfKind::Unknown;
47 if (Version & VARIANT_MASK_IR_PROF) {
48 ProfileKind |= InstrProfKind::IRInstrumentation;
49 }
50 if (Version & VARIANT_MASK_CSIR_PROF) {
51 ProfileKind |= InstrProfKind::ContextSensitive;
52 }
53 if (Version & VARIANT_MASK_INSTR_ENTRY) {
54 ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;
55 }
56 if (Version & VARIANT_MASK_BYTE_COVERAGE) {
57 ProfileKind |= InstrProfKind::SingleByteCoverage;
58 }
59 if (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) {
60 ProfileKind |= InstrProfKind::FunctionEntryOnly;
61 }
62 if (Version & VARIANT_MASK_MEMPROF) {
63 ProfileKind |= InstrProfKind::MemProf;
64 }
65 if (Version & VARIANT_MASK_TEMPORAL_PROF) {
66 ProfileKind |= InstrProfKind::TemporalProfile;
67 }
68 return ProfileKind;
69}
70
73 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
74 : FS.getBufferForFile(Filename);
75 if (std::error_code EC = BufferOrErr.getError())
76 return errorCodeToError(EC);
77 return std::move(BufferOrErr.get());
78}
79
81 return Reader.readHeader();
82}
83
84/// Read a list of binary ids from a profile that consist of
85/// a. uint64_t binary id length
86/// b. uint8_t binary id data
87/// c. uint8_t padding (if necessary)
88/// This function is shared between raw and indexed profiles.
89/// Raw profiles are in host-endian format, and indexed profiles are in
90/// little-endian format. So, this function takes an argument indicating the
91/// associated endian format to read the binary ids correctly.
92static Error
94 const uint64_t BinaryIdsSize,
95 const uint8_t *BinaryIdsStart,
96 std::vector<llvm::object::BuildID> &BinaryIds,
97 const llvm::endianness Endian) {
98 using namespace support;
99
100 if (BinaryIdsSize == 0)
101 return Error::success();
102
103 const uint8_t *BI = BinaryIdsStart;
104 const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize;
105 const uint8_t *End =
106 reinterpret_cast<const uint8_t *>(DataBuffer.getBufferEnd());
107
108 while (BI < BIEnd) {
109 size_t Remaining = BIEnd - BI;
110 // There should be enough left to read the binary id length.
111 if (Remaining < sizeof(uint64_t))
112 return make_error<InstrProfError>(
113 instrprof_error::malformed,
114 "not enough data to read binary id length");
115
116 uint64_t BILen = 0;
118 BILen = endian::readNext<uint64_t, llvm::endianness::little>(BI);
119 else
120 BILen = endian::readNext<uint64_t, llvm::endianness::big>(BI);
121
122 if (BILen == 0)
123 return make_error<InstrProfError>(instrprof_error::malformed,
124 "binary id length is 0");
125
126 Remaining = BIEnd - BI;
127 // There should be enough left to read the binary id data.
128 if (Remaining < alignToPowerOf2(BILen, sizeof(uint64_t)))
129 return make_error<InstrProfError>(
130 instrprof_error::malformed, "not enough data to read binary id data");
131
132 // Add binary id to the binary ids list.
133 BinaryIds.push_back(object::BuildID(BI, BI + BILen));
134
135 // Increment by binary id data length, which aligned to the size of uint64.
136 BI += alignToPowerOf2(BILen, sizeof(uint64_t));
137 if (BI > End)
138 return make_error<InstrProfError>(
139 instrprof_error::malformed,
140 "binary id section is greater than buffer size");
141 }
142
143 return Error::success();
144}
145
146static void
148 std::vector<llvm::object::BuildID> &BinaryIds) {
149 OS << "Binary IDs: \n";
150 for (auto BI : BinaryIds) {
151 for (uint64_t I = 0; I < BI.size(); I++)
152 OS << format("%02x", BI[I]);
153 OS << "\n";
154 }
155}
156
159 const InstrProfCorrelator *Correlator,
160 std::function<void(Error)> Warn) {
161 // Set up the buffer to read.
162 auto BufferOrError = setupMemoryBuffer(Path, FS);
163 if (Error E = BufferOrError.takeError())
164 return std::move(E);
165 return InstrProfReader::create(std::move(BufferOrError.get()), Correlator,
166 Warn);
167}
168
170InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
171 const InstrProfCorrelator *Correlator,
172 std::function<void(Error)> Warn) {
173 if (Buffer->getBufferSize() == 0)
174 return make_error<InstrProfError>(instrprof_error::empty_raw_profile);
175
176 std::unique_ptr<InstrProfReader> Result;
177 // Create the reader.
179 Result.reset(new IndexedInstrProfReader(std::move(Buffer)));
180 else if (RawInstrProfReader64::hasFormat(*Buffer))
181 Result.reset(new RawInstrProfReader64(std::move(Buffer), Correlator, Warn));
182 else if (RawInstrProfReader32::hasFormat(*Buffer))
183 Result.reset(new RawInstrProfReader32(std::move(Buffer), Correlator, Warn));
184 else if (TextInstrProfReader::hasFormat(*Buffer))
185 Result.reset(new TextInstrProfReader(std::move(Buffer)));
186 else
187 return make_error<InstrProfError>(instrprof_error::unrecognized_format);
188
189 // Initialize the reader and return the result.
190 if (Error E = initializeReader(*Result))
191 return std::move(E);
192
193 return std::move(Result);
194}
195
198 const Twine &RemappingPath) {
199 // Set up the buffer to read.
200 auto BufferOrError = setupMemoryBuffer(Path, FS);
201 if (Error E = BufferOrError.takeError())
202 return std::move(E);
203
204 // Set up the remapping buffer if requested.
205 std::unique_ptr<MemoryBuffer> RemappingBuffer;
206 std::string RemappingPathStr = RemappingPath.str();
207 if (!RemappingPathStr.empty()) {
208 auto RemappingBufferOrError = setupMemoryBuffer(RemappingPathStr, FS);
209 if (Error E = RemappingBufferOrError.takeError())
210 return std::move(E);
211 RemappingBuffer = std::move(RemappingBufferOrError.get());
212 }
213
214 return IndexedInstrProfReader::create(std::move(BufferOrError.get()),
215 std::move(RemappingBuffer));
216}
217
219IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
220 std::unique_ptr<MemoryBuffer> RemappingBuffer) {
221 // Create the reader.
223 return make_error<InstrProfError>(instrprof_error::bad_magic);
224 auto Result = std::make_unique<IndexedInstrProfReader>(
225 std::move(Buffer), std::move(RemappingBuffer));
226
227 // Initialize the reader and return the result.
228 if (Error E = initializeReader(*Result))
229 return std::move(E);
230
231 return std::move(Result);
232}
233
235 // Verify that this really looks like plain ASCII text by checking a
236 // 'reasonable' number of characters (up to profile magic size).
237 size_t count = std::min(Buffer.getBufferSize(), sizeof(uint64_t));
238 StringRef buffer = Buffer.getBufferStart();
239 return count == 0 ||
240 std::all_of(buffer.begin(), buffer.begin() + count,
241 [](char c) { return isPrint(c) || isSpace(c); });
242}
243
244// Read the profile variant flag from the header: ":FE" means this is a FE
245// generated profile. ":IR" means this is an IR level profile. Other strings
246// with a leading ':' will be reported an error format.
248 Symtab.reset(new InstrProfSymtab());
249
250 while (Line->starts_with(":")) {
251 StringRef Str = Line->substr(1);
252 if (Str.equals_insensitive("ir"))
254 else if (Str.equals_insensitive("fe"))
256 else if (Str.equals_insensitive("csir")) {
258 ProfileKind |= InstrProfKind::ContextSensitive;
259 } else if (Str.equals_insensitive("entry_first"))
261 else if (Str.equals_insensitive("not_entry_first"))
263 else if (Str.equals_insensitive("single_byte_coverage"))
265 else if (Str.equals_insensitive("temporal_prof_traces")) {
266 ProfileKind |= InstrProfKind::TemporalProfile;
267 if (auto Err = readTemporalProfTraceData())
268 return error(std::move(Err));
269 } else
271 ++Line;
272 }
273 return success();
274}
275
276/// Temporal profile trace data is stored in the header immediately after
277/// ":temporal_prof_traces". The first integer is the number of traces, the
278/// second integer is the stream size, then the following lines are the actual
279/// traces which consist of a weight and a comma separated list of function
280/// names.
281Error TextInstrProfReader::readTemporalProfTraceData() {
282 if ((++Line).is_at_end())
284
285 uint32_t NumTraces;
286 if (Line->getAsInteger(0, NumTraces))
288
289 if ((++Line).is_at_end())
291
294
295 for (uint32_t i = 0; i < NumTraces; i++) {
296 if ((++Line).is_at_end())
298
300 if (Line->getAsInteger(0, Trace.Weight))
302
303 if ((++Line).is_at_end())
305
306 SmallVector<StringRef> FuncNames;
307 Line->split(FuncNames, ",", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
308 for (auto &FuncName : FuncNames)
309 Trace.FunctionNameRefs.push_back(
310 IndexedInstrProf::ComputeHash(FuncName.trim()));
311 TemporalProfTraces.push_back(std::move(Trace));
312 }
313 return success();
314}
315
316Error
317TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
318
319#define CHECK_LINE_END(Line) \
320 if (Line.is_at_end()) \
321 return error(instrprof_error::truncated);
322#define READ_NUM(Str, Dst) \
323 if ((Str).getAsInteger(10, (Dst))) \
324 return error(instrprof_error::malformed);
325#define VP_READ_ADVANCE(Val) \
326 CHECK_LINE_END(Line); \
327 uint32_t Val; \
328 READ_NUM((*Line), (Val)); \
329 Line++;
330
331 if (Line.is_at_end())
332 return success();
333
334 uint32_t NumValueKinds;
335 if (Line->getAsInteger(10, NumValueKinds)) {
336 // No value profile data
337 return success();
338 }
339 if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
341 "number of value kinds is invalid");
342 Line++;
343
344 for (uint32_t VK = 0; VK < NumValueKinds; VK++) {
345 VP_READ_ADVANCE(ValueKind);
346 if (ValueKind > IPVK_Last)
347 return error(instrprof_error::malformed, "value kind is invalid");
348 ;
349 VP_READ_ADVANCE(NumValueSites);
350 if (!NumValueSites)
351 continue;
352
353 Record.reserveSites(VK, NumValueSites);
354 for (uint32_t S = 0; S < NumValueSites; S++) {
355 VP_READ_ADVANCE(NumValueData);
356
357 std::vector<InstrProfValueData> CurrentValues;
358 for (uint32_t V = 0; V < NumValueData; V++) {
359 CHECK_LINE_END(Line);
360 std::pair<StringRef, StringRef> VD = Line->rsplit(':');
361 uint64_t TakenCount, Value;
362 if (ValueKind == IPVK_IndirectCallTarget) {
363 if (InstrProfSymtab::isExternalSymbol(VD.first)) {
364 Value = 0;
365 } else {
366 if (Error E = Symtab->addFuncName(VD.first))
367 return E;
369 }
370 } else if (ValueKind == IPVK_VTableTarget) {
372 Value = 0;
373 else {
374 if (Error E = Symtab->addVTableName(VD.first))
375 return E;
377 }
378 } else {
379 READ_NUM(VD.first, Value);
380 }
381 READ_NUM(VD.second, TakenCount);
382 CurrentValues.push_back({Value, TakenCount});
383 Line++;
384 }
385 Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData,
386 nullptr);
387 }
388 }
389 return success();
390
391#undef CHECK_LINE_END
392#undef READ_NUM
393#undef VP_READ_ADVANCE
394}
395
397 // Skip empty lines and comments.
398 while (!Line.is_at_end() && (Line->empty() || Line->starts_with("#")))
399 ++Line;
400 // If we hit EOF while looking for a name, we're done.
401 if (Line.is_at_end()) {
403 }
404
405 // Read the function name.
406 Record.Name = *Line++;
407 if (Error E = Symtab->addFuncName(Record.Name))
408 return error(std::move(E));
409
410 // Read the function hash.
411 if (Line.is_at_end())
413 if ((Line++)->getAsInteger(0, Record.Hash))
415 "function hash is not a valid integer");
416
417 // Read the number of counters.
418 uint64_t NumCounters;
419 if (Line.is_at_end())
421 if ((Line++)->getAsInteger(10, NumCounters))
423 "number of counters is not a valid integer");
424 if (NumCounters == 0)
425 return error(instrprof_error::malformed, "number of counters is zero");
426
427 // Read each counter and fill our internal storage with the values.
428 Record.Clear();
429 Record.Counts.reserve(NumCounters);
430 for (uint64_t I = 0; I < NumCounters; ++I) {
431 if (Line.is_at_end())
433 uint64_t Count;
434 if ((Line++)->getAsInteger(10, Count))
435 return error(instrprof_error::malformed, "count is invalid");
436 Record.Counts.push_back(Count);
437 }
438
439 // Bitmap byte information is indicated with special character.
440 if (Line->starts_with("$")) {
441 Record.BitmapBytes.clear();
442 // Read the number of bitmap bytes.
443 uint64_t NumBitmapBytes;
444 if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes))
446 "number of bitmap bytes is not a valid integer");
447 if (NumBitmapBytes != 0) {
448 // Read each bitmap and fill our internal storage with the values.
449 Record.BitmapBytes.reserve(NumBitmapBytes);
450 for (uint8_t I = 0; I < NumBitmapBytes; ++I) {
451 if (Line.is_at_end())
453 uint8_t BitmapByte;
454 if ((Line++)->getAsInteger(0, BitmapByte))
456 "bitmap byte is not a valid integer");
457 Record.BitmapBytes.push_back(BitmapByte);
458 }
459 }
460 }
461
462 // Check if value profile data exists and read it if so.
463 if (Error E = readValueProfileData(Record))
464 return error(std::move(E));
465
466 return success();
467}
468
469template <class IntPtrT>
471 return getProfileKindFromVersion(Version);
472}
473
474template <class IntPtrT>
477 std::optional<uint64_t> Weight) {
478 if (TemporalProfTimestamps.empty()) {
479 assert(TemporalProfTraces.empty());
480 return TemporalProfTraces;
481 }
482 // Sort functions by their timestamps to build the trace.
483 std::sort(TemporalProfTimestamps.begin(), TemporalProfTimestamps.end());
485 if (Weight)
486 Trace.Weight = *Weight;
487 for (auto &[TimestampValue, NameRef] : TemporalProfTimestamps)
488 Trace.FunctionNameRefs.push_back(NameRef);
489 TemporalProfTraces = {std::move(Trace)};
490 return TemporalProfTraces;
491}
492
493template <class IntPtrT>
495 if (DataBuffer.getBufferSize() < sizeof(uint64_t))
496 return false;
497 uint64_t Magic =
498 *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
499 return RawInstrProf::getMagic<IntPtrT>() == Magic ||
500 llvm::byteswap(RawInstrProf::getMagic<IntPtrT>()) == Magic;
501}
502
503template <class IntPtrT>
505 if (!hasFormat(*DataBuffer))
507 if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header))
509 auto *Header = reinterpret_cast<const RawInstrProf::Header *>(
510 DataBuffer->getBufferStart());
511 ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
512 return readHeader(*Header);
513}
514
515template <class IntPtrT>
517 const char *End = DataBuffer->getBufferEnd();
518 // Skip zero padding between profiles.
519 while (CurrentPos != End && *CurrentPos == 0)
520 ++CurrentPos;
521 // If there's nothing left, we're done.
522 if (CurrentPos == End)
523 return make_error<InstrProfError>(instrprof_error::eof);
524 // If there isn't enough space for another header, this is probably just
525 // garbage at the end of the file.
526 if (CurrentPos + sizeof(RawInstrProf::Header) > End)
527 return make_error<InstrProfError>(instrprof_error::malformed,
528 "not enough space for another header");
529 // The writer ensures each profile is padded to start at an aligned address.
530 if (reinterpret_cast<size_t>(CurrentPos) % alignof(uint64_t))
531 return make_error<InstrProfError>(instrprof_error::malformed,
532 "insufficient padding");
533 // The magic should have the same byte order as in the previous header.
534 uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos);
535 if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
536 return make_error<InstrProfError>(instrprof_error::bad_magic);
537
538 // There's another profile to read, so we need to process the header.
539 auto *Header = reinterpret_cast<const RawInstrProf::Header *>(CurrentPos);
540 return readHeader(*Header);
541}
542
543template <class IntPtrT>
545 if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart),
546 StringRef(VNamesStart, VNamesEnd - VNamesStart)))
547 return error(std::move(E));
548 for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
549 const IntPtrT FPtr = swap(I->FunctionPointer);
550 if (!FPtr)
551 continue;
552 Symtab.mapAddress(FPtr, swap(I->NameRef));
553 }
554
555 if (VTableBegin != nullptr && VTableEnd != nullptr) {
556 for (const RawInstrProf::VTableProfileData<IntPtrT> *I = VTableBegin;
557 I != VTableEnd; ++I) {
558 const IntPtrT VPtr = swap(I->VTablePointer);
559 if (!VPtr)
560 continue;
561 // Map both begin and end address to the name hash, since the instrumented
562 // address could be somewhere in the middle.
563 // VPtr is of type uint32_t or uint64_t so 'VPtr + I->VTableSize' marks
564 // the end of vtable address.
565 Symtab.mapVTableAddress(VPtr, VPtr + swap(I->VTableSize),
566 swap(I->VTableNameHash));
567 }
568 }
569 return success();
570}
571
572template <class IntPtrT>
574 const RawInstrProf::Header &Header) {
575 Version = swap(Header.Version);
576 if (GET_VERSION(Version) != RawInstrProf::Version)
578 ("Profile uses raw profile format version = " +
579 Twine(GET_VERSION(Version)) +
580 "; expected version = " + Twine(RawInstrProf::Version) +
581 "\nPLEASE update this tool to version in the raw profile, or "
582 "regenerate raw profile with expected version.")
583 .str());
584
585 uint64_t BinaryIdSize = swap(Header.BinaryIdsSize);
586 // Binary id start just after the header if exists.
587 const uint8_t *BinaryIdStart =
588 reinterpret_cast<const uint8_t *>(&Header) + sizeof(RawInstrProf::Header);
589 const uint8_t *BinaryIdEnd = BinaryIdStart + BinaryIdSize;
590 const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd();
591 if (BinaryIdSize % sizeof(uint64_t) || BinaryIdEnd > BufferEnd)
593 if (BinaryIdSize != 0) {
594 if (Error Err =
595 readBinaryIdsInternal(*DataBuffer, BinaryIdSize, BinaryIdStart,
596 BinaryIds, getDataEndianness()))
597 return Err;
598 }
599
600 CountersDelta = swap(Header.CountersDelta);
601 BitmapDelta = swap(Header.BitmapDelta);
602 NamesDelta = swap(Header.NamesDelta);
603 auto NumData = swap(Header.NumData);
604 auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters);
605 auto CountersSize = swap(Header.NumCounters) * getCounterTypeSize();
606 auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters);
607 auto NumBitmapBytes = swap(Header.NumBitmapBytes);
608 auto PaddingBytesAfterBitmapBytes = swap(Header.PaddingBytesAfterBitmapBytes);
609 auto NamesSize = swap(Header.NamesSize);
610 auto VTableNameSize = swap(Header.VNamesSize);
611 auto NumVTables = swap(Header.NumVTables);
612 ValueKindLast = swap(Header.ValueKindLast);
613
614 auto DataSize = NumData * sizeof(RawInstrProf::ProfileData<IntPtrT>);
615 auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize);
616 auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize);
617
618 auto VTableSectionSize =
619 NumVTables * sizeof(RawInstrProf::VTableProfileData<IntPtrT>);
620 auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize);
621
622 // Profile data starts after profile header and binary ids if exist.
623 ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdSize;
624 ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
625 ptrdiff_t BitmapOffset =
626 CountersOffset + CountersSize + PaddingBytesAfterCounters;
627 ptrdiff_t NamesOffset =
628 BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes;
629 ptrdiff_t VTableProfDataOffset =
630 NamesOffset + NamesSize + PaddingBytesAfterNames;
631 ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +
632 PaddingBytesAfterVTableProfData;
633 ptrdiff_t ValueDataOffset =
634 VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;
635
636 auto *Start = reinterpret_cast<const char *>(&Header);
637 if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
639
640 if (Correlator) {
641 // These sizes in the raw file are zero because we constructed them in the
642 // Correlator.
643 if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&
644 NamesDelta == 0))
646 Data = Correlator->getDataPointer();
647 DataEnd = Data + Correlator->getDataSize();
648 NamesStart = Correlator->getNamesPointer();
649 NamesEnd = NamesStart + Correlator->getNamesSize();
650 } else {
651 Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
652 Start + DataOffset);
653 DataEnd = Data + NumData;
654 VTableBegin =
655 reinterpret_cast<const RawInstrProf::VTableProfileData<IntPtrT> *>(
656 Start + VTableProfDataOffset);
657 VTableEnd = VTableBegin + NumVTables;
658 NamesStart = Start + NamesOffset;
659 NamesEnd = NamesStart + NamesSize;
660 VNamesStart = Start + VTableNameOffset;
661 VNamesEnd = VNamesStart + VTableNameSize;
662 }
663
664 CountersStart = Start + CountersOffset;
665 CountersEnd = CountersStart + CountersSize;
666 BitmapStart = Start + BitmapOffset;
667 BitmapEnd = BitmapStart + NumBitmapBytes;
668 ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
669
670 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
671 if (Error E = createSymtab(*NewSymtab))
672 return E;
673
674 Symtab = std::move(NewSymtab);
675 return success();
676}
677
678template <class IntPtrT>
680 Record.Name = getName(Data->NameRef);
681 return success();
682}
683
684template <class IntPtrT>
686 Record.Hash = swap(Data->FuncHash);
687 return success();
688}
689
690template <class IntPtrT>
693 uint32_t NumCounters = swap(Data->NumCounters);
694 if (NumCounters == 0)
695 return error(instrprof_error::malformed, "number of counters is zero");
696
697 ptrdiff_t CounterBaseOffset = swap(Data->CounterPtr) - CountersDelta;
698 if (CounterBaseOffset < 0)
699 return error(
701 ("counter offset " + Twine(CounterBaseOffset) + " is negative").str());
702
703 if (CounterBaseOffset >= CountersEnd - CountersStart)
705 ("counter offset " + Twine(CounterBaseOffset) +
706 " is greater than the maximum counter offset " +
707 Twine(CountersEnd - CountersStart - 1))
708 .str());
709
710 uint64_t MaxNumCounters =
711 (CountersEnd - (CountersStart + CounterBaseOffset)) /
712 getCounterTypeSize();
713 if (NumCounters > MaxNumCounters)
715 ("number of counters " + Twine(NumCounters) +
716 " is greater than the maximum number of counters " +
717 Twine(MaxNumCounters))
718 .str());
719
720 Record.Counts.clear();
721 Record.Counts.reserve(NumCounters);
722 for (uint32_t I = 0; I < NumCounters; I++) {
723 const char *Ptr =
724 CountersStart + CounterBaseOffset + I * getCounterTypeSize();
725 if (I == 0 && hasTemporalProfile()) {
726 uint64_t TimestampValue = swap(*reinterpret_cast<const uint64_t *>(Ptr));
727 if (TimestampValue != 0 &&
728 TimestampValue != std::numeric_limits<uint64_t>::max()) {
729 TemporalProfTimestamps.emplace_back(TimestampValue,
730 swap(Data->NameRef));
731 TemporalProfTraceStreamSize = 1;
732 }
733 if (hasSingleByteCoverage()) {
734 // In coverage mode, getCounterTypeSize() returns 1 byte but our
735 // timestamp field has size uint64_t. Increment I so that the next
736 // iteration of this for loop points to the byte after the timestamp
737 // field, i.e., I += 8.
738 I += 7;
739 }
740 continue;
741 }
742 if (hasSingleByteCoverage()) {
743 // A value of zero signifies the block is covered.
744 Record.Counts.push_back(*Ptr == 0 ? 1 : 0);
745 } else {
746 uint64_t CounterValue = swap(*reinterpret_cast<const uint64_t *>(Ptr));
747 if (CounterValue > MaxCounterValue && Warn)
748 Warn(make_error<InstrProfError>(
750
751 Record.Counts.push_back(CounterValue);
752 }
753 }
754
755 return success();
756}
757
758template <class IntPtrT>
760 uint32_t NumBitmapBytes = swap(Data->NumBitmapBytes);
761
762 Record.BitmapBytes.clear();
763 Record.BitmapBytes.reserve(NumBitmapBytes);
764
765 // It's possible MCDC is either not enabled or only used for some functions
766 // and not others. So if we record 0 bytes, just move on.
767 if (NumBitmapBytes == 0)
768 return success();
769
770 // BitmapDelta decreases as we advance to the next data record.
771 ptrdiff_t BitmapOffset = swap(Data->BitmapPtr) - BitmapDelta;
772 if (BitmapOffset < 0)
773 return error(
775 ("bitmap offset " + Twine(BitmapOffset) + " is negative").str());
776
777 if (BitmapOffset >= BitmapEnd - BitmapStart)
779 ("bitmap offset " + Twine(BitmapOffset) +
780 " is greater than the maximum bitmap offset " +
781 Twine(BitmapEnd - BitmapStart - 1))
782 .str());
783
784 uint64_t MaxNumBitmapBytes =
785 (BitmapEnd - (BitmapStart + BitmapOffset)) / sizeof(uint8_t);
786 if (NumBitmapBytes > MaxNumBitmapBytes)
788 ("number of bitmap bytes " + Twine(NumBitmapBytes) +
789 " is greater than the maximum number of bitmap bytes " +
790 Twine(MaxNumBitmapBytes))
791 .str());
792
793 for (uint32_t I = 0; I < NumBitmapBytes; I++) {
794 const char *Ptr = BitmapStart + BitmapOffset + I;
795 Record.BitmapBytes.push_back(swap(*Ptr));
796 }
797
798 return success();
799}
800
801template <class IntPtrT>
804 Record.clearValueData();
805 CurValueDataSize = 0;
806 // Need to match the logic in value profile dumper code in compiler-rt:
807 uint32_t NumValueKinds = 0;
808 for (uint32_t I = 0; I < IPVK_Last + 1; I++)
809 NumValueKinds += (Data->NumValueSites[I] != 0);
810
811 if (!NumValueKinds)
812 return success();
813
815 ValueProfData::getValueProfData(
816 ValueDataStart, (const unsigned char *)DataBuffer->getBufferEnd(),
817 getDataEndianness());
818
819 if (Error E = VDataPtrOrErr.takeError())
820 return E;
821
822 // Note that besides deserialization, this also performs the conversion for
823 // indirect call targets. The function pointers from the raw profile are
824 // remapped into function name hashes.
825 VDataPtrOrErr.get()->deserializeTo(Record, Symtab.get());
826 CurValueDataSize = VDataPtrOrErr.get()->getSize();
827 return success();
828}
829
830template <class IntPtrT>
832 // Keep reading profiles that consist of only headers and no profile data and
833 // counters.
834 while (atEnd())
835 // At this point, ValueDataStart field points to the next header.
836 if (Error E = readNextHeader(getNextHeaderPos()))
837 return error(std::move(E));
838
839 // Read name and set it in Record.
840 if (Error E = readName(Record))
841 return error(std::move(E));
842
843 // Read FuncHash and set it in Record.
844 if (Error E = readFuncHash(Record))
845 return error(std::move(E));
846
847 // Read raw counts and set Record.
848 if (Error E = readRawCounts(Record))
849 return error(std::move(E));
850
851 // Read raw bitmap bytes and set Record.
852 if (Error E = readRawBitmapBytes(Record))
853 return error(std::move(E));
854
855 // Read value data and set Record.
856 if (Error E = readValueProfilingData(Record))
857 return error(std::move(E));
858
859 // Iterate.
860 advanceData();
861 return success();
862}
863
864template <class IntPtrT>
866 std::vector<llvm::object::BuildID> &BinaryIds) {
867 BinaryIds.insert(BinaryIds.begin(), this->BinaryIds.begin(),
868 this->BinaryIds.end());
869 return Error::success();
870}
871
872template <class IntPtrT>
874 if (!BinaryIds.empty())
875 printBinaryIdsInternal(OS, BinaryIds);
876 return Error::success();
877}
878
879namespace llvm {
880
881template class RawInstrProfReader<uint32_t>;
882template class RawInstrProfReader<uint64_t>;
883
884} // end namespace llvm
885
888 return IndexedInstrProf::ComputeHash(HashType, K);
889}
890
893
895 const unsigned char *&D, const unsigned char *const End) {
897 ValueProfData::getValueProfData(D, End, ValueProfDataEndianness);
898
899 if (VDataPtrOrErr.takeError())
900 return false;
901
902 VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), nullptr);
903 D += VDataPtrOrErr.get()->TotalSize;
904
905 return true;
906}
907
909 offset_type N) {
910 using namespace support;
911
912 // Check if the data is corrupt. If so, don't try to read it.
913 if (N % sizeof(uint64_t))
914 return data_type();
915
916 DataBuffer.clear();
917 std::vector<uint64_t> CounterBuffer;
918 std::vector<uint8_t> BitmapByteBuffer;
919
920 const unsigned char *End = D + N;
921 while (D < End) {
922 // Read hash.
923 if (D + sizeof(uint64_t) >= End)
924 return data_type();
925 uint64_t Hash = endian::readNext<uint64_t, llvm::endianness::little>(D);
926
927 // Initialize number of counters for GET_VERSION(FormatVersion) == 1.
928 uint64_t CountsSize = N / sizeof(uint64_t) - 1;
929 // If format version is different then read the number of counters.
930 if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) {
931 if (D + sizeof(uint64_t) > End)
932 return data_type();
933 CountsSize = endian::readNext<uint64_t, llvm::endianness::little>(D);
934 }
935 // Read counter values.
936 if (D + CountsSize * sizeof(uint64_t) > End)
937 return data_type();
938
939 CounterBuffer.clear();
940 CounterBuffer.reserve(CountsSize);
941 for (uint64_t J = 0; J < CountsSize; ++J)
942 CounterBuffer.push_back(
943 endian::readNext<uint64_t, llvm::endianness::little>(D));
944
945 // Read bitmap bytes for GET_VERSION(FormatVersion) > 10.
946 if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version10) {
947 uint64_t BitmapBytes = 0;
948 if (D + sizeof(uint64_t) > End)
949 return data_type();
950 BitmapBytes = endian::readNext<uint64_t, llvm::endianness::little>(D);
951 // Read bitmap byte values.
952 if (D + BitmapBytes * sizeof(uint8_t) > End)
953 return data_type();
954 BitmapByteBuffer.clear();
955 BitmapByteBuffer.reserve(BitmapBytes);
956 for (uint64_t J = 0; J < BitmapBytes; ++J)
957 BitmapByteBuffer.push_back(static_cast<uint8_t>(
958 endian::readNext<uint64_t, llvm::endianness::little>(D)));
959 }
960
961 DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer),
962 std::move(BitmapByteBuffer));
963
964 // Read value profiling data.
965 if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 &&
967 DataBuffer.clear();
968 return data_type();
969 }
970 }
971 return DataBuffer;
972}
973
974template <typename HashTableImpl>
977 auto Iter = HashTable->find(FuncName);
978 if (Iter == HashTable->end())
979 return make_error<InstrProfError>(instrprof_error::unknown_function);
980
981 Data = (*Iter);
982 if (Data.empty())
983 return make_error<InstrProfError>(instrprof_error::malformed,
984 "profile data is empty");
985
986 return Error::success();
987}
988
989template <typename HashTableImpl>
992 if (atEnd())
993 return make_error<InstrProfError>(instrprof_error::eof);
994
995 Data = *RecordIterator;
996
997 if (Data.empty())
998 return make_error<InstrProfError>(instrprof_error::malformed,
999 "profile data is empty");
1000
1001 return Error::success();
1002}
1003
1004template <typename HashTableImpl>
1006 const unsigned char *Buckets, const unsigned char *const Payload,
1007 const unsigned char *const Base, IndexedInstrProf::HashT HashType,
1008 uint64_t Version) {
1009 FormatVersion = Version;
1010 HashTable.reset(HashTableImpl::Create(
1011 Buckets, Payload, Base,
1012 typename HashTableImpl::InfoType(HashType, Version)));
1013 RecordIterator = HashTable->data_begin();
1014}
1015
1016template <typename HashTableImpl>
1018 return getProfileKindFromVersion(FormatVersion);
1019}
1020
1021namespace {
1022/// A remapper that does not apply any remappings.
1023class InstrProfReaderNullRemapper : public InstrProfReaderRemapper {
1024 InstrProfReaderIndexBase &Underlying;
1025
1026public:
1027 InstrProfReaderNullRemapper(InstrProfReaderIndexBase &Underlying)
1028 : Underlying(Underlying) {}
1029
1030 Error getRecords(StringRef FuncName,
1031 ArrayRef<NamedInstrProfRecord> &Data) override {
1032 return Underlying.getRecords(FuncName, Data);
1033 }
1034};
1035} // namespace
1036
1037/// A remapper that applies remappings based on a symbol remapping file.
1038template <typename HashTableImpl>
1040 : public InstrProfReaderRemapper {
1041public:
1043 std::unique_ptr<MemoryBuffer> RemapBuffer,
1045 : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {
1046 }
1047
1048 /// Extract the original function name from a PGO function name.
1050 // We can have multiple pieces separated by kGlobalIdentifierDelimiter (
1051 // semicolon now and colon in older profiles); there can be pieces both
1052 // before and after the mangled name. Find the first part that starts with
1053 // '_Z'; we'll assume that's the mangled name we want.
1054 std::pair<StringRef, StringRef> Parts = {StringRef(), Name};
1055 while (true) {
1056 Parts = Parts.second.split(kGlobalIdentifierDelimiter);
1057 if (Parts.first.starts_with("_Z"))
1058 return Parts.first;
1059 if (Parts.second.empty())
1060 return Name;
1061 }
1062 }
1063
1064 /// Given a mangled name extracted from a PGO function name, and a new
1065 /// form for that mangled name, reconstitute the name.
1066 static void reconstituteName(StringRef OrigName, StringRef ExtractedName,
1067 StringRef Replacement,
1068 SmallVectorImpl<char> &Out) {
1069 Out.reserve(OrigName.size() + Replacement.size() - ExtractedName.size());
1070 Out.insert(Out.end(), OrigName.begin(), ExtractedName.begin());
1071 Out.insert(Out.end(), Replacement.begin(), Replacement.end());
1072 Out.insert(Out.end(), ExtractedName.end(), OrigName.end());
1073 }
1074
1076 if (Error E = Remappings.read(*RemapBuffer))
1077 return E;
1078 for (StringRef Name : Underlying.HashTable->keys()) {
1079 StringRef RealName = extractName(Name);
1080 if (auto Key = Remappings.insert(RealName)) {
1081 // FIXME: We could theoretically map the same equivalence class to
1082 // multiple names in the profile data. If that happens, we should
1083 // return NamedInstrProfRecords from all of them.
1084 MappedNames.insert({Key, RealName});
1085 }
1086 }
1087 return Error::success();
1088 }
1089
1092 StringRef RealName = extractName(FuncName);
1093 if (auto Key = Remappings.lookup(RealName)) {
1094 StringRef Remapped = MappedNames.lookup(Key);
1095 if (!Remapped.empty()) {
1096 if (RealName.begin() == FuncName.begin() &&
1097 RealName.end() == FuncName.end())
1098 FuncName = Remapped;
1099 else {
1100 // Try rebuilding the name from the given remapping.
1101 SmallString<256> Reconstituted;
1102 reconstituteName(FuncName, RealName, Remapped, Reconstituted);
1103 Error E = Underlying.getRecords(Reconstituted, Data);
1104 if (!E)
1105 return E;
1106
1107 // If we failed because the name doesn't exist, fall back to asking
1108 // about the original name.
1109 if (Error Unhandled = handleErrors(
1110 std::move(E), [](std::unique_ptr<InstrProfError> Err) {
1111 return Err->get() == instrprof_error::unknown_function
1112 ? Error::success()
1113 : Error(std::move(Err));
1114 }))
1115 return Unhandled;
1116 }
1117 }
1118 }
1119 return Underlying.getRecords(FuncName, Data);
1120 }
1121
1122private:
1123 /// The memory buffer containing the remapping configuration. Remappings
1124 /// holds pointers into this buffer.
1125 std::unique_ptr<MemoryBuffer> RemapBuffer;
1126
1127 /// The mangling remapper.
1128 SymbolRemappingReader Remappings;
1129
1130 /// Mapping from mangled name keys to the name used for the key in the
1131 /// profile data.
1132 /// FIXME: Can we store a location within the on-disk hash table instead of
1133 /// redoing lookup?
1135
1136 /// The real profile data reader.
1138};
1139
1141 using namespace support;
1142
1143 if (DataBuffer.getBufferSize() < 8)
1144 return false;
1145 uint64_t Magic = endian::read<uint64_t, llvm::endianness::little, aligned>(
1146 DataBuffer.getBufferStart());
1147 // Verify that it's magical.
1148 return Magic == IndexedInstrProf::Magic;
1149}
1150
1151const unsigned char *
1152IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
1153 const unsigned char *Cur, bool UseCS) {
1154 using namespace IndexedInstrProf;
1155 using namespace support;
1156
1157 if (Version >= IndexedInstrProf::Version4) {
1158 const IndexedInstrProf::Summary *SummaryInLE =
1159 reinterpret_cast<const IndexedInstrProf::Summary *>(Cur);
1160 uint64_t NFields = endian::byte_swap<uint64_t, llvm::endianness::little>(
1161 SummaryInLE->NumSummaryFields);
1162 uint64_t NEntries = endian::byte_swap<uint64_t, llvm::endianness::little>(
1163 SummaryInLE->NumCutoffEntries);
1164 uint32_t SummarySize =
1165 IndexedInstrProf::Summary::getSize(NFields, NEntries);
1166 std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
1167 IndexedInstrProf::allocSummary(SummarySize);
1168
1169 const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE);
1170 uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get());
1171 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
1172 Dst[I] = endian::byte_swap<uint64_t, llvm::endianness::little>(Src[I]);
1173
1174 SummaryEntryVector DetailedSummary;
1175 for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) {
1176 const IndexedInstrProf::Summary::Entry &Ent = SummaryData->getEntry(I);
1177 DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount,
1178 Ent.NumBlocks);
1179 }
1180 std::unique_ptr<llvm::ProfileSummary> &Summary =
1181 UseCS ? this->CS_Summary : this->Summary;
1182
1183 // initialize InstrProfSummary using the SummaryData from disk.
1184 Summary = std::make_unique<ProfileSummary>(
1186 DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
1187 SummaryData->get(Summary::MaxBlockCount),
1188 SummaryData->get(Summary::MaxInternalBlockCount),
1189 SummaryData->get(Summary::MaxFunctionCount),
1190 SummaryData->get(Summary::TotalNumBlocks),
1191 SummaryData->get(Summary::TotalNumFunctions));
1192 return Cur + SummarySize;
1193 } else {
1194 // The older versions do not support a profile summary. This just computes
1195 // an empty summary, which will not result in accurate hot/cold detection.
1196 // We would need to call addRecord for all NamedInstrProfRecords to get the
1197 // correct summary. However, this version is old (prior to early 2016) and
1198 // has not been supporting an accurate summary for several years.
1200 Summary = Builder.getSummary();
1201 return Cur;
1202 }
1203}
1204
1205Error IndexedMemProfReader::deserialize(const unsigned char *Start,
1206 uint64_t MemProfOffset) {
1207 const unsigned char *Ptr = Start + MemProfOffset;
1208
1209 // Read the first 64-bit word, which may be RecordTableOffset in
1210 // memprof::MemProfVersion0 or the MemProf version number in
1211 // memprof::MemProfVersion1 and above.
1212 const uint64_t FirstWord =
1213 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1214
1216 if (FirstWord == memprof::Version1 || FirstWord == memprof::Version2) {
1217 // Everything is good. We can proceed to deserialize the rest.
1218 Version = static_cast<memprof::IndexedVersion>(FirstWord);
1219 } else if (FirstWord >= 24) {
1220 // This is a heuristic/hack to detect memprof::MemProfVersion0,
1221 // which does not have a version field in the header.
1222 // In memprof::MemProfVersion0, FirstWord will be RecordTableOffset,
1223 // which should be at least 24 because of the MemProf header size.
1224 Version = memprof::Version0;
1225 } else {
1226 return make_error<InstrProfError>(
1228 formatv("MemProf version {} not supported; "
1229 "requires version between {} and {}, inclusive",
1232 }
1233
1234 // The value returned from RecordTableGenerator.Emit.
1235 const uint64_t RecordTableOffset =
1236 Version == memprof::Version0
1237 ? FirstWord
1238 : support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1239 // The offset in the stream right before invoking
1240 // FrameTableGenerator.Emit.
1241 const uint64_t FramePayloadOffset =
1242 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1243 // The value returned from FrameTableGenerator.Emit.
1244 const uint64_t FrameTableOffset =
1245 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1246
1247 // The offset in the stream right before invoking
1248 // CallStackTableGenerator.Emit.
1249 uint64_t CallStackPayloadOffset = 0;
1250 // The value returned from CallStackTableGenerator.Emit.
1251 uint64_t CallStackTableOffset = 0;
1252 if (Version >= memprof::Version2) {
1253 CallStackPayloadOffset =
1254 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1255 CallStackTableOffset =
1256 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1257 }
1258
1259 // Read the schema.
1260 auto SchemaOr = memprof::readMemProfSchema(Ptr);
1261 if (!SchemaOr)
1262 return SchemaOr.takeError();
1263 Schema = SchemaOr.get();
1264
1265 // Now initialize the table reader with a pointer into data buffer.
1266 MemProfRecordTable.reset(MemProfRecordHashTable::Create(
1267 /*Buckets=*/Start + RecordTableOffset,
1268 /*Payload=*/Ptr,
1269 /*Base=*/Start, memprof::RecordLookupTrait(Version, Schema)));
1270
1271 // Initialize the frame table reader with the payload and bucket offsets.
1272 MemProfFrameTable.reset(MemProfFrameHashTable::Create(
1273 /*Buckets=*/Start + FrameTableOffset,
1274 /*Payload=*/Start + FramePayloadOffset,
1275 /*Base=*/Start));
1276
1277 if (Version >= memprof::Version2)
1278 MemProfCallStackTable.reset(MemProfCallStackHashTable::Create(
1279 /*Buckets=*/Start + CallStackTableOffset,
1280 /*Payload=*/Start + CallStackPayloadOffset,
1281 /*Base=*/Start));
1282
1283#ifdef EXPENSIVE_CHECKS
1284 // Go through all the records and verify that CSId has been correctly
1285 // populated. Do this only under EXPENSIVE_CHECKS. Otherwise, we
1286 // would defeat the purpose of OnDiskIterableChainedHashTable.
1287 // Note that we can compare CSId against actual call stacks only for
1288 // Version0 and Version1 because IndexedAllocationInfo::CallStack and
1289 // IndexedMemProfRecord::CallSites are not populated in Version2.
1290 if (Version <= memprof::Version1)
1291 for (const auto &Record : MemProfRecordTable->data())
1292 verifyIndexedMemProfRecord(Record);
1293#endif
1294
1295 return Error::success();
1296}
1297
1299 using namespace support;
1300
1301 const unsigned char *Start =
1302 (const unsigned char *)DataBuffer->getBufferStart();
1303 const unsigned char *Cur = Start;
1304 if ((const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
1306
1307 auto HeaderOr = IndexedInstrProf::Header::readFromBuffer(Start);
1308 if (!HeaderOr)
1309 return HeaderOr.takeError();
1310
1311 const IndexedInstrProf::Header *Header = &HeaderOr.get();
1312 Cur += Header->size();
1313
1314 Cur = readSummary((IndexedInstrProf::ProfVersion)Header->formatVersion(), Cur,
1315 /* UseCS */ false);
1316 if (Header->formatVersion() & VARIANT_MASK_CSIR_PROF)
1317 Cur =
1318 readSummary((IndexedInstrProf::ProfVersion)Header->formatVersion(), Cur,
1319 /* UseCS */ true);
1320 // Read the hash type and start offset.
1321 IndexedInstrProf::HashT HashType = static_cast<IndexedInstrProf::HashT>(
1322 endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashType));
1323 if (HashType > IndexedInstrProf::HashT::Last)
1325
1326 uint64_t HashOffset =
1327 endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashOffset);
1328
1329 // The hash table with profile counts comes next.
1330 auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
1331 Start + HashOffset, Cur, Start, HashType, Header->formatVersion());
1332
1333 // The MemProfOffset field in the header is only valid when the format
1334 // version is higher than 8 (when it was introduced).
1335 if (GET_VERSION(Header->formatVersion()) >= 8 &&
1336 Header->formatVersion() & VARIANT_MASK_MEMPROF) {
1337 uint64_t MemProfOffset =
1338 endian::byte_swap<uint64_t, llvm::endianness::little>(
1339 Header->MemProfOffset);
1340 if (Error E = MemProfReader.deserialize(Start, MemProfOffset))
1341 return E;
1342 }
1343
1344 // BinaryIdOffset field in the header is only valid when the format version
1345 // is higher than 9 (when it was introduced).
1346 if (GET_VERSION(Header->formatVersion()) >= 9) {
1347 uint64_t BinaryIdOffset =
1348 endian::byte_swap<uint64_t, llvm::endianness::little>(
1349 Header->BinaryIdOffset);
1350 const unsigned char *Ptr = Start + BinaryIdOffset;
1351 // Read binary ids size.
1352 BinaryIdsSize =
1353 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1354 if (BinaryIdsSize % sizeof(uint64_t))
1356 // Set the binary ids start.
1357 BinaryIdsStart = Ptr;
1358 if (BinaryIdsStart > (const unsigned char *)DataBuffer->getBufferEnd())
1359 return make_error<InstrProfError>(instrprof_error::malformed,
1360 "corrupted binary ids");
1361 }
1362
1363 if (GET_VERSION(Header->formatVersion()) >= 12) {
1364 uint64_t VTableNamesOffset =
1365 endian::byte_swap<uint64_t, llvm::endianness::little>(
1366 Header->VTableNamesOffset);
1367 const unsigned char *Ptr = Start + VTableNamesOffset;
1368
1369 CompressedVTableNamesLen =
1370 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1371
1372 // Writer first writes the length of compressed string, and then the actual
1373 // content.
1374 VTableNamePtr = (const char *)Ptr;
1375 if (VTableNamePtr > (const char *)DataBuffer->getBufferEnd())
1376 return make_error<InstrProfError>(instrprof_error::truncated);
1377 }
1378
1379 if (GET_VERSION(Header->formatVersion()) >= 10 &&
1380 Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) {
1381 uint64_t TemporalProfTracesOffset =
1382 endian::byte_swap<uint64_t, llvm::endianness::little>(
1383 Header->TemporalProfTracesOffset);
1384 const unsigned char *Ptr = Start + TemporalProfTracesOffset;
1385 const auto *PtrEnd = (const unsigned char *)DataBuffer->getBufferEnd();
1386 // Expect at least two 64 bit fields: NumTraces, and TraceStreamSize
1387 if (Ptr + 2 * sizeof(uint64_t) > PtrEnd)
1389 const uint64_t NumTraces =
1390 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1392 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1393 for (unsigned i = 0; i < NumTraces; i++) {
1394 // Expect at least two 64 bit fields: Weight and NumFunctions
1395 if (Ptr + 2 * sizeof(uint64_t) > PtrEnd)
1398 Trace.Weight =
1399 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1400 const uint64_t NumFunctions =
1401 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1402 // Expect at least NumFunctions 64 bit fields
1403 if (Ptr + NumFunctions * sizeof(uint64_t) > PtrEnd)
1405 for (unsigned j = 0; j < NumFunctions; j++) {
1406 const uint64_t NameRef =
1407 support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
1408 Trace.FunctionNameRefs.push_back(NameRef);
1409 }
1410 TemporalProfTraces.push_back(std::move(Trace));
1411 }
1412 }
1413
1414 // Load the remapping table now if requested.
1415 if (RemappingBuffer) {
1416 Remapper =
1417 std::make_unique<InstrProfReaderItaniumRemapper<OnDiskHashTableImplV3>>(
1418 std::move(RemappingBuffer), *IndexPtr);
1419 if (Error E = Remapper->populateRemappings())
1420 return E;
1421 } else {
1422 Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
1423 }
1424 Index = std::move(IndexPtr);
1425
1426 return success();
1427}
1428
1430 if (Symtab)
1431 return *Symtab;
1432
1433 auto NewSymtab = std::make_unique<InstrProfSymtab>();
1434
1435 if (Error E = NewSymtab->initVTableNamesFromCompressedStrings(
1436 StringRef(VTableNamePtr, CompressedVTableNamesLen))) {
1437 auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
1438 consumeError(error(ErrCode, Msg));
1439 }
1440
1441 // finalizeSymtab is called inside populateSymtab.
1442 if (Error E = Index->populateSymtab(*NewSymtab)) {
1443 auto [ErrCode, Msg] = InstrProfError::take(std::move(E));
1444 consumeError(error(ErrCode, Msg));
1445 }
1446
1447 Symtab = std::move(NewSymtab);
1448 return *Symtab;
1449}
1450
1452 StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName,
1453 uint64_t *MismatchedFuncSum) {
1455 uint64_t FuncSum = 0;
1456 auto Err = Remapper->getRecords(FuncName, Data);
1457 if (Err) {
1458 // If we don't find FuncName, try DeprecatedFuncName to handle profiles
1459 // built by older compilers.
1460 auto Err2 =
1461 handleErrors(std::move(Err), [&](const InstrProfError &IE) -> Error {
1462 if (IE.get() != instrprof_error::unknown_function)
1463 return make_error<InstrProfError>(IE);
1464 if (auto Err = Remapper->getRecords(DeprecatedFuncName, Data))
1465 return Err;
1466 return Error::success();
1467 });
1468 if (Err2)
1469 return std::move(Err2);
1470 }
1471 // Found it. Look for counters with the right hash.
1472
1473 // A flag to indicate if the records are from the same type
1474 // of profile (i.e cs vs nocs).
1475 bool CSBitMatch = false;
1476 auto getFuncSum = [](const std::vector<uint64_t> &Counts) {
1477 uint64_t ValueSum = 0;
1478 for (uint64_t CountValue : Counts) {
1479 if (CountValue == (uint64_t)-1)
1480 continue;
1481 // Handle overflow -- if that happens, return max.
1482 if (std::numeric_limits<uint64_t>::max() - CountValue <= ValueSum)
1483 return std::numeric_limits<uint64_t>::max();
1484 ValueSum += CountValue;
1485 }
1486 return ValueSum;
1487 };
1488
1489 for (const NamedInstrProfRecord &I : Data) {
1490 // Check for a match and fill the vector if there is one.
1491 if (I.Hash == FuncHash)
1492 return std::move(I);
1495 CSBitMatch = true;
1496 if (MismatchedFuncSum == nullptr)
1497 continue;
1498 FuncSum = std::max(FuncSum, getFuncSum(I.Counts));
1499 }
1500 }
1501 if (CSBitMatch) {
1502 if (MismatchedFuncSum != nullptr)
1503 *MismatchedFuncSum = FuncSum;
1505 }
1507}
1508
1511 // TODO: Add memprof specific errors.
1512 if (MemProfRecordTable == nullptr)
1513 return make_error<InstrProfError>(instrprof_error::invalid_prof,
1514 "no memprof data available in profile");
1515 auto Iter = MemProfRecordTable->find(FuncNameHash);
1516 if (Iter == MemProfRecordTable->end())
1517 return make_error<InstrProfError>(
1519 "memprof record not found for function hash " + Twine(FuncNameHash));
1520
1521 // Setup a callback to convert from frame ids to frame using the on-disk
1522 // FrameData hash table.
1523 std::optional<memprof::FrameId> LastUnmappedFrameId;
1524 auto IdToFrameCallback = [&](const memprof::FrameId Id) {
1525 auto FrIter = MemProfFrameTable->find(Id);
1526 if (FrIter == MemProfFrameTable->end()) {
1527 LastUnmappedFrameId = Id;
1528 return memprof::Frame(0, 0, 0, false);
1529 }
1530 return *FrIter;
1531 };
1532
1533 // Setup a callback to convert call stack ids to call stacks using the on-disk
1534 // hash table.
1535 std::optional<memprof::CallStackId> LastUnmappedCSId;
1536 auto CSIdToCallStackCallback = [&](memprof::CallStackId CSId) {
1538 auto CSIter = MemProfCallStackTable->find(CSId);
1539 if (CSIter == MemProfCallStackTable->end()) {
1540 LastUnmappedCSId = CSId;
1541 } else {
1542 const llvm::SmallVector<memprof::FrameId> &CS = *CSIter;
1543 Frames.reserve(CS.size());
1544 for (memprof::FrameId Id : CS)
1545 Frames.push_back(IdToFrameCallback(Id));
1546 }
1547 return Frames;
1548 };
1549
1550 const memprof::IndexedMemProfRecord IndexedRecord = *Iter;
1552 if (MemProfCallStackTable)
1553 Record = IndexedRecord.toMemProfRecord(CSIdToCallStackCallback);
1554 else
1555 Record = memprof::MemProfRecord(IndexedRecord, IdToFrameCallback);
1556
1557 // Check that all frame ids were successfully converted to frames.
1558 if (LastUnmappedFrameId) {
1559 return make_error<InstrProfError>(instrprof_error::hash_mismatch,
1560 "memprof frame not found for frame id " +
1561 Twine(*LastUnmappedFrameId));
1562 }
1563
1564 // Check that all call stack ids were successfully converted to call stacks.
1565 if (LastUnmappedCSId) {
1566 return make_error<InstrProfError>(
1568 "memprof call stack not found for call stack id " +
1569 Twine(*LastUnmappedCSId));
1570 }
1571 return Record;
1572}
1573
1575 uint64_t FuncHash,
1576 std::vector<uint64_t> &Counts) {
1578 if (Error E = Record.takeError())
1579 return error(std::move(E));
1580
1581 Counts = Record.get().Counts;
1582 return success();
1583}
1584
1586 uint64_t FuncHash,
1587 BitVector &Bitmap) {
1589 if (Error E = Record.takeError())
1590 return error(std::move(E));
1591
1592 const auto &BitmapBytes = Record.get().BitmapBytes;
1593 size_t I = 0, E = BitmapBytes.size();
1594 Bitmap.resize(E * CHAR_BIT);
1596 [&](auto X) {
1597 using XTy = decltype(X);
1598 alignas(XTy) uint8_t W[sizeof(X)];
1599 size_t N = std::min(E - I, sizeof(W));
1600 std::memset(W, 0, sizeof(W));
1601 std::memcpy(W, &BitmapBytes[I], N);
1602 I += N;
1604 support::aligned>(W);
1605 },
1606 Bitmap, Bitmap);
1607 assert(I == E);
1608
1609 return success();
1610}
1611
1614
1615 Error E = Index->getRecords(Data);
1616 if (E)
1617 return error(std::move(E));
1618
1619 Record = Data[RecordIndex++];
1620 if (RecordIndex >= Data.size()) {
1621 Index->advanceToNextKey();
1622 RecordIndex = 0;
1623 }
1624 return success();
1625}
1626
1628 std::vector<llvm::object::BuildID> &BinaryIds) {
1629 return readBinaryIdsInternal(*DataBuffer, BinaryIdsSize, BinaryIdsStart,
1630 BinaryIds, llvm::endianness::little);
1631}
1632
1634 std::vector<llvm::object::BuildID> BinaryIds;
1635 if (Error E = readBinaryIds(BinaryIds))
1636 return E;
1637 printBinaryIdsInternal(OS, BinaryIds);
1638 return Error::success();
1639}
1640
1642 uint64_t NumFuncs = 0;
1643 for (const auto &Func : *this) {
1644 if (isIRLevelProfile()) {
1645 bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(Func.Hash);
1646 if (FuncIsCS != IsCS)
1647 continue;
1648 }
1649 Func.accumulateCounts(Sum);
1650 ++NumFuncs;
1651 }
1652 Sum.NumEntries = NumFuncs;
1653}
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This file defines the DenseMap class.
std::string Name
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Provides ErrorOr<T> smart pointer.
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS)
static Error initializeReader(InstrProfReader &Reader)
static void printBinaryIdsInternal(raw_ostream &OS, std::vector< llvm::object::BuildID > &BinaryIds)
static Error readBinaryIdsInternal(const MemoryBuffer &DataBuffer, const uint64_t BinaryIdsSize, const uint8_t *BinaryIdsStart, std::vector< llvm::object::BuildID > &BinaryIds, const llvm::endianness Endian)
Read a list of binary ids from a profile that consist of a.
#define READ_NUM(Str, Dst)
#define CHECK_LINE_END(Line)
#define VP_READ_ADVANCE(Val)
static InstrProfKind getProfileKindFromVersion(uint64_t Version)
#define I(x, y, z)
Definition: MD5.cpp:58
if(VerifyEach)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
endianness Endian
raw_pwrite_stream & OS
This file contains some functions that are useful when dealing with strings.
#define error(X)
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
static BitVector & apply(F &&f, BitVector &Out, BitVector const &Arg, ArgTys const &...Args)
Definition: BitVector.h:552
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
Tagged union holding either a T or a Error.
Definition: Error.h:474
Error takeError()
Take ownership of the stored error.
Definition: Error.h:601
reference get()
Returns a reference to the stored T value.
Definition: Error.h:571
Reader for the indexed binary instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Error readHeader() override
Read the file header.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
Error getFunctionBitmap(StringRef FuncName, uint64_t FuncHash, BitVector &Bitmap)
Fill Bitmap with the profile data for the given function name.
InstrProfSymtab & getSymtab() override
Return the PGO symtab.
static bool hasFormat(const MemoryBuffer &DataBuffer)
Return true if the given buffer is in an indexed instrprof format.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)
Fill Counts with the profile data for the given function name.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
Error deserialize(const unsigned char *Start, uint64_t MemProfOffset)
Expected< memprof::MemProfRecord > getMemProfRecord(const uint64_t FuncNameHash) const
InstrProfCorrelator - A base class used to create raw instrumentation data to their functions.
const char * getNamesPointer() const
Return a pointer to the names string that this class constructs.
std::optional< size_t > getDataSize() const
Return the number of ProfileData elements.
size_t getNamesSize() const
Return the number of bytes in the names string.
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
Definition: InstrProf.h:416
data_type ReadData(StringRef K, const unsigned char *D, offset_type N)
bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)
hash_value_type ComputeHash(StringRef K)
ArrayRef< NamedInstrProfRecord > data_type
InstrProfKind getProfileKind() const override
Error getRecords(ArrayRef< NamedInstrProfRecord > &Data) override
InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)
A remapper that applies remappings based on a symbol remapping file.
static StringRef extractName(StringRef Name)
Extract the original function name from a PGO function name.
InstrProfReaderItaniumRemapper(std::unique_ptr< MemoryBuffer > RemapBuffer, InstrProfReaderIndex< HashTableImpl > &Underlying)
static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl< char > &Out)
Given a mangled name extracted from a PGO function name, and a new form for that mangled name,...
Error getRecords(StringRef FuncName, ArrayRef< NamedInstrProfRecord > &Data) override
Name matcher supporting fuzzy matching of symbol names to names in profiles.
Base class and interface for reading profiling data of any known instrprof format.
std::unique_ptr< InstrProfSymtab > Symtab
Error success()
Clear the current error and return a successful one.
SmallVector< TemporalProfTraceTy > TemporalProfTraces
A list of temporal profile traces.
uint64_t TemporalProfTraceStreamSize
The total number of temporal profile traces seen.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
virtual bool isIRLevelProfile() const =0
virtual Error readHeader()=0
Read the header. Required before reading first record.
void accumulateCounts(CountSumOrPercent &Sum, bool IsCS)
Compute the sum of counts and return in Sum.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
Definition: InstrProf.h:451
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
Definition: InstrProf.h:626
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Definition: InstrProf.h:594
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val)
Map the address range (i.e., [start_address, end_address)) of a variable to its names' MD5 hash.
Definition: InstrProf.h:601
This interface provides simple read-only access to a block of memory, and provides simple methods for...
Definition: MemoryBuffer.h:51
size_t getBufferSize() const
Definition: MemoryBuffer.h:68
const char * getBufferEnd() const
Definition: MemoryBuffer.h:67
static ErrorOr< std::unique_ptr< MemoryBuffer > > getSTDIN()
Read all of stdin into a file buffer, and return it.
const char * getBufferStart() const
Definition: MemoryBuffer.h:66
static OnDiskIterableChainedHashTable * Create(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, const Info &InfoObj=Info())
Create the hash table.
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:70
Reader for the raw instrprof binary format from runtime.
Error readHeader() override
Read the header. Required before reading first record.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
static bool hasFormat(const MemoryBuffer &DataBuffer)
InstrProfKind getProfileKind() const override
Returns a BitsetEnum describing the attributes of the raw instr profile.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
SmallVector< TemporalProfTraceTy > & getTemporalProfTraces(std::optional< uint64_t > Weight={}) override
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void reserve(size_type N)
Definition: SmallVector.h:676
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:818
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:696
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:466
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:567
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:257
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
iterator begin() const
Definition: StringRef.h:111
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
iterator end() const
Definition: StringRef.h:113
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Definition: StringRef.h:729
Reader for symbol remapping files.
Key insert(StringRef FunctionName)
Construct a key for the given symbol, or return an existing one if an equivalent name has already bee...
Key lookup(StringRef FunctionName)
Map the given symbol name into the key for the corresponding equivalence class.
Error read(MemoryBuffer &B)
Read remappings from the given buffer, which must live as long as the remapper.
Reader for the simple text based instrprof format.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if the given buffer is in text instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error readHeader() override
Read the header.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
LLVM Value Representation.
Definition: Value.h:74
bool is_at_end() const
Return true if we're an "end" iterator or have reached EOF.
Definition: LineIterator.h:63
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
The virtual file system interface.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1275
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1157
const uint64_t Version
Definition: InstrProf.h:1153
const uint64_t Magic
Definition: InstrProf.h:1114
const uint64_t Version
Definition: InstrProf.h:1298
constexpr uint64_t MaximumSupportedVersion
Definition: MemProf.h:32
constexpr uint64_t MinimumSupportedVersion
Definition: MemProf.h:31
Expected< MemProfSchema > readMemProfSchema(const unsigned char *&Buffer)
Definition: MemProf.cpp:260
value_type read(const void *memory, endianness endian)
Read a value of a particular endianness from memory.
Definition: Endian.h:58
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
RawInstrProfReader< uint32_t > RawInstrProfReader32
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
Definition: MathExtras.h:393
constexpr T byteswap(T V) noexcept
Reverses the bytes in the given integer value V.
Definition: bit.h:101
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
Definition: Error.h:947
auto formatv(const char *Fmt, Ts &&...Vals) -> formatv_object< decltype(std::make_tuple(support::detail::build_format_adapter(std::forward< Ts >(Vals))...))>
constexpr char kGlobalIdentifierDelimiter
Definition: GlobalValue.h:46
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:125
std::vector< ProfileSummaryEntry > SummaryEntryVector
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1914
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1849
RawInstrProfReader< uint64_t > RawInstrProfReader64
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:103
endianness
Definition: bit.h:70
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1041
InstrProfKind
An enum describing the attributes of an instrumented profile.
Definition: InstrProf.h:323
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
static Expected< Header > readFromBuffer(const unsigned char *Buffer)
Definition: InstrProf.cpp:1600
uint64_t Cutoff
The required percentile of total execution count.
Definition: InstrProf.h:1199
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
Definition: InstrProf.h:1202
uint64_t MinBlockCount
The minimum execution count for this percentile.
Definition: InstrProf.h:1201
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
Definition: InstrProf.h:1235
Profiling information for a single function.
Definition: InstrProf.h:808
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:1016
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
Definition: InstrProf.h:377
MemProfRecord toMemProfRecord(std::function< const llvm::SmallVector< Frame >(const CallStackId)> Callback) const
Definition: MemProf.cpp:226