37#include <system_error>
47 if (Version & VARIANT_MASK_IR_PROF) {
48 ProfileKind |= InstrProfKind::IRInstrumentation;
50 if (Version & VARIANT_MASK_CSIR_PROF) {
51 ProfileKind |= InstrProfKind::ContextSensitive;
53 if (Version & VARIANT_MASK_INSTR_ENTRY) {
54 ProfileKind |= InstrProfKind::FunctionEntryInstrumentation;
56 if (Version & VARIANT_MASK_BYTE_COVERAGE) {
57 ProfileKind |= InstrProfKind::SingleByteCoverage;
59 if (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) {
60 ProfileKind |= InstrProfKind::FunctionEntryOnly;
62 if (Version & VARIANT_MASK_MEMPROF) {
63 ProfileKind |= InstrProfKind::MemProf;
65 if (Version & VARIANT_MASK_TEMPORAL_PROF) {
66 ProfileKind |= InstrProfKind::TemporalProfile;
74 : FS.getBufferForFile(Filename);
75 if (std::error_code EC = BufferOrErr.getError())
77 return std::move(BufferOrErr.get());
95 const uint8_t *BinaryIdsStart,
96 std::vector<llvm::object::BuildID> &BinaryIds,
98 using namespace support;
100 if (BinaryIdsSize == 0)
103 const uint8_t *BI = BinaryIdsStart;
104 const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize;
106 reinterpret_cast<const uint8_t *
>(DataBuffer.
getBufferEnd());
109 size_t Remaining = BIEnd - BI;
112 return make_error<InstrProfError>(
113 instrprof_error::malformed,
114 "not enough data to read binary id length");
118 BILen = endian::readNext<uint64_t, llvm::endianness::little>(BI);
120 BILen = endian::readNext<uint64_t, llvm::endianness::big>(BI);
123 return make_error<InstrProfError>(instrprof_error::malformed,
124 "binary id length is 0");
126 Remaining = BIEnd - BI;
129 return make_error<InstrProfError>(
130 instrprof_error::malformed,
"not enough data to read binary id data");
138 return make_error<InstrProfError>(
139 instrprof_error::malformed,
140 "binary id section is greater than buffer size");
148 std::vector<llvm::object::BuildID> &BinaryIds) {
149 OS <<
"Binary IDs: \n";
150 for (
auto BI : BinaryIds) {
160 std::function<
void(
Error)> Warn) {
163 if (
Error E = BufferOrError.takeError())
172 std::function<
void(
Error)> Warn) {
173 if (Buffer->getBufferSize() == 0)
176 std::unique_ptr<InstrProfReader> Result;
193 return std::move(Result);
198 const Twine &RemappingPath) {
201 if (
Error E = BufferOrError.takeError())
205 std::unique_ptr<MemoryBuffer> RemappingBuffer;
206 std::string RemappingPathStr = RemappingPath.
str();
207 if (!RemappingPathStr.empty()) {
209 if (
Error E = RemappingBufferOrError.takeError())
211 RemappingBuffer = std::move(RemappingBufferOrError.get());
215 std::move(RemappingBuffer));
220 std::unique_ptr<MemoryBuffer> RemappingBuffer) {
224 auto Result = std::make_unique<IndexedInstrProfReader>(
225 std::move(Buffer), std::move(RemappingBuffer));
231 return std::move(Result);
241 [](
char c) { return isPrint(c) || isSpace(c); });
252 if (Str.equals_insensitive(
"ir"))
254 else if (Str.equals_insensitive(
"fe"))
256 else if (Str.equals_insensitive(
"csir")) {
259 }
else if (Str.equals_insensitive(
"entry_first"))
261 else if (Str.equals_insensitive(
"not_entry_first"))
263 else if (Str.equals_insensitive(
"single_byte_coverage"))
265 else if (Str.equals_insensitive(
"temporal_prof_traces")) {
267 if (
auto Err = readTemporalProfTraceData())
268 return error(std::move(Err));
281Error TextInstrProfReader::readTemporalProfTraceData() {
282 if ((++Line).is_at_end())
289 if ((++Line).is_at_end())
295 for (
uint32_t i = 0; i < NumTraces; i++) {
296 if ((++Line).is_at_end())
303 if ((++Line).is_at_end())
307 Line->
split(FuncNames,
",", -1,
false);
308 for (
auto &FuncName : FuncNames)
309 Trace.FunctionNameRefs.push_back(
319#define CHECK_LINE_END(Line) \
320 if (Line.is_at_end()) \
321 return error(instrprof_error::truncated);
322#define READ_NUM(Str, Dst) \
323 if ((Str).getAsInteger(10, (Dst))) \
324 return error(instrprof_error::malformed);
325#define VP_READ_ADVANCE(Val) \
326 CHECK_LINE_END(Line); \
328 READ_NUM((*Line), (Val)); \
339 if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
341 "number of value kinds is invalid");
344 for (
uint32_t VK = 0; VK < NumValueKinds; VK++) {
346 if (ValueKind > IPVK_Last)
353 Record.reserveSites(VK, NumValueSites);
354 for (
uint32_t S = 0; S < NumValueSites; S++) {
357 std::vector<InstrProfValueData> CurrentValues;
360 std::pair<StringRef, StringRef> VD = Line->
rsplit(
':');
362 if (ValueKind == IPVK_IndirectCallTarget) {
370 }
else if (ValueKind == IPVK_VTableTarget) {
382 CurrentValues.push_back({
Value, TakenCount});
385 Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData,
393#undef VP_READ_ADVANCE
408 return error(std::move(E));
413 if ((Line++)->getAsInteger(0,
Record.Hash))
415 "function hash is not a valid integer");
421 if ((Line++)->getAsInteger(10, NumCounters))
423 "number of counters is not a valid integer");
424 if (NumCounters == 0)
429 Record.Counts.reserve(NumCounters);
434 if ((Line++)->getAsInteger(10, Count))
436 Record.Counts.push_back(Count);
441 Record.BitmapBytes.clear();
444 if ((Line++)->drop_front(1).trim().getAsInteger(0, NumBitmapBytes))
446 "number of bitmap bytes is not a valid integer");
447 if (NumBitmapBytes != 0) {
449 Record.BitmapBytes.reserve(NumBitmapBytes);
450 for (uint8_t
I = 0;
I < NumBitmapBytes; ++
I) {
454 if ((Line++)->getAsInteger(0, BitmapByte))
456 "bitmap byte is not a valid integer");
457 Record.BitmapBytes.push_back(BitmapByte);
464 return error(std::move(E));
469template <
class IntPtrT>
474template <
class IntPtrT>
477 std::optional<uint64_t> Weight) {
478 if (TemporalProfTimestamps.empty()) {
479 assert(TemporalProfTraces.empty());
480 return TemporalProfTraces;
483 std::sort(TemporalProfTimestamps.begin(), TemporalProfTimestamps.end());
486 Trace.Weight = *Weight;
487 for (
auto &[TimestampValue, NameRef] : TemporalProfTimestamps)
488 Trace.FunctionNameRefs.push_back(NameRef);
489 TemporalProfTraces = {std::move(
Trace)};
490 return TemporalProfTraces;
493template <
class IntPtrT>
499 return RawInstrProf::getMagic<IntPtrT>() == Magic ||
503template <
class IntPtrT>
505 if (!hasFormat(*DataBuffer))
510 DataBuffer->getBufferStart());
511 ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
512 return readHeader(*Header);
515template <
class IntPtrT>
517 const char *
End = DataBuffer->getBufferEnd();
519 while (CurrentPos !=
End && *CurrentPos == 0)
522 if (CurrentPos ==
End)
528 "not enough space for another header");
530 if (
reinterpret_cast<size_t>(CurrentPos) %
alignof(
uint64_t))
532 "insufficient padding");
535 if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
540 return readHeader(*Header);
543template <
class IntPtrT>
546 StringRef(VNamesStart, VNamesEnd - VNamesStart)))
547 return error(std::move(E));
549 const IntPtrT FPtr = swap(
I->FunctionPointer);
555 if (VTableBegin !=
nullptr && VTableEnd !=
nullptr) {
557 I != VTableEnd; ++
I) {
566 swap(
I->VTableNameHash));
572template <
class IntPtrT>
578 (
"Profile uses raw profile format version = " +
579 Twine(GET_VERSION(Version)) +
581 "\nPLEASE update this tool to version in the raw profile, or "
582 "regenerate raw profile with expected version.")
587 const uint8_t *BinaryIdStart =
589 const uint8_t *BinaryIdEnd = BinaryIdStart + BinaryIdSize;
590 const uint8_t *BufferEnd = (
const uint8_t *)DataBuffer->getBufferEnd();
591 if (BinaryIdSize %
sizeof(
uint64_t) || BinaryIdEnd > BufferEnd)
593 if (BinaryIdSize != 0) {
596 BinaryIds, getDataEndianness()))
600 CountersDelta =
swap(Header.CountersDelta);
601 BitmapDelta =
swap(Header.BitmapDelta);
602 NamesDelta =
swap(Header.NamesDelta);
603 auto NumData =
swap(Header.NumData);
604 auto PaddingBytesBeforeCounters =
swap(Header.PaddingBytesBeforeCounters);
605 auto CountersSize =
swap(Header.NumCounters) * getCounterTypeSize();
606 auto PaddingBytesAfterCounters =
swap(Header.PaddingBytesAfterCounters);
607 auto NumBitmapBytes =
swap(Header.NumBitmapBytes);
608 auto PaddingBytesAfterBitmapBytes =
swap(Header.PaddingBytesAfterBitmapBytes);
609 auto NamesSize =
swap(Header.NamesSize);
610 auto VTableNameSize =
swap(Header.VNamesSize);
611 auto NumVTables =
swap(Header.NumVTables);
612 ValueKindLast =
swap(Header.ValueKindLast);
615 auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize);
616 auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize);
618 auto VTableSectionSize =
620 auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize);
624 ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
626 CountersOffset + CountersSize + PaddingBytesAfterCounters;
628 BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes;
630 NamesOffset + NamesSize + PaddingBytesAfterNames;
631 ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize +
632 PaddingBytesAfterVTableProfData;
634 VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames;
636 auto *Start =
reinterpret_cast<const char *
>(&Header);
637 if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
643 if (!(DataSize == 0 && NamesSize == 0 && CountersDelta == 0 &&
646 Data = Correlator->getDataPointer();
653 DataEnd =
Data + NumData;
656 Start + VTableProfDataOffset);
657 VTableEnd = VTableBegin + NumVTables;
658 NamesStart = Start + NamesOffset;
659 NamesEnd = NamesStart + NamesSize;
660 VNamesStart = Start + VTableNameOffset;
661 VNamesEnd = VNamesStart + VTableNameSize;
664 CountersStart = Start + CountersOffset;
665 CountersEnd = CountersStart + CountersSize;
666 BitmapStart = Start + BitmapOffset;
667 BitmapEnd = BitmapStart + NumBitmapBytes;
668 ValueDataStart =
reinterpret_cast<const uint8_t *
>(Start + ValueDataOffset);
670 std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
671 if (
Error E = createSymtab(*NewSymtab))
674 Symtab = std::move(NewSymtab);
678template <
class IntPtrT>
684template <
class IntPtrT>
690template <
class IntPtrT>
694 if (NumCounters == 0)
698 if (CounterBaseOffset < 0)
701 (
"counter offset " +
Twine(CounterBaseOffset) +
" is negative").str());
703 if (CounterBaseOffset >= CountersEnd - CountersStart)
705 (
"counter offset " +
Twine(CounterBaseOffset) +
706 " is greater than the maximum counter offset " +
707 Twine(CountersEnd - CountersStart - 1))
711 (CountersEnd - (CountersStart + CounterBaseOffset)) /
712 getCounterTypeSize();
713 if (NumCounters > MaxNumCounters)
715 (
"number of counters " +
Twine(NumCounters) +
716 " is greater than the maximum number of counters " +
717 Twine(MaxNumCounters))
721 Record.Counts.reserve(NumCounters);
724 CountersStart + CounterBaseOffset +
I * getCounterTypeSize();
725 if (
I == 0 && hasTemporalProfile()) {
727 if (TimestampValue != 0 &&
728 TimestampValue != std::numeric_limits<uint64_t>::max()) {
729 TemporalProfTimestamps.emplace_back(TimestampValue,
731 TemporalProfTraceStreamSize = 1;
733 if (hasSingleByteCoverage()) {
742 if (hasSingleByteCoverage()) {
744 Record.Counts.push_back(*
Ptr == 0 ? 1 : 0);
747 if (CounterValue > MaxCounterValue && Warn)
748 Warn(make_error<InstrProfError>(
751 Record.Counts.push_back(CounterValue);
758template <
class IntPtrT>
762 Record.BitmapBytes.clear();
763 Record.BitmapBytes.reserve(NumBitmapBytes);
767 if (NumBitmapBytes == 0)
772 if (BitmapOffset < 0)
775 (
"bitmap offset " +
Twine(BitmapOffset) +
" is negative").str());
777 if (BitmapOffset >= BitmapEnd - BitmapStart)
779 (
"bitmap offset " +
Twine(BitmapOffset) +
780 " is greater than the maximum bitmap offset " +
781 Twine(BitmapEnd - BitmapStart - 1))
785 (BitmapEnd - (BitmapStart + BitmapOffset)) /
sizeof(uint8_t);
786 if (NumBitmapBytes > MaxNumBitmapBytes)
788 (
"number of bitmap bytes " +
Twine(NumBitmapBytes) +
789 " is greater than the maximum number of bitmap bytes " +
790 Twine(MaxNumBitmapBytes))
794 const char *
Ptr = BitmapStart + BitmapOffset +
I;
801template <
class IntPtrT>
805 CurValueDataSize = 0;
809 NumValueKinds += (
Data->NumValueSites[
I] != 0);
815 ValueProfData::getValueProfData(
816 ValueDataStart, (
const unsigned char *)DataBuffer->getBufferEnd(),
817 getDataEndianness());
825 VDataPtrOrErr.
get()->deserializeTo(
Record, Symtab.get());
826 CurValueDataSize = VDataPtrOrErr.
get()->getSize();
830template <
class IntPtrT>
836 if (
Error E = readNextHeader(getNextHeaderPos()))
837 return error(std::move(E));
841 return error(std::move(E));
845 return error(std::move(E));
849 return error(std::move(E));
853 return error(std::move(E));
857 return error(std::move(E));
864template <
class IntPtrT>
866 std::vector<llvm::object::BuildID> &BinaryIds) {
867 BinaryIds.insert(BinaryIds.begin(), this->BinaryIds.begin(),
868 this->BinaryIds.end());
872template <
class IntPtrT>
874 if (!BinaryIds.empty())
895 const unsigned char *&
D,
const unsigned char *
const End) {
897 ValueProfData::getValueProfData(
D,
End, ValueProfDataEndianness);
902 VDataPtrOrErr.
get()->deserializeTo(DataBuffer.back(),
nullptr);
903 D += VDataPtrOrErr.
get()->TotalSize;
910 using namespace support;
917 std::vector<uint64_t> CounterBuffer;
918 std::vector<uint8_t> BitmapByteBuffer;
920 const unsigned char *
End =
D +
N;
925 uint64_t Hash = endian::readNext<uint64_t, llvm::endianness::little>(
D);
933 CountsSize = endian::readNext<uint64_t, llvm::endianness::little>(
D);
939 CounterBuffer.clear();
940 CounterBuffer.reserve(CountsSize);
941 for (
uint64_t J = 0; J < CountsSize; ++J)
942 CounterBuffer.push_back(
943 endian::readNext<uint64_t, llvm::endianness::little>(
D));
950 BitmapBytes = endian::readNext<uint64_t, llvm::endianness::little>(
D);
952 if (
D + BitmapBytes *
sizeof(uint8_t) >
End)
954 BitmapByteBuffer.clear();
955 BitmapByteBuffer.reserve(BitmapBytes);
956 for (
uint64_t J = 0; J < BitmapBytes; ++J)
957 BitmapByteBuffer.push_back(
static_cast<uint8_t
>(
958 endian::readNext<uint64_t, llvm::endianness::little>(
D)));
961 DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer),
962 std::move(BitmapByteBuffer));
974template <
typename HashTableImpl>
977 auto Iter = HashTable->find(FuncName);
978 if (Iter == HashTable->end())
984 "profile data is empty");
989template <
typename HashTableImpl>
995 Data = *RecordIterator;
999 "profile data is empty");
1004template <
typename HashTableImpl>
1006 const unsigned char *Buckets,
const unsigned char *
const Payload,
1009 FormatVersion = Version;
1010 HashTable.reset(HashTableImpl::Create(
1011 Buckets, Payload,
Base,
1012 typename HashTableImpl::InfoType(HashType, Version)));
1013 RecordIterator = HashTable->data_begin();
1016template <
typename HashTableImpl>
1028 : Underlying(Underlying) {}
1032 return Underlying.getRecords(FuncName, Data);
1038template <
typename HashTableImpl>
1043 std::unique_ptr<MemoryBuffer> RemapBuffer,
1045 : RemapBuffer(
std::
move(RemapBuffer)), Underlying(Underlying) {
1054 std::pair<StringRef, StringRef> Parts = {
StringRef(),
Name};
1057 if (Parts.first.starts_with(
"_Z"))
1059 if (Parts.second.empty())
1076 if (
Error E = Remappings.
read(*RemapBuffer))
1080 if (
auto Key = Remappings.
insert(RealName)) {
1084 MappedNames.insert({Key, RealName});
1093 if (
auto Key = Remappings.
lookup(RealName)) {
1094 StringRef Remapped = MappedNames.lookup(Key);
1095 if (!Remapped.
empty()) {
1097 RealName.
end() == FuncName.
end())
1098 FuncName = Remapped;
1103 Error E = Underlying.getRecords(Reconstituted,
Data);
1110 std::move(E), [](std::unique_ptr<InstrProfError> Err) {
1113 :
Error(std::move(Err));
1119 return Underlying.getRecords(FuncName,
Data);
1125 std::unique_ptr<MemoryBuffer> RemapBuffer;
1141 using namespace support;
1145 uint64_t Magic = endian::read<uint64_t, llvm::endianness::little, aligned>(
1151const unsigned char *
1153 const unsigned char *Cur,
bool UseCS) {
1154 using namespace IndexedInstrProf;
1155 using namespace support;
1160 uint64_t NFields = endian::byte_swap<uint64_t, llvm::endianness::little>(
1162 uint64_t NEntries = endian::byte_swap<uint64_t, llvm::endianness::little>(
1166 std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
1171 for (
unsigned I = 0;
I < SummarySize /
sizeof(
uint64_t);
I++)
1172 Dst[
I] = endian::byte_swap<uint64_t, llvm::endianness::little>(Src[
I]);
1175 for (
unsigned I = 0;
I < SummaryData->NumCutoffEntries;
I++) {
1180 std::unique_ptr<llvm::ProfileSummary> &Summary =
1181 UseCS ? this->CS_Summary : this->Summary;
1184 Summary = std::make_unique<ProfileSummary>(
1186 DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
1187 SummaryData->get(Summary::MaxBlockCount),
1188 SummaryData->get(Summary::MaxInternalBlockCount),
1189 SummaryData->get(Summary::MaxFunctionCount),
1190 SummaryData->get(Summary::TotalNumBlocks),
1191 SummaryData->get(Summary::TotalNumFunctions));
1192 return Cur + SummarySize;
1200 Summary = Builder.getSummary();
1206 using namespace support;
1208 const unsigned char *Start =
1209 (
const unsigned char *)DataBuffer->getBufferStart();
1210 const unsigned char *Cur = Start;
1211 if ((
const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
1216 return HeaderOr.takeError();
1219 Cur += Header->size();
1223 if (Header->formatVersion() & VARIANT_MASK_CSIR_PROF)
1229 endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashType));
1234 endian::byte_swap<uint64_t, llvm::endianness::little>(Header->HashOffset);
1237 auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
1238 Start + HashOffset, Cur, Start, HashType, Header->formatVersion());
1242 if (GET_VERSION(Header->formatVersion()) >= 8 &&
1243 Header->formatVersion() & VARIANT_MASK_MEMPROF) {
1245 endian::byte_swap<uint64_t, llvm::endianness::little>(
1246 Header->MemProfOffset);
1248 const unsigned char *
Ptr = Start + MemProfOffset;
1254 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1260 }
else if (FirstWord >= 24) {
1267 return make_error<InstrProfError>(
1269 formatv(
"MemProf version {} not supported; "
1270 "requires version between {} and {}, inclusive",
1279 : support::endian::readNext<uint64_t, llvm::endianness::little>(
1283 const uint64_t FramePayloadOffset =
1284 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1287 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1292 return SchemaOr.takeError();
1293 Schema = SchemaOr.get();
1297 Start + RecordTableOffset,
1303 Start + FrameTableOffset,
1304 Start + FramePayloadOffset,
1307#ifdef EXPENSIVE_CHECKS
1311 for (
const auto &
Record : MemProfRecordTable->data())
1312 verifyIndexedMemProfRecord(
Record);
1318 if (GET_VERSION(Header->formatVersion()) >= 9) {
1320 endian::byte_swap<uint64_t, llvm::endianness::little>(
1321 Header->BinaryIdOffset);
1322 const unsigned char *
Ptr = Start + BinaryIdOffset;
1325 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1326 if (BinaryIdsSize %
sizeof(
uint64_t))
1329 BinaryIdsStart =
Ptr;
1330 if (BinaryIdsStart > (
const unsigned char *)DataBuffer->getBufferEnd())
1332 "corrupted binary ids");
1335 if (GET_VERSION(Header->formatVersion()) >= 12) {
1337 endian::byte_swap<uint64_t, llvm::endianness::little>(
1338 Header->VTableNamesOffset);
1339 const unsigned char *
Ptr = Start + VTableNamesOffset;
1341 CompressedVTableNamesLen =
1342 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1346 VTableNamePtr = (
const char *)
Ptr;
1347 if (VTableNamePtr > (
const char *)DataBuffer->getBufferEnd())
1351 if (GET_VERSION(Header->formatVersion()) >= 10 &&
1352 Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) {
1353 uint64_t TemporalProfTracesOffset =
1354 endian::byte_swap<uint64_t, llvm::endianness::little>(
1355 Header->TemporalProfTracesOffset);
1356 const unsigned char *
Ptr = Start + TemporalProfTracesOffset;
1357 const auto *PtrEnd = (
const unsigned char *)DataBuffer->getBufferEnd();
1362 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1364 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1365 for (
unsigned i = 0; i < NumTraces; i++) {
1371 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1373 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1375 if (
Ptr + NumFunctions *
sizeof(
uint64_t) > PtrEnd)
1377 for (
unsigned j = 0; j < NumFunctions; j++) {
1379 support::endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
1380 Trace.FunctionNameRefs.push_back(NameRef);
1387 if (RemappingBuffer) {
1389 std::make_unique<InstrProfReaderItaniumRemapper<OnDiskHashTableImplV3>>(
1390 std::move(RemappingBuffer), *IndexPtr);
1391 if (
Error E = Remapper->populateRemappings())
1394 Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
1396 Index = std::move(IndexPtr);
1405 auto NewSymtab = std::make_unique<InstrProfSymtab>();
1407 if (
Error E = NewSymtab->initVTableNamesFromCompressedStrings(
1408 StringRef(VTableNamePtr, CompressedVTableNamesLen))) {
1414 if (
Error E =
Index->populateSymtab(*NewSymtab)) {
1419 Symtab = std::move(NewSymtab);
1428 auto Err = Remapper->getRecords(FuncName,
Data);
1435 return make_error<InstrProfError>(IE);
1436 if (
auto Err = Remapper->getRecords(DeprecatedFuncName,
Data))
1441 return std::move(Err2);
1447 bool CSBitMatch =
false;
1448 auto getFuncSum = [](
const std::vector<uint64_t> &Counts) {
1450 for (
uint64_t CountValue : Counts) {
1454 if (std::numeric_limits<uint64_t>::max() - CountValue <= ValueSum)
1455 return std::numeric_limits<uint64_t>::max();
1456 ValueSum += CountValue;
1463 if (
I.Hash == FuncHash)
1464 return std::move(
I);
1468 if (MismatchedFuncSum ==
nullptr)
1470 FuncSum = std::max(FuncSum, getFuncSum(
I.Counts));
1474 if (MismatchedFuncSum !=
nullptr)
1475 *MismatchedFuncSum = FuncSum;
1484 if (MemProfRecordTable ==
nullptr)
1486 "no memprof data available in profile");
1487 auto Iter = MemProfRecordTable->find(FuncNameHash);
1488 if (Iter == MemProfRecordTable->end())
1489 return make_error<InstrProfError>(
1491 "memprof record not found for function hash " +
Twine(FuncNameHash));
1495 std::optional<memprof::FrameId> LastUnmappedFrameId;
1497 auto FrIter = MemProfFrameTable->find(Id);
1498 if (FrIter == MemProfFrameTable->end()) {
1499 LastUnmappedFrameId = Id;
1508 if (LastUnmappedFrameId) {
1510 "memprof frame not found for frame id " +
1511 Twine(*LastUnmappedFrameId));
1518 std::vector<uint64_t> &Counts) {
1521 return error(std::move(E));
1523 Counts =
Record.get().Counts;
1532 return error(std::move(E));
1534 const auto &BitmapBytes =
Record.get().BitmapBytes;
1535 size_t I = 0, E = BitmapBytes.size();
1536 Bitmap.
resize(E * CHAR_BIT);
1539 using XTy =
decltype(
X);
1540 alignas(XTy) uint8_t W[
sizeof(
X)];
1541 size_t N = std::min(E -
I,
sizeof(W));
1542 std::memset(W, 0,
sizeof(W));
1543 std::memcpy(W, &BitmapBytes[
I],
N);
1559 return error(std::move(E));
1562 if (RecordIndex >=
Data.size()) {
1563 Index->advanceToNextKey();
1570 std::vector<llvm::object::BuildID> &BinaryIds) {
1576 std::vector<llvm::object::BuildID> BinaryIds;
1585 for (
const auto &Func : *
this) {
1588 if (FuncIsCS != IsCS)
1591 Func.accumulateCounts(Sum);
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Provides ErrorOr<T> smart pointer.
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS)
static Error initializeReader(InstrProfReader &Reader)
static void printBinaryIdsInternal(raw_ostream &OS, std::vector< llvm::object::BuildID > &BinaryIds)
static Error readBinaryIdsInternal(const MemoryBuffer &DataBuffer, const uint64_t BinaryIdsSize, const uint8_t *BinaryIdsStart, std::vector< llvm::object::BuildID > &BinaryIds, const llvm::endianness Endian)
Read a list of binary ids from a profile that consist of a.
#define READ_NUM(Str, Dst)
#define CHECK_LINE_END(Line)
#define VP_READ_ADVANCE(Val)
static InstrProfKind getProfileKindFromVersion(uint64_t Version)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
static BitVector & apply(F &&f, BitVector &Out, BitVector const &Arg, ArgTys const &...Args)
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
Error takeError()
Take ownership of the stored error.
reference get()
Returns a reference to the stored T value.
Reader for the indexed binary instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Expected< memprof::MemProfRecord > getMemProfRecord(uint64_t FuncNameHash)
Return the memprof record for the function identified by llvm::md5(Name).
Error readHeader() override
Read the file header.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
Error getFunctionBitmap(StringRef FuncName, uint64_t FuncHash, BitVector &Bitmap)
Fill Bitmap with the profile data for the given function name.
InstrProfSymtab & getSymtab() override
Return the PGO symtab.
static bool hasFormat(const MemoryBuffer &DataBuffer)
Return true if the given buffer is in an indexed instrprof format.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)
Fill Counts with the profile data for the given function name.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
InstrProfCorrelator - A base class used to create raw instrumentation data to their functions.
const char * getNamesPointer() const
Return a pointer to the names string that this class constructs.
std::optional< size_t > getDataSize() const
Return the number of ProfileData elements.
size_t getNamesSize() const
Return the number of bytes in the names string.
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
data_type ReadData(StringRef K, const unsigned char *D, offset_type N)
bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)
hash_value_type ComputeHash(StringRef K)
ArrayRef< NamedInstrProfRecord > data_type
InstrProfKind getProfileKind() const override
Error getRecords(ArrayRef< NamedInstrProfRecord > &Data) override
InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)
A remapper that applies remappings based on a symbol remapping file.
static StringRef extractName(StringRef Name)
Extract the original function name from a PGO function name.
InstrProfReaderItaniumRemapper(std::unique_ptr< MemoryBuffer > RemapBuffer, InstrProfReaderIndex< HashTableImpl > &Underlying)
Error populateRemappings() override
static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl< char > &Out)
Given a mangled name extracted from a PGO function name, and a new form for that mangled name,...
Error getRecords(StringRef FuncName, ArrayRef< NamedInstrProfRecord > &Data) override
Name matcher supporting fuzzy matching of symbol names to names in profiles.
Base class and interface for reading profiling data of any known instrprof format.
std::unique_ptr< InstrProfSymtab > Symtab
Error success()
Clear the current error and return a successful one.
SmallVector< TemporalProfTraceTy > TemporalProfTraces
A list of temporal profile traces.
uint64_t TemporalProfTraceStreamSize
The total number of temporal profile traces seen.
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, std::function< void(Error)> Warn=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
virtual bool isIRLevelProfile() const =0
virtual Error readHeader()=0
Read the header. Required before reading first record.
void accumulateCounts(CountSumOrPercent &Sum, bool IsCS)
Compute the sum of counts and return in Sum.
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val)
Map the address range (i.e., [start_address, end_address)) of a variable to its names' MD5 hash.
This interface provides simple read-only access to a block of memory, and provides simple methods for...
size_t getBufferSize() const
const char * getBufferEnd() const
static ErrorOr< std::unique_ptr< MemoryBuffer > > getSTDIN()
Read all of stdin into a file buffer, and return it.
const char * getBufferStart() const
static OnDiskIterableChainedHashTable * Create(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, const Info &InfoObj=Info())
Create the hash table.
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Reader for the raw instrprof binary format from runtime.
Error readHeader() override
Read the header. Required before reading first record.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error printBinaryIds(raw_ostream &OS) override
Print binary ids.
static bool hasFormat(const MemoryBuffer &DataBuffer)
InstrProfKind getProfileKind() const override
Returns a BitsetEnum describing the attributes of the raw instr profile.
Error readBinaryIds(std::vector< llvm::object::BuildID > &BinaryIds) override
Read a list of binary ids.
SmallVector< TemporalProfTraceTy > & getTemporalProfTraces(std::optional< uint64_t > Weight={}) override
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
iterator insert(iterator I, T &&Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr size_t size() const
size - Get the string size.
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Reader for symbol remapping files.
Key insert(StringRef FunctionName)
Construct a key for the given symbol, or return an existing one if an equivalent name has already bee...
Key lookup(StringRef FunctionName)
Map the given symbol name into the key for the corresponding equivalence class.
Error read(MemoryBuffer &B)
Read remappings from the given buffer, which must live as long as the remapper.
Reader for the simple text based instrprof format.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if the given buffer is in text instrprof format.
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Error readHeader() override
Read the header.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
LLVM Value Representation.
bool is_at_end() const
Return true if we're an "end" iterator or have reached EOF.
This class implements an extremely fast bulk output stream that can only output to a stream.
The virtual file system interface.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
uint64_t ComputeHash(StringRef K)
constexpr uint64_t MaximumSupportedVersion
constexpr uint64_t MinimumSupportedVersion
Expected< MemProfSchema > readMemProfSchema(const unsigned char *&Buffer)
value_type read(const void *memory, endianness endian)
Read a value of a particular endianness from memory.
This is an optimization pass for GlobalISel generic memory operations.
RawInstrProfReader< uint32_t > RawInstrProfReader32
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
constexpr T byteswap(T V) noexcept
Reverses the bytes in the given integer value V.
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
auto formatv(const char *Fmt, Ts &&...Vals) -> formatv_object< decltype(std::make_tuple(support::detail::build_format_adapter(std::forward< Ts >(Vals))...))>
constexpr char kGlobalIdentifierDelimiter
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
std::vector< ProfileSummaryEntry > SummaryEntryVector
@ raw_profile_version_mismatch
@ counter_value_too_large
@ unexpected_correlation_info
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
RawInstrProfReader< uint64_t > RawInstrProfReader64
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
void consumeError(Error Err)
Consume a Error without doing anything.
InstrProfKind
An enum describing the attributes of an instrumented profile.
@ FunctionEntryInstrumentation
@ FrontendInstrumentation
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
uint64_t Cutoff
The required percentile of total execution count.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
uint64_t MinBlockCount
The minimum execution count for this percentile.
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
uint64_t NumSummaryFields
uint64_t NumCutoffEntries
Profiling information for a single function.
static bool hasCSFlagInHash(uint64_t FuncHash)
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.