43 #include <system_error>
47 using namespace sampleprof;
49 #define DEBUG_TYPE "samplepgo-reader"
56 cl::desc(
"Profile uses flow sensitive discriminators"));
69 std::vector<NameFunctionSamples> V;
71 for (
const auto &
I : V)
86 size_t n2 = Input.rfind(
':');
87 size_t n1 = Input.rfind(
':', n2 - 1);
88 FName = Input.substr(0, n1);
89 if (Input.substr(n1 + 1, n2 - n1 - 1).getAsInteger(10, NumSamples))
91 if (Input.substr(n2 + 1).getAsInteger(10, NumHeadSamples))
108 if (Input.startswith(
"!CFGChecksum:")) {
109 StringRef CFGInfo = Input.substr(strlen(
"!CFGChecksum:")).trim();
113 if (Input.startswith(
"!Attributes:")) {
114 StringRef Attrib = Input.substr(strlen(
"!Attributes:")).trim();
149 if (Input[
Depth] ==
'!') {
154 size_t n1 = Input.find(
':');
156 size_t n2 = Loc.
find(
'.');
171 size_t n3 = Rest.
find(
' ');
221 if (n4 == Rest.
size())
229 CalleeName = Rest.
substr(0, n3);
247 uint32_t TopLevelProbeProfileCount = 0;
256 if ((*LineIt)[(*LineIt).find_first_not_of(
' ')] ==
'#')
270 if ((*LineIt)[0] !=
' ') {
271 uint64_t NumSamples, NumHeadSamples;
273 if (!
ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) {
275 "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
288 InlineStack.push_back(&FProfile);
298 Discriminator, FName, TargetCountMap, FunctionHash,
301 "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
308 "Found non-metadata after metadata: " + *LineIt);
315 while (InlineStack.size() >
Depth) {
316 InlineStack.pop_back();
321 LineLocation(LineOffset, Discriminator))[std::string(FName)];
324 InlineStack.push_back(&FSamples);
329 while (InlineStack.size() >
Depth) {
330 InlineStack.pop_back();
333 for (
const auto &name_count : TargetCountMap) {
335 LineOffset, Discriminator, name_count.first,
347 ++TopLevelProbeProfileCount;
352 DepthMetadata =
Depth;
360 "Cannot have both context-sensitive and regular profile");
362 assert((TopLevelProbeProfileCount == 0 ||
363 TopLevelProbeProfileCount ==
Profiles.size()) &&
364 "Cannot have both probe-based profiles and regular profiles");
382 if ((*LineIt)[0] !=
' ') {
383 uint64_t NumSamples, NumHeadSamples;
393 unsigned NumBytesRead = 0;
399 else if (
Data + NumBytesRead >
End)
409 Data += NumBytesRead;
410 return static_cast<T>(Val);
416 if (
Data + Str.size() + 1 >
End) {
422 Data += Str.size() + 1;
426 template <
typename T>
437 T Val = endian::readNext<T, little, unaligned>(
Data);
441 template <
typename T>
444 auto Idx = readNumber<uint32_t>();
445 if (std::error_code EC = Idx.getError())
447 if (*Idx >= Table.size())
454 if (std::error_code EC = Idx.getError())
462 if (std::error_code EC = FName.getError())
473 if (std::error_code EC = Idx.getError())
480 const uint8_t *SavedData =
Data;
482 auto FID = readUnencodedNumber<uint64_t>();
483 if (std::error_code EC = FID.getError())
489 SR = MD5StringBuf->
back();
497 if (std::error_code EC = Idx.getError())
505 auto NumSamples = readNumber<uint64_t>();
506 if (std::error_code EC = NumSamples.getError())
511 auto NumRecords = readNumber<uint32_t>();
512 if (std::error_code EC = NumRecords.getError())
516 auto LineOffset = readNumber<uint64_t>();
517 if (std::error_code EC = LineOffset.getError())
521 return std::error_code();
524 auto Discriminator = readNumber<uint64_t>();
525 if (std::error_code EC = Discriminator.getError())
528 auto NumSamples = readNumber<uint64_t>();
529 if (std::error_code EC = NumSamples.getError())
532 auto NumCalls = readNumber<uint32_t>();
533 if (std::error_code EC = NumCalls.getError())
539 for (
uint32_t J = 0; J < *NumCalls; ++J) {
541 if (std::error_code EC = CalledFunction.getError())
544 auto CalledFunctionSamples = readNumber<uint64_t>();
545 if (std::error_code EC = CalledFunctionSamples.getError())
549 *CalledFunction, *CalledFunctionSamples);
552 FProfile.
addBodySamples(*LineOffset, DiscriminatorVal, *NumSamples);
556 auto NumCallsites = readNumber<uint32_t>();
557 if (std::error_code EC = NumCallsites.getError())
560 for (
uint32_t J = 0; J < *NumCallsites; ++J) {
561 auto LineOffset = readNumber<uint64_t>();
562 if (std::error_code EC = LineOffset.getError())
565 auto Discriminator = readNumber<uint64_t>();
566 if (std::error_code EC = Discriminator.getError())
570 if (std::error_code EC = FName.getError())
577 LineLocation(*LineOffset, DiscriminatorVal))[std::string(*FName)];
579 if (std::error_code EC =
readProfile(CalleeProfile))
589 auto NumHeadSamples = readNumber<uint64_t>();
590 if (std::error_code EC = NumHeadSamples.getError())
594 if (std::error_code EC = FContext.
getError())
602 if (FContext->hasContext())
623 auto ContextIdx = readNumber<uint32_t>();
624 if (std::error_code EC = ContextIdx.getError())
626 if (*ContextIdx >= CSNameTable->size())
628 return (*CSNameTable)[*ContextIdx];
634 auto FContext(readContextFromTable());
635 if (std::error_code EC = FContext.getError())
640 if (std::error_code EC = FName.getError())
650 switch (Entry.Type) {
655 Summary->setPartialProfile(
true);
667 assert((!FixedLengthMD5 || UseMD5) &&
668 "If FixedLengthMD5 is true, UseMD5 has to be true");
671 if (std::error_code EC = readNameTableSec(UseMD5))
676 if (std::error_code EC = readCSNameTableSec())
681 if (std::error_code EC = readFuncProfiles())
686 if (std::error_code EC = readFuncOffsetTable())
695 if (std::error_code EC = readFuncMetadata(HasAttribute))
700 if (std::error_code EC = readProfileSymbolList())
704 if (std::error_code EC = readCustomSection(Entry))
724 FuncOffsetTable.clear();
726 auto Size = readNumber<uint64_t>();
727 if (std::error_code EC = Size.getError())
730 FuncOffsetTable.reserve(*Size);
732 if (FuncOffsetsOrdered) {
734 std::make_unique<std::vector<std::pair<SampleContext, uint64_t>>>();
735 OrderedFuncOffsets->reserve(*Size);
740 if (std::error_code EC = FContext.getError())
743 auto Offset = readNumber<uint64_t>();
744 if (std::error_code EC = Offset.getError())
747 FuncOffsetTable[*FContext] = *Offset;
748 if (FuncOffsetsOrdered)
749 OrderedFuncOffsets->emplace_back(*FContext, *Offset);
765 const uint8_t *Start =
Data;
766 if (!LoadFuncsToBeUsed) {
775 for (
auto Name : FuncsToUse) {
783 for (
auto Name : FuncsToUse)
794 assert(OrderedFuncOffsets.get() &&
795 "func offset table should always be sorted in CS profile");
797 for (
const auto &NameOffset : *OrderedFuncOffsets) {
798 const auto &FContext = NameOffset.first;
799 auto FName = FContext.
getName();
803 if ((
useMD5() && FuncGuidsToUse.
count(std::stoull(FName.data()))) ||
804 (!
useMD5() && (FuncsToUse.count(FName) ||
806 if (!CommonContext || !CommonContext->
IsPrefixOf(FContext))
807 CommonContext = &FContext;
810 if (CommonContext == &FContext ||
811 (CommonContext && CommonContext->
IsPrefixOf(FContext))) {
814 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
815 assert(FuncProfileAddr <
End &&
"out of LBRProfile section");
822 for (
auto Name : FuncsToUse) {
824 auto iter = FuncOffsetTable.find(
StringRef(GUID));
825 if (iter == FuncOffsetTable.end())
827 const uint8_t *FuncProfileAddr = Start + iter->second;
828 assert(FuncProfileAddr <
End &&
"out of LBRProfile section");
833 for (
auto NameOffset : FuncOffsetTable) {
835 auto FuncName = FContext.
getName();
836 if (!FuncsToUse.count(FuncName) &&
839 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
840 assert(FuncProfileAddr <
End &&
"out of LBRProfile section");
849 "Cannot have both context-sensitive and regular profile");
851 "Section flag should be consistent with actual profile");
857 ProfSymList = std::make_unique<ProfileSymbolList>();
859 if (std::error_code EC = ProfSymList->read(
Data,
End -
Data))
866 std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
867 const uint8_t *SecStart,
const uint64_t SecSize,
868 const uint8_t *&DecompressBuf,
uint64_t &DecompressBufSize) {
870 End = SecStart + SecSize;
871 auto DecompressSize = readNumber<uint64_t>();
872 if (std::error_code EC = DecompressSize.getError())
874 DecompressBufSize = *DecompressSize;
876 auto CompressSize = readNumber<uint64_t>();
877 if (std::error_code EC = CompressSize.getError())
883 StringRef CompressedStrings(
reinterpret_cast<const char *
>(
Data),
886 size_t UCSize = DecompressBufSize;
891 DecompressBuf =
reinterpret_cast<const uint8_t *
>(
Buffer);
896 const uint8_t *BufStart =
897 reinterpret_cast<const uint8_t *
>(
Buffer->getBufferStart());
899 for (
auto &Entry : SecHdrTable) {
908 const uint8_t *SecStart = BufStart + Entry.Offset;
917 const uint8_t *DecompressBuf;
919 if (std::error_code EC = decompressSection(
920 SecStart, SecSize, DecompressBuf, DecompressBufSize))
922 SecStart = DecompressBuf;
923 SecSize = DecompressBufSize;
926 if (std::error_code EC = readOneSection(SecStart, SecSize, Entry))
928 if (
Data != SecStart + SecSize)
933 Data = BufStart + Entry.Offset;
934 End = BufStart +
Buffer->getBufferSize();
947 std::vector<uint64_t> OffsetsToUse;
948 if (!LoadFuncsToBeUsed) {
950 for (
auto FuncEntry : FuncOffsetTable) {
951 OffsetsToUse.push_back(FuncEntry.second);
955 for (
auto Name : FuncsToUse) {
957 auto iter = FuncOffsetTable.find(
StringRef(GUID));
958 if (iter == FuncOffsetTable.end())
960 OffsetsToUse.push_back(iter->second);
964 for (
auto Offset : OffsetsToUse) {
965 const uint8_t *SavedData =
Data;
967 reinterpret_cast<const uint8_t *
>(
Buffer->getBufferStart()) +
975 std::error_code SampleProfileReaderRawBinary::verifySPMagic(
uint64_t Magic) {
981 std::error_code SampleProfileReaderExtBinary::verifySPMagic(
uint64_t Magic) {
988 SampleProfileReaderCompactBinary::verifySPMagic(
uint64_t Magic) {
995 auto Size = readNumber<uint32_t>();
996 if (std::error_code EC = Size.getError())
1001 if (std::error_code EC =
Name.getError())
1010 auto Size = readNumber<uint64_t>();
1011 if (std::error_code EC = Size.getError())
1013 MD5StringBuf = std::make_unique<std::vector<std::string>>();
1014 MD5StringBuf->reserve(*Size);
1015 if (FixedLengthMD5) {
1022 MD5NameMemStart =
Data;
1028 auto FID = readNumber<uint64_t>();
1029 if (std::error_code EC = FID.getError())
1034 NameTable.push_back(MD5StringBuf->back());
1041 return readMD5NameTable();
1050 auto Size = readNumber<uint32_t>();
1051 if (std::error_code EC = Size.getError())
1054 std::vector<SampleContextFrameVector> *PNameVec =
1055 new std::vector<SampleContextFrameVector>();
1056 PNameVec->reserve(*Size);
1059 auto ContextSize = readNumber<uint32_t>();
1060 if (std::error_code EC = ContextSize.getError())
1062 for (
uint32_t J = 0; J < *ContextSize; ++J) {
1064 if (std::error_code EC = FName.getError())
1066 auto LineOffset = readNumber<uint64_t>();
1067 if (std::error_code EC = LineOffset.getError())
1071 return std::error_code();
1073 auto Discriminator = readNumber<uint64_t>();
1074 if (std::error_code EC = Discriminator.getError())
1077 PNameVec->back().emplace_back(
1078 FName.get(),
LineLocation(LineOffset.get(), Discriminator.get()));
1083 CSNameTable.reset(PNameVec);
1093 auto Checksum = readNumber<uint64_t>();
1094 if (std::error_code EC = Checksum.getError())
1100 if (ProfileHasAttribute) {
1102 if (std::error_code EC =
Attributes.getError())
1110 auto NumCallsites = readNumber<uint32_t>();
1111 if (std::error_code EC = NumCallsites.getError())
1114 for (
uint32_t J = 0; J < *NumCallsites; ++J) {
1115 auto LineOffset = readNumber<uint64_t>();
1116 if (std::error_code EC = LineOffset.getError())
1119 auto Discriminator = readNumber<uint64_t>();
1120 if (std::error_code EC = Discriminator.getError())
1124 if (std::error_code EC = FContext.getError())
1132 *Discriminator))[std::string(FContext.get().getName())]);
1134 if (std::error_code EC =
1135 readFuncMetadata(ProfileHasAttribute, CalleeProfile))
1148 if (std::error_code EC = FContext.getError())
1151 auto It =
Profiles.find(*FContext);
1153 FProfile = &It->second;
1155 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
1163 std::error_code SampleProfileReaderCompactBinary::readNameTable() {
1164 auto Size = readNumber<uint64_t>();
1165 if (std::error_code EC = Size.getError())
1169 auto FID = readNumber<uint64_t>();
1170 if (std::error_code EC = FID.getError())
1180 auto Type = readUnencodedNumber<uint64_t>();
1181 if (std::error_code EC =
Type.getError())
1183 Entry.Type =
static_cast<SecType>(*Type);
1185 auto Flags = readUnencodedNumber<uint64_t>();
1186 if (std::error_code EC = Flags.getError())
1188 Entry.Flags = *Flags;
1190 auto Offset = readUnencodedNumber<uint64_t>();
1191 if (std::error_code EC = Offset.getError())
1193 Entry.Offset = *Offset;
1195 auto Size = readUnencodedNumber<uint64_t>();
1196 if (std::error_code EC = Size.getError())
1200 Entry.LayoutIndex = Idx;
1201 SecHdrTable.push_back(
std::move(Entry));
1206 auto EntryNum = readUnencodedNumber<uint64_t>();
1207 if (std::error_code EC = EntryNum.getError())
1211 if (std::error_code EC = readSecHdrTableEntry(
i))
1218 const uint8_t *BufStart =
1219 reinterpret_cast<const uint8_t *
>(
Buffer->getBufferStart());
1221 End = BufStart +
Buffer->getBufferSize();
1226 if (std::error_code EC = readSecHdrTable())
1234 for (
auto &Entry : SecHdrTable) {
1235 if (Entry.Type ==
Type)
1248 for (
auto &Entry : SecHdrTable) {
1249 FileSize =
std::max(Entry.Offset + Entry.Size, FileSize);
1257 Flags.append(
"{compressed,");
1262 Flags.append(
"flat,");
1264 switch (Entry.Type) {
1267 Flags.append(
"fixlenmd5,");
1269 Flags.append(
"md5,");
1271 Flags.append(
"uniq,");
1275 Flags.append(
"partial,");
1277 Flags.append(
"context,");
1279 Flags.append(
"preInlined,");
1281 Flags.append(
"fs-discriminator,");
1285 Flags.append(
"ordered,");
1289 Flags.append(
"probe,");
1291 Flags.append(
"attr,");
1296 char &
last = Flags.back();
1306 for (
auto &Entry : SecHdrTable) {
1307 OS <<
getSecName(Entry.Type) <<
" - Offset: " << Entry.Offset
1308 <<
", Size: " << Entry.Size <<
", Flags: " <<
getSecFlagsStr(Entry)
1311 TotalSecsSize += Entry.Size;
1315 "Size of 'header + sections' doesn't match the total size of profile");
1318 OS <<
"Total Sections Size: " << TotalSecsSize <<
"\n";
1319 OS <<
"File Size: " << getFileSize() <<
"\n";
1325 auto Magic = readNumber<uint64_t>();
1326 if (std::error_code EC =
Magic.getError())
1328 else if (std::error_code EC = verifySPMagic(*
Magic))
1332 auto Version = readNumber<uint64_t>();
1333 if (std::error_code EC =
Version.getError())
1342 Data =
reinterpret_cast<const uint8_t *
>(
Buffer->getBufferStart());
1356 std::error_code SampleProfileReaderCompactBinary::readHeader() {
1358 if (std::error_code EC = readFuncOffsetTable())
1363 std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() {
1364 auto TableOffset = readUnencodedNumber<uint64_t>();
1365 if (std::error_code EC = TableOffset.getError())
1368 const uint8_t *SavedData =
Data;
1369 const uint8_t *TableStart =
1370 reinterpret_cast<const uint8_t *
>(
Buffer->getBufferStart()) +
1374 auto Size = readNumber<uint64_t>();
1375 if (std::error_code EC = Size.getError())
1378 FuncOffsetTable.reserve(*Size);
1381 if (std::error_code EC = FName.getError())
1384 auto Offset = readNumber<uint64_t>();
1385 if (std::error_code EC = Offset.getError())
1388 FuncOffsetTable[*FName] = *Offset;
1404 std::error_code SampleProfileReaderBinary::readSummaryEntry(
1405 std::vector<ProfileSummaryEntry> &Entries) {
1406 auto Cutoff = readNumber<uint64_t>();
1407 if (std::error_code EC = Cutoff.getError())
1410 auto MinBlockCount = readNumber<uint64_t>();
1411 if (std::error_code EC = MinBlockCount.getError())
1414 auto NumBlocks = readNumber<uint64_t>();
1415 if (std::error_code EC = NumBlocks.getError())
1418 Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks);
1423 auto TotalCount = readNumber<uint64_t>();
1424 if (std::error_code EC = TotalCount.getError())
1427 auto MaxBlockCount = readNumber<uint64_t>();
1428 if (std::error_code EC = MaxBlockCount.getError())
1431 auto MaxFunctionCount = readNumber<uint64_t>();
1432 if (std::error_code EC = MaxFunctionCount.getError())
1435 auto NumBlocks = readNumber<uint64_t>();
1436 if (std::error_code EC = NumBlocks.getError())
1439 auto NumFunctions = readNumber<uint64_t>();
1440 if (std::error_code EC = NumFunctions.getError())
1443 auto NumSummaryEntries = readNumber<uint64_t>();
1444 if (std::error_code EC = NumSummaryEntries.getError())
1447 std::vector<ProfileSummaryEntry> Entries;
1448 for (
unsigned i = 0;
i < *NumSummaryEntries;
i++) {
1449 std::error_code EC = readSummaryEntry(Entries);
1453 Summary = std::make_unique<ProfileSummary>(
1455 *MaxFunctionCount, *NumBlocks, *NumFunctions);
1461 const uint8_t *
Data =
1462 reinterpret_cast<const uint8_t *
>(
Buffer.getBufferStart());
1468 const uint8_t *
Data =
1469 reinterpret_cast<const uint8_t *
>(
Buffer.getBufferStart());
1475 const uint8_t *
Data =
1476 reinterpret_cast<const uint8_t *
>(
Buffer.getBufferStart());
1483 if (!GcovBuffer.readInt(dummy))
1492 return static_cast<T>(Val);
1493 }
else if (
sizeof(
T) <=
sizeof(
uint64_t)) {
1496 return static_cast<T>(Val);
1506 if (!GcovBuffer.readString(Str))
1513 if (!GcovBuffer.readGCDAFormat())
1519 if (!GcovBuffer.readGCOVVersion(version))
1526 if (std::error_code EC = skipNextWord())
1534 if (!GcovBuffer.readInt(
Tag))
1540 if (std::error_code EC = skipNextWord())
1547 if (std::error_code EC = readSectionTag(GCOVTagAFDOFileNames))
1551 if (!GcovBuffer.readInt(Size))
1556 if (!GcovBuffer.readString(Str))
1558 Names.push_back(std::string(Str));
1565 if (std::error_code EC = readSectionTag(GCOVTagAFDOFunction))
1569 if (!GcovBuffer.readInt(NumFunctions))
1574 if (std::error_code EC = readOneFunctionProfile(Stack,
true, 0))
1584 if (InlineStack.size() == 0)
1585 if (!GcovBuffer.readInt64(HeadCount))
1589 if (!GcovBuffer.readInt(NameIdx))
1595 if (!GcovBuffer.readInt(NumPosCounts))
1599 if (!GcovBuffer.readInt(NumCallsites))
1603 if (InlineStack.size() == 0) {
1619 uint32_t LineOffset = Offset >> 16;
1620 uint32_t Discriminator = Offset & 0xffff;
1628 if (!GcovBuffer.readInt(Offset))
1632 if (!GcovBuffer.readInt(NumTargets))
1636 if (!GcovBuffer.readInt64(Count))
1642 uint32_t LineOffset = Offset >> 16;
1643 uint32_t Discriminator = Offset & 0xffff;
1646 NewStack.push_back(FProfile);
1651 for (
auto CallerProfile : NewStack)
1652 CallerProfile->addTotalSamples(Count);
1661 for (
uint32_t J = 0; J < NumTargets; J++) {
1663 if (!GcovBuffer.readInt(HistVal))
1670 if (!GcovBuffer.readInt64(TargetIdx))
1675 if (!GcovBuffer.readInt64(TargetCount))
1680 TargetName, TargetCount);
1691 if (!GcovBuffer.readInt(Offset))
1694 NewStack.push_back(FProfile);
1696 if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset))
1714 if (std::error_code EC = readFunctionProfiles())
1722 return Magic ==
"adcg*704";
1728 if (Reader.useMD5()) {
1730 Reader.getBuffer()->getBufferIdentifier(),
1731 "Profile data remapping cannot be applied to profile data "
1732 "in compact format (original mangled names are not available).",
1739 assert(Remappings &&
"should be initialized while creating remapper");
1740 for (
auto &Sample : Reader.getProfiles()) {
1742 Sample.second.findAllNames(NamesInSample);
1743 for (
auto &
Name : NamesInSample)
1744 if (
auto Key = Remappings->insert(
Name))
1748 RemappingApplied =
true;
1753 if (
auto Key = Remappings->lookup(Fname))
1754 return NameMap.lookup(
Key);
1764 if (std::error_code EC = BufferOrErr.getError())
1789 const std::string RemapFilename) {
1791 if (std::error_code EC = BufferOrError.getError())
1793 return create(BufferOrError.get(),
C,
P, RemapFilename);
1811 if (std::error_code EC = BufferOrError.getError())
1813 return create(BufferOrError.get(), Reader,
C);
1830 auto Remappings = std::make_unique<SymbolRemappingReader>();
1831 if (
Error E = Remappings->read(*
B)) {
1841 return std::make_unique<SampleProfileReaderItaniumRemapper>(
1859 const std::string RemapFilename) {
1860 std::unique_ptr<SampleProfileReader> Reader;
1874 if (!RemapFilename.empty()) {
1877 if (std::error_code EC = ReaderOrErr.getError()) {
1878 std::string
Msg =
"Could not create remapper: " + EC.message();
1882 Reader->Remapper =
std::move(ReaderOrErr.get());
1885 if (std::error_code EC = Reader->readHeader()) {
1889 Reader->setDiscriminatorMaskedBitFrom(
P);