LLVM 19.0.0git
SampleProfReader.cpp
Go to the documentation of this file.
1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that reads LLVM sample profiles. It
10// supports three file formats: text, binary and gcov.
11//
12// The textual representation is useful for debugging and testing purposes. The
13// binary representation is more compact, resulting in smaller file sizes.
14//
15// The gcov encoding is the one generated by GCC's AutoFDO profile creation
16// tool (https://github.com/google/autofdo)
17//
18// All three encodings can be used interchangeably as an input sample profile.
19//
20//===----------------------------------------------------------------------===//
21
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/IR/Module.h"
33#include "llvm/Support/JSON.h"
34#include "llvm/Support/LEB128.h"
36#include "llvm/Support/MD5.h"
40#include <algorithm>
41#include <cstddef>
42#include <cstdint>
43#include <limits>
44#include <memory>
45#include <system_error>
46#include <vector>
47
48using namespace llvm;
49using namespace sampleprof;
50
51#define DEBUG_TYPE "samplepgo-reader"
52
53// This internal option specifies if the profile uses FS discriminators.
54// It only applies to text, and binary format profiles.
55// For ext-binary format profiles, the flag is set in the summary.
57 "profile-isfs", cl::Hidden, cl::init(false),
58 cl::desc("Profile uses flow sensitive discriminators"));
59
60/// Dump the function profile for \p FName.
61///
62/// \param FContext Name + context of the function to print.
63/// \param OS Stream to emit the output to.
65 raw_ostream &OS) {
66 OS << "Function: " << FS.getContext().toString() << ": " << FS;
67}
68
69/// Dump all the function profiles found on stream \p OS.
71 std::vector<NameFunctionSamples> V;
73 for (const auto &I : V)
74 dumpFunctionProfile(*I.second, OS);
75}
76
78 json::OStream &JOS, bool TopLevel = false) {
79 auto DumpBody = [&](const BodySampleMap &BodySamples) {
80 for (const auto &I : BodySamples) {
81 const LineLocation &Loc = I.first;
82 const SampleRecord &Sample = I.second;
83 JOS.object([&] {
84 JOS.attribute("line", Loc.LineOffset);
85 if (Loc.Discriminator)
86 JOS.attribute("discriminator", Loc.Discriminator);
87 JOS.attribute("samples", Sample.getSamples());
88
89 auto CallTargets = Sample.getSortedCallTargets();
90 if (!CallTargets.empty()) {
91 JOS.attributeArray("calls", [&] {
92 for (const auto &J : CallTargets) {
93 JOS.object([&] {
94 JOS.attribute("function", J.first.str());
95 JOS.attribute("samples", J.second);
96 });
97 }
98 });
99 }
100 });
101 }
102 };
103
104 auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {
105 for (const auto &I : CallsiteSamples)
106 for (const auto &FS : I.second) {
107 const LineLocation &Loc = I.first;
108 const FunctionSamples &CalleeSamples = FS.second;
109 JOS.object([&] {
110 JOS.attribute("line", Loc.LineOffset);
111 if (Loc.Discriminator)
112 JOS.attribute("discriminator", Loc.Discriminator);
113 JOS.attributeArray(
114 "samples", [&] { dumpFunctionProfileJson(CalleeSamples, JOS); });
115 });
116 }
117 };
118
119 JOS.object([&] {
120 JOS.attribute("name", S.getFunction().str());
121 JOS.attribute("total", S.getTotalSamples());
122 if (TopLevel)
123 JOS.attribute("head", S.getHeadSamples());
124
125 const auto &BodySamples = S.getBodySamples();
126 if (!BodySamples.empty())
127 JOS.attributeArray("body", [&] { DumpBody(BodySamples); });
128
129 const auto &CallsiteSamples = S.getCallsiteSamples();
130 if (!CallsiteSamples.empty())
131 JOS.attributeArray("callsites",
132 [&] { DumpCallsiteSamples(CallsiteSamples); });
133 });
134}
135
136/// Dump all the function profiles found on stream \p OS in the JSON format.
138 std::vector<NameFunctionSamples> V;
140 json::OStream JOS(OS, 2);
141 JOS.arrayBegin();
142 for (const auto &F : V)
143 dumpFunctionProfileJson(*F.second, JOS, true);
144 JOS.arrayEnd();
145
146 // Emit a newline character at the end as json::OStream doesn't emit one.
147 OS << "\n";
148}
149
150/// Parse \p Input as function head.
151///
152/// Parse one line of \p Input, and update function name in \p FName,
153/// function's total sample count in \p NumSamples, function's entry
154/// count in \p NumHeadSamples.
155///
156/// \returns true if parsing is successful.
157static bool ParseHead(const StringRef &Input, StringRef &FName,
158 uint64_t &NumSamples, uint64_t &NumHeadSamples) {
159 if (Input[0] == ' ')
160 return false;
161 size_t n2 = Input.rfind(':');
162 size_t n1 = Input.rfind(':', n2 - 1);
163 FName = Input.substr(0, n1);
164 if (Input.substr(n1 + 1, n2 - n1 - 1).getAsInteger(10, NumSamples))
165 return false;
166 if (Input.substr(n2 + 1).getAsInteger(10, NumHeadSamples))
167 return false;
168 return true;
169}
170
171/// Returns true if line offset \p L is legal (only has 16 bits).
172static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
173
174/// Parse \p Input that contains metadata.
175/// Possible metadata:
176/// - CFG Checksum information:
177/// !CFGChecksum: 12345
178/// - CFG Checksum information:
179/// !Attributes: 1
180/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
181static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
182 uint32_t &Attributes) {
183 if (Input.starts_with("!CFGChecksum:")) {
184 StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim();
185 return !CFGInfo.getAsInteger(10, FunctionHash);
186 }
187
188 if (Input.starts_with("!Attributes:")) {
189 StringRef Attrib = Input.substr(strlen("!Attributes:")).trim();
190 return !Attrib.getAsInteger(10, Attributes);
191 }
192
193 return false;
194}
195
196enum class LineType {
199 Metadata,
200};
201
202/// Parse \p Input as line sample.
203///
204/// \param Input input line.
205/// \param LineTy Type of this line.
206/// \param Depth the depth of the inline stack.
207/// \param NumSamples total samples of the line/inlined callsite.
208/// \param LineOffset line offset to the start of the function.
209/// \param Discriminator discriminator of the line.
210/// \param TargetCountMap map from indirect call target to count.
211/// \param FunctionHash the function's CFG hash, used by pseudo probe.
212///
213/// returns true if parsing is successful.
214static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
215 uint64_t &NumSamples, uint32_t &LineOffset,
216 uint32_t &Discriminator, StringRef &CalleeName,
217 DenseMap<StringRef, uint64_t> &TargetCountMap,
218 uint64_t &FunctionHash, uint32_t &Attributes) {
219 for (Depth = 0; Input[Depth] == ' '; Depth++)
220 ;
221 if (Depth == 0)
222 return false;
223
224 if (Input[Depth] == '!') {
225 LineTy = LineType::Metadata;
226 return parseMetadata(Input.substr(Depth), FunctionHash, Attributes);
227 }
228
229 size_t n1 = Input.find(':');
230 StringRef Loc = Input.substr(Depth, n1 - Depth);
231 size_t n2 = Loc.find('.');
232 if (n2 == StringRef::npos) {
233 if (Loc.getAsInteger(10, LineOffset) || !isOffsetLegal(LineOffset))
234 return false;
235 Discriminator = 0;
236 } else {
237 if (Loc.substr(0, n2).getAsInteger(10, LineOffset))
238 return false;
239 if (Loc.substr(n2 + 1).getAsInteger(10, Discriminator))
240 return false;
241 }
242
243 StringRef Rest = Input.substr(n1 + 2);
244 if (isDigit(Rest[0])) {
245 LineTy = LineType::BodyProfile;
246 size_t n3 = Rest.find(' ');
247 if (n3 == StringRef::npos) {
248 if (Rest.getAsInteger(10, NumSamples))
249 return false;
250 } else {
251 if (Rest.substr(0, n3).getAsInteger(10, NumSamples))
252 return false;
253 }
254 // Find call targets and their sample counts.
255 // Note: In some cases, there are symbols in the profile which are not
256 // mangled. To accommodate such cases, use colon + integer pairs as the
257 // anchor points.
258 // An example:
259 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
260 // ":1000" and ":437" are used as anchor points so the string above will
261 // be interpreted as
262 // target: _M_construct<char *>
263 // count: 1000
264 // target: string_view<std::allocator<char> >
265 // count: 437
266 while (n3 != StringRef::npos) {
267 n3 += Rest.substr(n3).find_first_not_of(' ');
268 Rest = Rest.substr(n3);
269 n3 = Rest.find_first_of(':');
270 if (n3 == StringRef::npos || n3 == 0)
271 return false;
272
274 uint64_t count, n4;
275 while (true) {
276 // Get the segment after the current colon.
277 StringRef AfterColon = Rest.substr(n3 + 1);
278 // Get the target symbol before the current colon.
279 Target = Rest.substr(0, n3);
280 // Check if the word after the current colon is an integer.
281 n4 = AfterColon.find_first_of(' ');
282 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
283 StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1);
284 if (!WordAfterColon.getAsInteger(10, count))
285 break;
286
287 // Try to find the next colon.
288 uint64_t n5 = AfterColon.find_first_of(':');
289 if (n5 == StringRef::npos)
290 return false;
291 n3 += n5 + 1;
292 }
293
294 // An anchor point is found. Save the {target, count} pair
295 TargetCountMap[Target] = count;
296 if (n4 == Rest.size())
297 break;
298 // Change n3 to the next blank space after colon + integer pair.
299 n3 = n4;
300 }
301 } else {
302 LineTy = LineType::CallSiteProfile;
303 size_t n3 = Rest.find_last_of(':');
304 CalleeName = Rest.substr(0, n3);
305 if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples))
306 return false;
307 }
308 return true;
309}
310
311/// Load samples from a text file.
312///
313/// See the documentation at the top of the file for an explanation of
314/// the expected format.
315///
316/// \returns true if the file was loaded successfully, false otherwise.
318 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
320
321 InlineCallStack InlineStack;
322 uint32_t TopLevelProbeProfileCount = 0;
323
324 // DepthMetadata tracks whether we have processed metadata for the current
325 // top-level or nested function profile.
326 uint32_t DepthMetadata = 0;
327
330 for (; !LineIt.is_at_eof(); ++LineIt) {
331 size_t pos = LineIt->find_first_not_of(' ');
332 if (pos == LineIt->npos || (*LineIt)[pos] == '#')
333 continue;
334 // Read the header of each function.
335 //
336 // Note that for function identifiers we are actually expecting
337 // mangled names, but we may not always get them. This happens when
338 // the compiler decides not to emit the function (e.g., it was inlined
339 // and removed). In this case, the binary will not have the linkage
340 // name for the function, so the profiler will emit the function's
341 // unmangled name, which may contain characters like ':' and '>' in its
342 // name (member functions, templates, etc).
343 //
344 // The only requirement we place on the identifier, then, is that it
345 // should not begin with a number.
346 if ((*LineIt)[0] != ' ') {
347 uint64_t NumSamples, NumHeadSamples;
348 StringRef FName;
349 if (!ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) {
350 reportError(LineIt.line_number(),
351 "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
353 }
354 DepthMetadata = 0;
355 SampleContext FContext(FName, CSNameTable);
356 if (FContext.hasContext())
358 FunctionSamples &FProfile = Profiles.Create(FContext);
359 MergeResult(Result, FProfile.addTotalSamples(NumSamples));
360 MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples));
361 InlineStack.clear();
362 InlineStack.push_back(&FProfile);
363 } else {
364 uint64_t NumSamples;
365 StringRef FName;
366 DenseMap<StringRef, uint64_t> TargetCountMap;
367 uint32_t Depth, LineOffset, Discriminator;
368 LineType LineTy;
369 uint64_t FunctionHash = 0;
371 if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset,
372 Discriminator, FName, TargetCountMap, FunctionHash,
373 Attributes)) {
374 reportError(LineIt.line_number(),
375 "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
376 *LineIt);
378 }
379 if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
380 // Metadata must be put at the end of a function profile.
381 reportError(LineIt.line_number(),
382 "Found non-metadata after metadata: " + *LineIt);
384 }
385
386 // Here we handle FS discriminators.
387 Discriminator &= getDiscriminatorMask();
388
389 while (InlineStack.size() > Depth) {
390 InlineStack.pop_back();
391 }
392 switch (LineTy) {
393 case LineType::CallSiteProfile: {
394 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
395 LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
396 FSamples.setFunction(FunctionId(FName));
397 MergeResult(Result, FSamples.addTotalSamples(NumSamples));
398 InlineStack.push_back(&FSamples);
399 DepthMetadata = 0;
400 break;
401 }
402 case LineType::BodyProfile: {
403 while (InlineStack.size() > Depth) {
404 InlineStack.pop_back();
405 }
406 FunctionSamples &FProfile = *InlineStack.back();
407 for (const auto &name_count : TargetCountMap) {
408 MergeResult(Result, FProfile.addCalledTargetSamples(
409 LineOffset, Discriminator,
410 FunctionId(name_count.first),
411 name_count.second));
412 }
413 MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator,
414 NumSamples));
415 break;
416 }
417 case LineType::Metadata: {
418 FunctionSamples &FProfile = *InlineStack.back();
419 if (FunctionHash) {
420 FProfile.setFunctionHash(FunctionHash);
421 if (Depth == 1)
422 ++TopLevelProbeProfileCount;
423 }
426 ProfileIsPreInlined = true;
427 DepthMetadata = Depth;
428 break;
429 }
430 }
431 }
432 }
433
434 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
435 "Cannot have both context-sensitive and regular profile");
437 assert((TopLevelProbeProfileCount == 0 ||
438 TopLevelProbeProfileCount == Profiles.size()) &&
439 "Cannot have both probe-based profiles and regular profiles");
440 ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
444
445 if (Result == sampleprof_error::success)
447
448 return Result;
449}
450
452 bool result = false;
453
454 // Check that the first non-comment line is a valid function header.
455 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
456 if (!LineIt.is_at_eof()) {
457 if ((*LineIt)[0] != ' ') {
458 uint64_t NumSamples, NumHeadSamples;
459 StringRef FName;
460 result = ParseHead(*LineIt, FName, NumSamples, NumHeadSamples);
461 }
462 }
463
464 return result;
465}
466
468 unsigned NumBytesRead = 0;
469 uint64_t Val = decodeULEB128(Data, &NumBytesRead);
470
471 if (Val > std::numeric_limits<T>::max()) {
472 std::error_code EC = sampleprof_error::malformed;
473 reportError(0, EC.message());
474 return EC;
475 } else if (Data + NumBytesRead > End) {
476 std::error_code EC = sampleprof_error::truncated;
477 reportError(0, EC.message());
478 return EC;
479 }
480
481 Data += NumBytesRead;
482 return static_cast<T>(Val);
483}
484
486 StringRef Str(reinterpret_cast<const char *>(Data));
487 if (Data + Str.size() + 1 > End) {
488 std::error_code EC = sampleprof_error::truncated;
489 reportError(0, EC.message());
490 return EC;
491 }
492
493 Data += Str.size() + 1;
494 return Str;
495}
496
497template <typename T>
499 if (Data + sizeof(T) > End) {
500 std::error_code EC = sampleprof_error::truncated;
501 reportError(0, EC.message());
502 return EC;
503 }
504
505 using namespace support;
506 T Val = endian::readNext<T, llvm::endianness::little>(Data);
507 return Val;
508}
509
510template <typename T>
512 auto Idx = readNumber<size_t>();
513 if (std::error_code EC = Idx.getError())
514 return EC;
515 if (*Idx >= Table.size())
517 return *Idx;
518}
519
523 if (std::error_code EC = Idx.getError())
524 return EC;
525 if (RetIdx)
526 *RetIdx = *Idx;
527 return NameTable[*Idx];
528}
529
532 auto ContextIdx = readNumber<size_t>();
533 if (std::error_code EC = ContextIdx.getError())
534 return EC;
535 if (*ContextIdx >= CSNameTable.size())
537 if (RetIdx)
538 *RetIdx = *ContextIdx;
539 return CSNameTable[*ContextIdx];
540}
541
544 SampleContext Context;
545 size_t Idx;
546 if (ProfileIsCS) {
547 auto FContext(readContextFromTable(&Idx));
548 if (std::error_code EC = FContext.getError())
549 return EC;
550 Context = SampleContext(*FContext);
551 } else {
552 auto FName(readStringFromTable(&Idx));
553 if (std::error_code EC = FName.getError())
554 return EC;
555 Context = SampleContext(*FName);
556 }
557 // Since MD5SampleContextStart may point to the profile's file data, need to
558 // make sure it is reading the same value on big endian CPU.
560 // Lazy computing of hash value, write back to the table to cache it. Only
561 // compute the context's hash value if it is being referenced for the first
562 // time.
563 if (Hash == 0) {
565 Hash = Context.getHashCode();
567 }
568 return std::make_pair(Context, Hash);
569}
570
571std::error_code
573 auto NumSamples = readNumber<uint64_t>();
574 if (std::error_code EC = NumSamples.getError())
575 return EC;
576 FProfile.addTotalSamples(*NumSamples);
577
578 // Read the samples in the body.
579 auto NumRecords = readNumber<uint32_t>();
580 if (std::error_code EC = NumRecords.getError())
581 return EC;
582
583 for (uint32_t I = 0; I < *NumRecords; ++I) {
584 auto LineOffset = readNumber<uint64_t>();
585 if (std::error_code EC = LineOffset.getError())
586 return EC;
587
588 if (!isOffsetLegal(*LineOffset)) {
589 return std::error_code();
590 }
591
592 auto Discriminator = readNumber<uint64_t>();
593 if (std::error_code EC = Discriminator.getError())
594 return EC;
595
596 auto NumSamples = readNumber<uint64_t>();
597 if (std::error_code EC = NumSamples.getError())
598 return EC;
599
600 auto NumCalls = readNumber<uint32_t>();
601 if (std::error_code EC = NumCalls.getError())
602 return EC;
603
604 // Here we handle FS discriminators:
605 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
606
607 for (uint32_t J = 0; J < *NumCalls; ++J) {
608 auto CalledFunction(readStringFromTable());
609 if (std::error_code EC = CalledFunction.getError())
610 return EC;
611
612 auto CalledFunctionSamples = readNumber<uint64_t>();
613 if (std::error_code EC = CalledFunctionSamples.getError())
614 return EC;
615
616 FProfile.addCalledTargetSamples(*LineOffset, DiscriminatorVal,
617 *CalledFunction, *CalledFunctionSamples);
618 }
619
620 FProfile.addBodySamples(*LineOffset, DiscriminatorVal, *NumSamples);
621 }
622
623 // Read all the samples for inlined function calls.
624 auto NumCallsites = readNumber<uint32_t>();
625 if (std::error_code EC = NumCallsites.getError())
626 return EC;
627
628 for (uint32_t J = 0; J < *NumCallsites; ++J) {
629 auto LineOffset = readNumber<uint64_t>();
630 if (std::error_code EC = LineOffset.getError())
631 return EC;
632
633 auto Discriminator = readNumber<uint64_t>();
634 if (std::error_code EC = Discriminator.getError())
635 return EC;
636
637 auto FName(readStringFromTable());
638 if (std::error_code EC = FName.getError())
639 return EC;
640
641 // Here we handle FS discriminators:
642 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
643
644 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
645 LineLocation(*LineOffset, DiscriminatorVal))[*FName];
646 CalleeProfile.setFunction(*FName);
647 if (std::error_code EC = readProfile(CalleeProfile))
648 return EC;
649 }
650
652}
653
654std::error_code
656 Data = Start;
657 auto NumHeadSamples = readNumber<uint64_t>();
658 if (std::error_code EC = NumHeadSamples.getError())
659 return EC;
660
661 auto FContextHash(readSampleContextFromTable());
662 if (std::error_code EC = FContextHash.getError())
663 return EC;
664
665 auto &[FContext, Hash] = *FContextHash;
666 // Use the cached hash value for insertion instead of recalculating it.
667 auto Res = Profiles.try_emplace(Hash, FContext, FunctionSamples());
668 FunctionSamples &FProfile = Res.first->second;
669 FProfile.setContext(FContext);
670 FProfile.addHeadSamples(*NumHeadSamples);
671
672 if (FContext.hasContext())
674
675 if (std::error_code EC = readProfile(FProfile))
676 return EC;
678}
679
683 while (Data < End) {
684 if (std::error_code EC = readFuncProfile(Data))
685 return EC;
686 }
687
689}
690
692 const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
693 Data = Start;
694 End = Start + Size;
695 switch (Entry.Type) {
696 case SecProfSummary:
697 if (std::error_code EC = readSummary())
698 return EC;
700 Summary->setPartialProfile(true);
707 break;
708 case SecNameTable: {
709 bool FixedLengthMD5 =
711 bool UseMD5 = hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name);
712 // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire
713 // profile uses MD5 for function name matching in IPO passes.
714 ProfileIsMD5 = ProfileIsMD5 || UseMD5;
717 if (std::error_code EC = readNameTableSec(UseMD5, FixedLengthMD5))
718 return EC;
719 break;
720 }
721 case SecCSNameTable: {
722 if (std::error_code EC = readCSNameTableSec())
723 return EC;
724 break;
725 }
726 case SecLBRProfile:
727 if (std::error_code EC = readFuncProfiles())
728 return EC;
729 break;
731 // If module is absent, we are using LLVM tools, and need to read all
732 // profiles, so skip reading the function offset table.
733 if (!M) {
734 Data = End;
735 } else {
738 "func offset table should always be sorted in CS profile");
739 if (std::error_code EC = readFuncOffsetTable())
740 return EC;
741 }
742 break;
743 case SecFuncMetadata: {
747 bool HasAttribute =
749 if (std::error_code EC = readFuncMetadata(HasAttribute))
750 return EC;
751 break;
752 }
754 if (std::error_code EC = readProfileSymbolList())
755 return EC;
756 break;
757 default:
758 if (std::error_code EC = readCustomSection(Entry))
759 return EC;
760 break;
761 }
763}
764
766 // If profile is CS, the function offset section is expected to consist of
767 // sequences of contexts in pre-order layout
768 // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched
769 // context in the module is found, the profiles of all its callees are
770 // recursively loaded. A list is needed since the order of profiles matters.
771 if (ProfileIsCS)
772 return true;
773
774 // If the profile is MD5, use the map container to lookup functions in
775 // the module. A remapper has no use on MD5 names.
776 if (useMD5())
777 return false;
778
779 // Profile is not MD5 and if a remapper is present, the remapped name of
780 // every function needed to be matched against the module, so use the list
781 // container since each entry is accessed.
782 if (Remapper)
783 return true;
784
785 // Otherwise use the map container for faster lookup.
786 // TODO: If the cardinality of the function offset section is much smaller
787 // than the number of functions in the module, using the list container can
788 // be always faster, but we need to figure out the constant factor to
789 // determine the cutoff.
790 return false;
791}
792
793
795 if (!M)
796 return false;
797 FuncsToUse.clear();
798 for (auto &F : *M)
800 return true;
801}
802
804 // If there are more than one function offset section, the profile associated
805 // with the previous section has to be done reading before next one is read.
806 FuncOffsetTable.clear();
807 FuncOffsetList.clear();
808
809 auto Size = readNumber<uint64_t>();
810 if (std::error_code EC = Size.getError())
811 return EC;
812
813 bool UseFuncOffsetList = useFuncOffsetList();
814 if (UseFuncOffsetList)
815 FuncOffsetList.reserve(*Size);
816 else
817 FuncOffsetTable.reserve(*Size);
818
819 for (uint64_t I = 0; I < *Size; ++I) {
820 auto FContextHash(readSampleContextFromTable());
821 if (std::error_code EC = FContextHash.getError())
822 return EC;
823
824 auto &[FContext, Hash] = *FContextHash;
825 auto Offset = readNumber<uint64_t>();
826 if (std::error_code EC = Offset.getError())
827 return EC;
828
829 if (UseFuncOffsetList)
830 FuncOffsetList.emplace_back(FContext, *Offset);
831 else
832 // Because Porfiles replace existing value with new value if collision
833 // happens, we also use the latest offset so that they are consistent.
834 FuncOffsetTable[Hash] = *Offset;
835 }
836
838}
839
841 // Collect functions used by current module if the Reader has been
842 // given a module.
843 // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName
844 // which will query FunctionSamples::HasUniqSuffix, so it has to be
845 // called after FunctionSamples::HasUniqSuffix is set, i.e. after
846 // NameTable section is read.
847 bool LoadFuncsToBeUsed = collectFuncsFromModule();
848
849 // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all
850 // profiles.
851 const uint8_t *Start = Data;
852 if (!LoadFuncsToBeUsed) {
853 while (Data < End) {
854 if (std::error_code EC = readFuncProfile(Data))
855 return EC;
856 }
857 assert(Data == End && "More data is read than expected");
858 } else {
859 // Load function profiles on demand.
860 if (Remapper) {
861 for (auto Name : FuncsToUse) {
862 Remapper->insert(Name);
863 }
864 }
865
866 if (ProfileIsCS) {
868 DenseSet<uint64_t> FuncGuidsToUse;
869 if (useMD5()) {
870 for (auto Name : FuncsToUse)
871 FuncGuidsToUse.insert(Function::getGUID(Name));
872 }
873
874 // For each function in current module, load all context profiles for
875 // the function as well as their callee contexts which can help profile
876 // guided importing for ThinLTO. This can be achieved by walking
877 // through an ordered context container, where contexts are laid out
878 // as if they were walked in preorder of a context trie. While
879 // traversing the trie, a link to the highest common ancestor node is
880 // kept so that all of its decendants will be loaded.
881 const SampleContext *CommonContext = nullptr;
882 for (const auto &NameOffset : FuncOffsetList) {
883 const auto &FContext = NameOffset.first;
884 FunctionId FName = FContext.getFunction();
885 StringRef FNameString;
886 if (!useMD5())
887 FNameString = FName.stringRef();
888
889 // For function in the current module, keep its farthest ancestor
890 // context. This can be used to load itself and its child and
891 // sibling contexts.
892 if ((useMD5() && FuncGuidsToUse.count(FName.getHashCode())) ||
893 (!useMD5() && (FuncsToUse.count(FNameString) ||
894 (Remapper && Remapper->exist(FNameString))))) {
895 if (!CommonContext || !CommonContext->IsPrefixOf(FContext))
896 CommonContext = &FContext;
897 }
898
899 if (CommonContext == &FContext ||
900 (CommonContext && CommonContext->IsPrefixOf(FContext))) {
901 // Load profile for the current context which originated from
902 // the common ancestor.
903 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
904 if (std::error_code EC = readFuncProfile(FuncProfileAddr))
905 return EC;
906 }
907 }
908 } else if (useMD5()) {
910 for (auto Name : FuncsToUse) {
911 auto GUID = MD5Hash(Name);
912 auto iter = FuncOffsetTable.find(GUID);
913 if (iter == FuncOffsetTable.end())
914 continue;
915 const uint8_t *FuncProfileAddr = Start + iter->second;
916 if (std::error_code EC = readFuncProfile(FuncProfileAddr))
917 return EC;
918 }
919 } else if (Remapper) {
921 for (auto NameOffset : FuncOffsetList) {
922 SampleContext FContext(NameOffset.first);
923 auto FuncName = FContext.getFunction();
924 StringRef FuncNameStr = FuncName.stringRef();
925 if (!FuncsToUse.count(FuncNameStr) && !Remapper->exist(FuncNameStr))
926 continue;
927 const uint8_t *FuncProfileAddr = Start + NameOffset.second;
928 if (std::error_code EC = readFuncProfile(FuncProfileAddr))
929 return EC;
930 }
931 } else {
933 for (auto Name : FuncsToUse) {
934 auto iter = FuncOffsetTable.find(MD5Hash(Name));
935 if (iter == FuncOffsetTable.end())
936 continue;
937 const uint8_t *FuncProfileAddr = Start + iter->second;
938 if (std::error_code EC = readFuncProfile(FuncProfileAddr))
939 return EC;
940 }
941 }
942 Data = End;
943 }
944 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
945 "Cannot have both context-sensitive and regular profile");
947 "Section flag should be consistent with actual profile");
949}
950
952 if (!ProfSymList)
953 ProfSymList = std::make_unique<ProfileSymbolList>();
954
955 if (std::error_code EC = ProfSymList->read(Data, End - Data))
956 return EC;
957
958 Data = End;
960}
961
962std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
963 const uint8_t *SecStart, const uint64_t SecSize,
964 const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
965 Data = SecStart;
966 End = SecStart + SecSize;
967 auto DecompressSize = readNumber<uint64_t>();
968 if (std::error_code EC = DecompressSize.getError())
969 return EC;
970 DecompressBufSize = *DecompressSize;
971
972 auto CompressSize = readNumber<uint64_t>();
973 if (std::error_code EC = CompressSize.getError())
974 return EC;
975
978
979 uint8_t *Buffer = Allocator.Allocate<uint8_t>(DecompressBufSize);
980 size_t UCSize = DecompressBufSize;
982 Buffer, UCSize);
983 if (E)
985 DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
987}
988
990 const uint8_t *BufStart =
991 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
992
993 for (auto &Entry : SecHdrTable) {
994 // Skip empty section.
995 if (!Entry.Size)
996 continue;
997
998 // Skip sections without context when SkipFlatProf is true.
1000 continue;
1001
1002 const uint8_t *SecStart = BufStart + Entry.Offset;
1003 uint64_t SecSize = Entry.Size;
1004
1005 // If the section is compressed, decompress it into a buffer
1006 // DecompressBuf before reading the actual data. The pointee of
1007 // 'Data' will be changed to buffer hold by DecompressBuf
1008 // temporarily when reading the actual data.
1009 bool isCompressed = hasSecFlag(Entry, SecCommonFlags::SecFlagCompress);
1010 if (isCompressed) {
1011 const uint8_t *DecompressBuf;
1012 uint64_t DecompressBufSize;
1013 if (std::error_code EC = decompressSection(
1014 SecStart, SecSize, DecompressBuf, DecompressBufSize))
1015 return EC;
1016 SecStart = DecompressBuf;
1017 SecSize = DecompressBufSize;
1018 }
1019
1020 if (std::error_code EC = readOneSection(SecStart, SecSize, Entry))
1021 return EC;
1022 if (Data != SecStart + SecSize)
1024
1025 // Change the pointee of 'Data' from DecompressBuf to original Buffer.
1026 if (isCompressed) {
1027 Data = BufStart + Entry.Offset;
1028 End = BufStart + Buffer->getBufferSize();
1029 }
1030 }
1031
1033}
1034
1035std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
1036 if (Magic == SPMagic())
1039}
1040
1041std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
1042 if (Magic == SPMagic(SPF_Ext_Binary))
1045}
1046
1048 auto Size = readNumber<size_t>();
1049 if (std::error_code EC = Size.getError())
1050 return EC;
1051
1052 // Normally if useMD5 is true, the name table should have MD5 values, not
1053 // strings, however in the case that ExtBinary profile has multiple name
1054 // tables mixing string and MD5, all of them have to be normalized to use MD5,
1055 // because optimization passes can only handle either type.
1056 bool UseMD5 = useMD5();
1057
1058 NameTable.clear();
1059 NameTable.reserve(*Size);
1060 if (!ProfileIsCS) {
1061 MD5SampleContextTable.clear();
1062 if (UseMD5)
1063 MD5SampleContextTable.reserve(*Size);
1064 else
1065 // If we are using strings, delay MD5 computation since only a portion of
1066 // names are used by top level functions. Use 0 to indicate MD5 value is
1067 // to be calculated as no known string has a MD5 value of 0.
1068 MD5SampleContextTable.resize(*Size);
1069 }
1070 for (size_t I = 0; I < *Size; ++I) {
1071 auto Name(readString());
1072 if (std::error_code EC = Name.getError())
1073 return EC;
1074 if (UseMD5) {
1075 FunctionId FID(*Name);
1076 if (!ProfileIsCS)
1077 MD5SampleContextTable.emplace_back(FID.getHashCode());
1078 NameTable.emplace_back(FID);
1079 } else
1080 NameTable.push_back(FunctionId(*Name));
1081 }
1082 if (!ProfileIsCS)
1085}
1086
1087std::error_code
1089 bool FixedLengthMD5) {
1090 if (FixedLengthMD5) {
1091 if (!IsMD5)
1092 errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";
1093 auto Size = readNumber<size_t>();
1094 if (std::error_code EC = Size.getError())
1095 return EC;
1096
1097 assert(Data + (*Size) * sizeof(uint64_t) == End &&
1098 "Fixed length MD5 name table does not contain specified number of "
1099 "entries");
1100 if (Data + (*Size) * sizeof(uint64_t) > End)
1102
1103 NameTable.clear();
1104 NameTable.reserve(*Size);
1105 for (size_t I = 0; I < *Size; ++I) {
1106 using namespace support;
1107 uint64_t FID = endian::read<uint64_t, endianness::little, unaligned>(
1108 Data + I * sizeof(uint64_t));
1109 NameTable.emplace_back(FunctionId(FID));
1110 }
1111 if (!ProfileIsCS)
1112 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1113 Data = Data + (*Size) * sizeof(uint64_t);
1115 }
1116
1117 if (IsMD5) {
1118 assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");
1119 auto Size = readNumber<size_t>();
1120 if (std::error_code EC = Size.getError())
1121 return EC;
1122
1123 NameTable.clear();
1124 NameTable.reserve(*Size);
1125 if (!ProfileIsCS)
1126 MD5SampleContextTable.resize(*Size);
1127 for (size_t I = 0; I < *Size; ++I) {
1128 auto FID = readNumber<uint64_t>();
1129 if (std::error_code EC = FID.getError())
1130 return EC;
1131 if (!ProfileIsCS)
1133 NameTable.emplace_back(FunctionId(*FID));
1134 }
1135 if (!ProfileIsCS)
1138 }
1139
1141}
1142
1143// Read in the CS name table section, which basically contains a list of context
1144// vectors. Each element of a context vector, aka a frame, refers to the
1145// underlying raw function names that are stored in the name table, as well as
1146// a callsite identifier that only makes sense for non-leaf frames.
1148 auto Size = readNumber<size_t>();
1149 if (std::error_code EC = Size.getError())
1150 return EC;
1151
1152 CSNameTable.clear();
1153 CSNameTable.reserve(*Size);
1154 if (ProfileIsCS) {
1155 // Delay MD5 computation of CS context until they are needed. Use 0 to
1156 // indicate MD5 value is to be calculated as no known string has a MD5
1157 // value of 0.
1158 MD5SampleContextTable.clear();
1159 MD5SampleContextTable.resize(*Size);
1161 }
1162 for (size_t I = 0; I < *Size; ++I) {
1163 CSNameTable.emplace_back(SampleContextFrameVector());
1164 auto ContextSize = readNumber<uint32_t>();
1165 if (std::error_code EC = ContextSize.getError())
1166 return EC;
1167 for (uint32_t J = 0; J < *ContextSize; ++J) {
1168 auto FName(readStringFromTable());
1169 if (std::error_code EC = FName.getError())
1170 return EC;
1171 auto LineOffset = readNumber<uint64_t>();
1172 if (std::error_code EC = LineOffset.getError())
1173 return EC;
1174
1175 if (!isOffsetLegal(*LineOffset))
1176 return std::error_code();
1177
1178 auto Discriminator = readNumber<uint64_t>();
1179 if (std::error_code EC = Discriminator.getError())
1180 return EC;
1181
1182 CSNameTable.back().emplace_back(
1183 FName.get(), LineLocation(LineOffset.get(), Discriminator.get()));
1184 }
1185 }
1186
1188}
1189
1190std::error_code
1192 FunctionSamples *FProfile) {
1193 if (Data < End) {
1194 if (ProfileIsProbeBased) {
1195 auto Checksum = readNumber<uint64_t>();
1196 if (std::error_code EC = Checksum.getError())
1197 return EC;
1198 if (FProfile)
1199 FProfile->setFunctionHash(*Checksum);
1200 }
1201
1202 if (ProfileHasAttribute) {
1203 auto Attributes = readNumber<uint32_t>();
1204 if (std::error_code EC = Attributes.getError())
1205 return EC;
1206 if (FProfile)
1208 }
1209
1210 if (!ProfileIsCS) {
1211 // Read all the attributes for inlined function calls.
1212 auto NumCallsites = readNumber<uint32_t>();
1213 if (std::error_code EC = NumCallsites.getError())
1214 return EC;
1215
1216 for (uint32_t J = 0; J < *NumCallsites; ++J) {
1217 auto LineOffset = readNumber<uint64_t>();
1218 if (std::error_code EC = LineOffset.getError())
1219 return EC;
1220
1221 auto Discriminator = readNumber<uint64_t>();
1222 if (std::error_code EC = Discriminator.getError())
1223 return EC;
1224
1225 auto FContextHash(readSampleContextFromTable());
1226 if (std::error_code EC = FContextHash.getError())
1227 return EC;
1228
1229 auto &[FContext, Hash] = *FContextHash;
1230 FunctionSamples *CalleeProfile = nullptr;
1231 if (FProfile) {
1232 CalleeProfile = const_cast<FunctionSamples *>(
1234 *LineOffset,
1235 *Discriminator))[FContext.getFunction()]);
1236 }
1237 if (std::error_code EC =
1238 readFuncMetadata(ProfileHasAttribute, CalleeProfile))
1239 return EC;
1240 }
1241 }
1242 }
1243
1245}
1246
1247std::error_code
1249 while (Data < End) {
1250 auto FContextHash(readSampleContextFromTable());
1251 if (std::error_code EC = FContextHash.getError())
1252 return EC;
1253 auto &[FContext, Hash] = *FContextHash;
1254 FunctionSamples *FProfile = nullptr;
1255 auto It = Profiles.find(FContext);
1256 if (It != Profiles.end())
1257 FProfile = &It->second;
1258
1259 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
1260 return EC;
1261 }
1262
1263 assert(Data == End && "More data is read than expected");
1265}
1266
1267std::error_code
1269 SecHdrTableEntry Entry;
1270 auto Type = readUnencodedNumber<uint64_t>();
1271 if (std::error_code EC = Type.getError())
1272 return EC;
1273 Entry.Type = static_cast<SecType>(*Type);
1274
1275 auto Flags = readUnencodedNumber<uint64_t>();
1276 if (std::error_code EC = Flags.getError())
1277 return EC;
1278 Entry.Flags = *Flags;
1279
1280 auto Offset = readUnencodedNumber<uint64_t>();
1281 if (std::error_code EC = Offset.getError())
1282 return EC;
1283 Entry.Offset = *Offset;
1284
1285 auto Size = readUnencodedNumber<uint64_t>();
1286 if (std::error_code EC = Size.getError())
1287 return EC;
1288 Entry.Size = *Size;
1289
1290 Entry.LayoutIndex = Idx;
1291 SecHdrTable.push_back(std::move(Entry));
1293}
1294
1296 auto EntryNum = readUnencodedNumber<uint64_t>();
1297 if (std::error_code EC = EntryNum.getError())
1298 return EC;
1299
1300 for (uint64_t i = 0; i < (*EntryNum); i++)
1301 if (std::error_code EC = readSecHdrTableEntry(i))
1302 return EC;
1303
1305}
1306
1308 const uint8_t *BufStart =
1309 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1310 Data = BufStart;
1311 End = BufStart + Buffer->getBufferSize();
1312
1313 if (std::error_code EC = readMagicIdent())
1314 return EC;
1315
1316 if (std::error_code EC = readSecHdrTable())
1317 return EC;
1318
1320}
1321
1323 uint64_t Size = 0;
1324 for (auto &Entry : SecHdrTable) {
1325 if (Entry.Type == Type)
1326 Size += Entry.Size;
1327 }
1328 return Size;
1329}
1330
1332 // Sections in SecHdrTable is not necessarily in the same order as
1333 // sections in the profile because section like FuncOffsetTable needs
1334 // to be written after section LBRProfile but needs to be read before
1335 // section LBRProfile, so we cannot simply use the last entry in
1336 // SecHdrTable to calculate the file size.
1337 uint64_t FileSize = 0;
1338 for (auto &Entry : SecHdrTable) {
1339 FileSize = std::max(Entry.Offset + Entry.Size, FileSize);
1340 }
1341 return FileSize;
1342}
1343
1344static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
1345 std::string Flags;
1347 Flags.append("{compressed,");
1348 else
1349 Flags.append("{");
1350
1352 Flags.append("flat,");
1353
1354 switch (Entry.Type) {
1355 case SecNameTable:
1357 Flags.append("fixlenmd5,");
1359 Flags.append("md5,");
1361 Flags.append("uniq,");
1362 break;
1363 case SecProfSummary:
1365 Flags.append("partial,");
1367 Flags.append("context,");
1369 Flags.append("preInlined,");
1371 Flags.append("fs-discriminator,");
1372 break;
1373 case SecFuncOffsetTable:
1375 Flags.append("ordered,");
1376 break;
1377 case SecFuncMetadata:
1379 Flags.append("probe,");
1381 Flags.append("attr,");
1382 break;
1383 default:
1384 break;
1385 }
1386 char &last = Flags.back();
1387 if (last == ',')
1388 last = '}';
1389 else
1390 Flags.append("}");
1391 return Flags;
1392}
1393
1395 uint64_t TotalSecsSize = 0;
1396 for (auto &Entry : SecHdrTable) {
1397 OS << getSecName(Entry.Type) << " - Offset: " << Entry.Offset
1398 << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
1399 << "\n";
1400 ;
1401 TotalSecsSize += Entry.Size;
1402 }
1403 uint64_t HeaderSize = SecHdrTable.front().Offset;
1404 assert(HeaderSize + TotalSecsSize == getFileSize() &&
1405 "Size of 'header + sections' doesn't match the total size of profile");
1406
1407 OS << "Header Size: " << HeaderSize << "\n";
1408 OS << "Total Sections Size: " << TotalSecsSize << "\n";
1409 OS << "File Size: " << getFileSize() << "\n";
1410 return true;
1411}
1412
1414 // Read and check the magic identifier.
1415 auto Magic = readNumber<uint64_t>();
1416 if (std::error_code EC = Magic.getError())
1417 return EC;
1418 else if (std::error_code EC = verifySPMagic(*Magic))
1419 return EC;
1420
1421 // Read the version number.
1422 auto Version = readNumber<uint64_t>();
1423 if (std::error_code EC = Version.getError())
1424 return EC;
1425 else if (*Version != SPVersion())
1427
1429}
1430
1432 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1433 End = Data + Buffer->getBufferSize();
1434
1435 if (std::error_code EC = readMagicIdent())
1436 return EC;
1437
1438 if (std::error_code EC = readSummary())
1439 return EC;
1440
1441 if (std::error_code EC = readNameTable())
1442 return EC;
1444}
1445
1446std::error_code SampleProfileReaderBinary::readSummaryEntry(
1447 std::vector<ProfileSummaryEntry> &Entries) {
1448 auto Cutoff = readNumber<uint64_t>();
1449 if (std::error_code EC = Cutoff.getError())
1450 return EC;
1451
1452 auto MinBlockCount = readNumber<uint64_t>();
1453 if (std::error_code EC = MinBlockCount.getError())
1454 return EC;
1455
1456 auto NumBlocks = readNumber<uint64_t>();
1457 if (std::error_code EC = NumBlocks.getError())
1458 return EC;
1459
1460 Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks);
1462}
1463
1465 auto TotalCount = readNumber<uint64_t>();
1466 if (std::error_code EC = TotalCount.getError())
1467 return EC;
1468
1469 auto MaxBlockCount = readNumber<uint64_t>();
1470 if (std::error_code EC = MaxBlockCount.getError())
1471 return EC;
1472
1473 auto MaxFunctionCount = readNumber<uint64_t>();
1474 if (std::error_code EC = MaxFunctionCount.getError())
1475 return EC;
1476
1477 auto NumBlocks = readNumber<uint64_t>();
1478 if (std::error_code EC = NumBlocks.getError())
1479 return EC;
1480
1481 auto NumFunctions = readNumber<uint64_t>();
1482 if (std::error_code EC = NumFunctions.getError())
1483 return EC;
1484
1485 auto NumSummaryEntries = readNumber<uint64_t>();
1486 if (std::error_code EC = NumSummaryEntries.getError())
1487 return EC;
1488
1489 std::vector<ProfileSummaryEntry> Entries;
1490 for (unsigned i = 0; i < *NumSummaryEntries; i++) {
1491 std::error_code EC = readSummaryEntry(Entries);
1492 if (EC != sampleprof_error::success)
1493 return EC;
1494 }
1495 Summary = std::make_unique<ProfileSummary>(
1496 ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0,
1497 *MaxFunctionCount, *NumBlocks, *NumFunctions);
1498
1500}
1501
1503 const uint8_t *Data =
1504 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1505 uint64_t Magic = decodeULEB128(Data);
1506 return Magic == SPMagic();
1507}
1508
1510 const uint8_t *Data =
1511 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1512 uint64_t Magic = decodeULEB128(Data);
1513 return Magic == SPMagic(SPF_Ext_Binary);
1514}
1515
1517 uint32_t dummy;
1518 if (!GcovBuffer.readInt(dummy))
1521}
1522
1524 if (sizeof(T) <= sizeof(uint32_t)) {
1525 uint32_t Val;
1526 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
1527 return static_cast<T>(Val);
1528 } else if (sizeof(T) <= sizeof(uint64_t)) {
1529 uint64_t Val;
1530 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
1531 return static_cast<T>(Val);
1532 }
1533
1534 std::error_code EC = sampleprof_error::malformed;
1535 reportError(0, EC.message());
1536 return EC;
1537}
1538
1540 StringRef Str;
1541 if (!GcovBuffer.readString(Str))
1543 return Str;
1544}
1545
1547 // Read the magic identifier.
1550
1551 // Read the version number. Note - the GCC reader does not validate this
1552 // version, but the profile creator generates v704.
1553 GCOV::GCOVVersion version;
1554 if (!GcovBuffer.readGCOVVersion(version))
1556
1557 if (version != GCOV::V407)
1559
1560 // Skip the empty integer.
1561 if (std::error_code EC = skipNextWord())
1562 return EC;
1563
1565}
1566
1568 uint32_t Tag;
1569 if (!GcovBuffer.readInt(Tag))
1571
1572 if (Tag != Expected)
1574
1575 if (std::error_code EC = skipNextWord())
1576 return EC;
1577
1579}
1580
1582 if (std::error_code EC = readSectionTag(GCOVTagAFDOFileNames))
1583 return EC;
1584
1585 uint32_t Size;
1586 if (!GcovBuffer.readInt(Size))
1588
1589 for (uint32_t I = 0; I < Size; ++I) {
1590 StringRef Str;
1591 if (!GcovBuffer.readString(Str))
1593 Names.push_back(std::string(Str));
1594 }
1595
1597}
1598
1600 if (std::error_code EC = readSectionTag(GCOVTagAFDOFunction))
1601 return EC;
1602
1603 uint32_t NumFunctions;
1604 if (!GcovBuffer.readInt(NumFunctions))
1606
1607 InlineCallStack Stack;
1608 for (uint32_t I = 0; I < NumFunctions; ++I)
1609 if (std::error_code EC = readOneFunctionProfile(Stack, true, 0))
1610 return EC;
1611
1614}
1615
1617 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
1618 uint64_t HeadCount = 0;
1619 if (InlineStack.size() == 0)
1620 if (!GcovBuffer.readInt64(HeadCount))
1622
1623 uint32_t NameIdx;
1624 if (!GcovBuffer.readInt(NameIdx))
1626
1627 StringRef Name(Names[NameIdx]);
1628
1629 uint32_t NumPosCounts;
1630 if (!GcovBuffer.readInt(NumPosCounts))
1632
1633 uint32_t NumCallsites;
1634 if (!GcovBuffer.readInt(NumCallsites))
1636
1637 FunctionSamples *FProfile = nullptr;
1638 if (InlineStack.size() == 0) {
1639 // If this is a top function that we have already processed, do not
1640 // update its profile again. This happens in the presence of
1641 // function aliases. Since these aliases share the same function
1642 // body, there will be identical replicated profiles for the
1643 // original function. In this case, we simply not bother updating
1644 // the profile of the original function.
1645 FProfile = &Profiles[FunctionId(Name)];
1646 FProfile->addHeadSamples(HeadCount);
1647 if (FProfile->getTotalSamples() > 0)
1648 Update = false;
1649 } else {
1650 // Otherwise, we are reading an inlined instance. The top of the
1651 // inline stack contains the profile of the caller. Insert this
1652 // callee in the caller's CallsiteMap.
1653 FunctionSamples *CallerProfile = InlineStack.front();
1654 uint32_t LineOffset = Offset >> 16;
1655 uint32_t Discriminator = Offset & 0xffff;
1656 FProfile = &CallerProfile->functionSamplesAt(
1657 LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
1658 }
1659 FProfile->setFunction(FunctionId(Name));
1660
1661 for (uint32_t I = 0; I < NumPosCounts; ++I) {
1665
1666 uint32_t NumTargets;
1667 if (!GcovBuffer.readInt(NumTargets))
1669
1670 uint64_t Count;
1671 if (!GcovBuffer.readInt64(Count))
1673
1674 // The line location is encoded in the offset as:
1675 // high 16 bits: line offset to the start of the function.
1676 // low 16 bits: discriminator.
1677 uint32_t LineOffset = Offset >> 16;
1678 uint32_t Discriminator = Offset & 0xffff;
1679
1680 InlineCallStack NewStack;
1681 NewStack.push_back(FProfile);
1682 llvm::append_range(NewStack, InlineStack);
1683 if (Update) {
1684 // Walk up the inline stack, adding the samples on this line to
1685 // the total sample count of the callers in the chain.
1686 for (auto *CallerProfile : NewStack)
1687 CallerProfile->addTotalSamples(Count);
1688
1689 // Update the body samples for the current profile.
1690 FProfile->addBodySamples(LineOffset, Discriminator, Count);
1691 }
1692
1693 // Process the list of functions called at an indirect call site.
1694 // These are all the targets that a function pointer (or virtual
1695 // function) resolved at runtime.
1696 for (uint32_t J = 0; J < NumTargets; J++) {
1697 uint32_t HistVal;
1698 if (!GcovBuffer.readInt(HistVal))
1700
1701 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
1703
1704 uint64_t TargetIdx;
1705 if (!GcovBuffer.readInt64(TargetIdx))
1707 StringRef TargetName(Names[TargetIdx]);
1708
1709 uint64_t TargetCount;
1710 if (!GcovBuffer.readInt64(TargetCount))
1712
1713 if (Update)
1714 FProfile->addCalledTargetSamples(LineOffset, Discriminator,
1715 FunctionId(TargetName),
1716 TargetCount);
1717 }
1718 }
1719
1720 // Process all the inlined callers into the current function. These
1721 // are all the callsites that were inlined into this function.
1722 for (uint32_t I = 0; I < NumCallsites; I++) {
1723 // The offset is encoded as:
1724 // high 16 bits: line offset to the start of the function.
1725 // low 16 bits: discriminator.
1729 InlineCallStack NewStack;
1730 NewStack.push_back(FProfile);
1731 llvm::append_range(NewStack, InlineStack);
1732 if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset))
1733 return EC;
1734 }
1735
1737}
1738
1739/// Read a GCC AutoFDO profile.
1740///
1741/// This format is generated by the Linux Perf conversion tool at
1742/// https://github.com/google/autofdo.
1744 assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");
1745 // Read the string table.
1746 if (std::error_code EC = readNameTable())
1747 return EC;
1748
1749 // Read the source profile.
1750 if (std::error_code EC = readFunctionProfiles())
1751 return EC;
1752
1754}
1755
1757 StringRef Magic(reinterpret_cast<const char *>(Buffer.getBufferStart()));
1758 return Magic == "adcg*704";
1759}
1760
1762 // If the reader uses MD5 to represent string, we can't remap it because
1763 // we don't know what the original function names were.
1764 if (Reader.useMD5()) {
1766 Reader.getBuffer()->getBufferIdentifier(),
1767 "Profile data remapping cannot be applied to profile data "
1768 "using MD5 names (original mangled names are not available).",
1769 DS_Warning));
1770 return;
1771 }
1772
1773 // CSSPGO-TODO: Remapper is not yet supported.
1774 // We will need to remap the entire context string.
1775 assert(Remappings && "should be initialized while creating remapper");
1776 for (auto &Sample : Reader.getProfiles()) {
1777 DenseSet<FunctionId> NamesInSample;
1778 Sample.second.findAllNames(NamesInSample);
1779 for (auto &Name : NamesInSample) {
1780 StringRef NameStr = Name.stringRef();
1781 if (auto Key = Remappings->insert(NameStr))
1782 NameMap.insert({Key, NameStr});
1783 }
1784 }
1785
1786 RemappingApplied = true;
1787}
1788
1789std::optional<StringRef>
1791 if (auto Key = Remappings->lookup(Fname)) {
1792 StringRef Result = NameMap.lookup(Key);
1793 if (!Result.empty())
1794 return Result;
1795 }
1796 return std::nullopt;
1797}
1798
1799/// Prepare a memory buffer for the contents of \p Filename.
1800///
1801/// \returns an error code indicating the status of the buffer.
1804 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
1805 : FS.getBufferForFile(Filename);
1806 if (std::error_code EC = BufferOrErr.getError())
1807 return EC;
1808 auto Buffer = std::move(BufferOrErr.get());
1809
1810 return std::move(Buffer);
1811}
1812
1813/// Create a sample profile reader based on the format of the input file.
1814///
1815/// \param Filename The file to open.
1816///
1817/// \param C The LLVM context to use to emit diagnostics.
1818///
1819/// \param P The FSDiscriminatorPass.
1820///
1821/// \param RemapFilename The file used for profile remapping.
1822///
1823/// \returns an error code indicating the status of the created reader.
1827 StringRef RemapFilename) {
1828 auto BufferOrError = setupMemoryBuffer(Filename, FS);
1829 if (std::error_code EC = BufferOrError.getError())
1830 return EC;
1831 return create(BufferOrError.get(), C, FS, P, RemapFilename);
1832}
1833
1834/// Create a sample profile remapper from the given input, to remap the
1835/// function names in the given profile data.
1836///
1837/// \param Filename The file to open.
1838///
1839/// \param Reader The profile reader the remapper is going to be applied to.
1840///
1841/// \param C The LLVM context to use to emit diagnostics.
1842///
1843/// \returns an error code indicating the status of the created reader.
1846 vfs::FileSystem &FS,
1847 SampleProfileReader &Reader,
1848 LLVMContext &C) {
1849 auto BufferOrError = setupMemoryBuffer(Filename, FS);
1850 if (std::error_code EC = BufferOrError.getError())
1851 return EC;
1852 return create(BufferOrError.get(), Reader, C);
1853}
1854
1855/// Create a sample profile remapper from the given input, to remap the
1856/// function names in the given profile data.
1857///
1858/// \param B The memory buffer to create the reader from (assumes ownership).
1859///
1860/// \param C The LLVM context to use to emit diagnostics.
1861///
1862/// \param Reader The profile reader the remapper is going to be applied to.
1863///
1864/// \returns an error code indicating the status of the created reader.
1866SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
1867 SampleProfileReader &Reader,
1868 LLVMContext &C) {
1869 auto Remappings = std::make_unique<SymbolRemappingReader>();
1870 if (Error E = Remappings->read(*B)) {
1872 std::move(E), [&](const SymbolRemappingParseError &ParseError) {
1873 C.diagnose(DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
1874 ParseError.getLineNum(),
1875 ParseError.getMessage()));
1876 });
1878 }
1879
1880 return std::make_unique<SampleProfileReaderItaniumRemapper>(
1881 std::move(B), std::move(Remappings), Reader);
1882}
1883
1884/// Create a sample profile reader based on the format of the input data.
1885///
1886/// \param B The memory buffer to create the reader from (assumes ownership).
1887///
1888/// \param C The LLVM context to use to emit diagnostics.
1889///
1890/// \param P The FSDiscriminatorPass.
1891///
1892/// \param RemapFilename The file used for profile remapping.
1893///
1894/// \returns an error code indicating the status of the created reader.
1896SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
1898 StringRef RemapFilename) {
1899 std::unique_ptr<SampleProfileReader> Reader;
1901 Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C));
1903 Reader.reset(new SampleProfileReaderExtBinary(std::move(B), C));
1905 Reader.reset(new SampleProfileReaderGCC(std::move(B), C));
1907 Reader.reset(new SampleProfileReaderText(std::move(B), C));
1908 else
1910
1911 if (!RemapFilename.empty()) {
1913 RemapFilename, FS, *Reader, C);
1914 if (std::error_code EC = ReaderOrErr.getError()) {
1915 std::string Msg = "Could not create remapper: " + EC.message();
1916 C.diagnose(DiagnosticInfoSampleProfile(RemapFilename, Msg));
1917 return EC;
1918 }
1919 Reader->Remapper = std::move(ReaderOrErr.get());
1920 }
1921
1922 if (std::error_code EC = Reader->readHeader()) {
1923 return EC;
1924 }
1925
1926 Reader->setDiscriminatorMaskedBitFrom(P);
1927
1928 return std::move(Reader);
1929}
1930
1931// For text and GCC file formats, we compute the summary after reading the
1932// profile. Binary format has the profile summary in its header.
1936}
AMDGPU Kernel Attributes
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
std::string Name
uint64_t Size
Provides ErrorOr<T> smart pointer.
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS)
This file supports working with JSON data.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Module.h This file contains the declarations for the Module class.
#define P(N)
static bool isDigit(const char C)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
static bool ParseHead(const StringRef &Input, StringRef &FName, uint64_t &NumSamples, uint64_t &NumHeadSamples)
Parse Input as function head.
static void dumpFunctionProfileJson(const FunctionSamples &S, json::OStream &JOS, bool TopLevel=false)
static bool isOffsetLegal(unsigned L)
Returns true if line offset L is legal (only has 16 bits).
static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, uint64_t &NumSamples, uint32_t &LineOffset, uint32_t &Discriminator, StringRef &CalleeName, DenseMap< StringRef, uint64_t > &TargetCountMap, uint64_t &FunctionHash, uint32_t &Attributes)
Parse Input as line sample.
static cl::opt< bool > ProfileIsFSDisciminator("profile-isfs", cl::Hidden, cl::init(false), cl::desc("Profile uses flow sensitive discriminators"))
static std::string getSecFlagsStr(const SecHdrTableEntry &Entry)
static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash, uint32_t &Attributes)
Parse Input that contains metadata.
@ CallSiteProfile
raw_pwrite_stream & OS
Defines the virtual file system interface vfs::FileSystem.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
LLVM_ATTRIBUTE_RETURNS_NONNULL void * Allocate(size_t Size, Align Alignment)
Allocate space at the specified alignment.
Definition: Allocator.h:148
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Diagnostic information for the sample profiler.
Represents either an error or a value T.
Definition: ErrorOr.h:56
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
Tagged union holding either a T or a Error.
Definition: Error.h:481
bool readInt(uint32_t &Val)
Definition: GCOV.h:152
bool readInt64(uint64_t &Val)
Definition: GCOV.h:162
bool readGCOVVersion(GCOV::GCOVVersion &version)
readGCOVVersion - Read GCOV version.
Definition: GCOV.h:108
bool readString(StringRef &str)
Definition: GCOV.h:170
bool readGCDAFormat()
readGCDAFormat - Check GCDA signature is valid at the beginning of buffer.
Definition: GCOV.h:94
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:594
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This interface provides simple read-only access to a block of memory, and provides simple methods for...
Definition: MemoryBuffer.h:51
virtual StringRef getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
Definition: MemoryBuffer.h:76
static ErrorOr< std::unique_ptr< MemoryBuffer > > getSTDIN()
Read all of stdin into a file buffer, and return it.
Root of the metadata hierarchy.
Definition: Metadata.h:62
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:70
std::unique_ptr< ProfileSummary > computeSummaryForProfiles(const sampleprof::SampleProfileMap &Profiles)
size_t size() const
Definition: SmallVector.h:91
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:463
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:564
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:258
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
size_t find_last_of(char C, size_t From=npos) const
Find the last character in the string that is C, or npos if not found.
Definition: StringRef.h:393
size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
Definition: StringRef.h:370
size_t rfind(char C, size_t From=npos) const
Search for the last character C in the string.
Definition: StringRef.h:340
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition: StringRef.h:290
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition: StringRef.h:808
static constexpr size_t npos
Definition: StringRef.h:52
size_t find_first_not_of(char C, size_t From=0) const
Find the first character in the string that is not C or npos if not found.
Definition: StringRef.cpp:251
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
json::OStream allows writing well-formed JSON without materializing all structures as json::Value ahe...
Definition: JSON.h:977
void object(Block Contents)
Emit an object whose elements are emitted in the provided Block.
Definition: JSON.h:1007
void attribute(llvm::StringRef Key, const Value &Contents)
Emit an attribute whose value is self-contained (number, vector<int> etc).
Definition: JSON.h:1032
void arrayBegin()
Definition: JSON.cpp:844
void attributeArray(llvm::StringRef Key, Block Contents)
Emit an attribute whose value is an array with elements from the Block.
Definition: JSON.h:1036
A forward iterator which reads text lines from a buffer.
Definition: LineIterator.h:33
int64_t line_number() const
Return the current line number. May return any number at EOF.
Definition: LineIterator.h:66
bool is_at_eof() const
Return true if we've reached EOF or are an "end" iterator.
Definition: LineIterator.h:60
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
This class represents a function that is read from a sample profile.
Definition: FunctionId.h:36
StringRef stringRef() const
Convert to StringRef.
Definition: FunctionId.h:108
uint64_t getHashCode() const
Get hash code of this object.
Definition: FunctionId.h:123
std::string str() const
Convert to a string, usually for output purpose.
Definition: FunctionId.h:97
Representation of the samples collected for a function.
Definition: SampleProf.h:744
sampleprof_error addTotalSamples(uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:751
uint64_t getHeadSamples() const
For top-level functions, return the total number of branch samples that have the function as the bran...
Definition: SampleProf.h:940
FunctionId getFunction() const
Return the function name.
Definition: SampleProf.h:1069
sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:770
sampleprof_error addCalledTargetSamples(uint32_t LineOffset, uint32_t Discriminator, FunctionId Func, uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:784
void setFunction(FunctionId newFunction)
Set the name of the function.
Definition: SampleProf.h:1064
FunctionSamplesMap & functionSamplesAt(const LineLocation &Loc)
Return the function samples at the given callsite location.
Definition: SampleProf.h:906
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes.
Definition: SampleProf.h:1085
sampleprof_error addBodySamples(uint32_t LineOffset, uint32_t Discriminator, uint64_t Num, uint64_t Weight=1)
Definition: SampleProf.h:778
void setFunctionHash(uint64_t Hash)
Definition: SampleProf.h:1074
static bool ProfileIsFS
If this profile uses flow sensitive discriminators.
Definition: SampleProf.h:1196
SampleContext & getContext() const
Definition: SampleProf.h:1185
static bool HasUniqSuffix
Whether the profile contains any ".__uniq." suffix in a name.
Definition: SampleProf.h:1193
uint64_t getTotalSamples() const
Return the total number of samples collected inside the function.
Definition: SampleProf.h:932
const CallsiteSampleMap & getCallsiteSamples() const
Return all the callsite samples collected in the body of the function.
Definition: SampleProf.h:974
void setContext(const SampleContext &FContext)
Definition: SampleProf.h:1187
const BodySampleMap & getBodySamples() const
Return all the samples collected in the body of the function.
Definition: SampleProf.h:971
std::pair< iterator, bool > try_emplace(const key_type &Hash, const original_key_type &Key, Ts &&...Args)
Definition: HashKeyMap.h:65
void setAllAttributes(uint32_t A)
Definition: SampleProf.h:610
uint64_t getHashCode() const
Definition: SampleProf.h:638
FunctionId getFunction() const
Definition: SampleProf.h:616
bool IsPrefixOf(const SampleContext &That) const
Definition: SampleProf.h:695
iterator find(const SampleContext &Ctx)
Definition: SampleProf.h:1317
mapped_type & Create(const SampleContext &Ctx)
Definition: SampleProf.h:1310
std::error_code readProfile(FunctionSamples &FProfile)
Read the contents of the given profile instance.
std::error_code readNameTable()
Read the whole name table.
const uint8_t * Data
Points to the current location in the buffer.
ErrorOr< StringRef > readString()
Read a string from the profile.
std::vector< FunctionId > NameTable
Function name table.
ErrorOr< T > readNumber()
Read a numeric value of type T from the profile.
ErrorOr< SampleContextFrames > readContextFromTable(size_t *RetIdx=nullptr)
Read a context indirectly via the CSNameTable.
ErrorOr< std::pair< SampleContext, uint64_t > > readSampleContextFromTable()
Read a context indirectly via the CSNameTable if the profile has context, otherwise same as readStrin...
std::error_code readHeader() override
Read and validate the file header.
const uint64_t * MD5SampleContextStart
The starting address of the table of MD5 values of sample contexts.
std::vector< SampleContextFrameVector > CSNameTable
CSNameTable is used to save full context vectors.
std::error_code readImpl() override
Read sample profiles from the associated file.
ErrorOr< FunctionId > readStringFromTable(size_t *RetIdx=nullptr)
Read a string indirectly via the name table. Optionally return the index.
std::vector< uint64_t > MD5SampleContextTable
Table to cache MD5 values of sample contexts corresponding to readSampleContextFromTable(),...
ErrorOr< size_t > readStringIndex(T &Table)
Read the string index and check whether it overflows the table.
const uint8_t * End
Points to the end of the buffer.
ErrorOr< T > readUnencodedNumber()
Read a numeric value of type T from the profile.
std::error_code readFuncProfile(const uint8_t *Start)
Read the next function profile instance.
std::error_code readSummary()
Read profile summary.
std::error_code readMagicIdent()
Read the contents of Magic number and Version number.
std::error_code readFuncMetadata(bool ProfileHasAttribute)
bool collectFuncsFromModule() override
Collect functions with definitions in Module M.
uint64_t getSectionSize(SecType Type)
Get the total size of all Type sections.
virtual std::error_code readCustomSection(const SecHdrTableEntry &Entry)=0
std::vector< std::pair< SampleContext, uint64_t > > FuncOffsetList
The list version of FuncOffsetTable.
DenseSet< StringRef > FuncsToUse
The set containing the functions to use when compiling a module.
std::unique_ptr< ProfileSymbolList > ProfSymList
bool useFuncOffsetList() const
Determine which container readFuncOffsetTable() should populate, the list FuncOffsetList or the map F...
std::error_code readNameTableSec(bool IsMD5, bool FixedLengthMD5)
std::error_code readImpl() override
Read sample profiles in extensible format from the associated file.
virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry)
bool dumpSectionInfo(raw_ostream &OS=dbgs()) override
bool SkipFlatProf
If SkipFlatProf is true, skip the sections with SecFlagFlat flag.
DenseMap< hash_code, uint64_t > FuncOffsetTable
The table mapping from a function context's MD5 to the offset of its FunctionSample towards file star...
std::error_code readHeader() override
Read and validate the file header.
uint64_t getFileSize()
Get the total size of header and all sections.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
GCOVBuffer GcovBuffer
GCOV buffer containing the profile.
std::vector< std::string > Names
Function names in this profile.
std::error_code readImpl() override
Read sample profiles from the associated file.
std::error_code readHeader() override
Read and validate the file header.
std::error_code readOneFunctionProfile(const InlineCallStack &InlineStack, bool Update, uint32_t Offset)
static const uint32_t GCOVTagAFDOFileNames
GCOV tags used to separate sections in the profile file.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
std::error_code readSectionTag(uint32_t Expected)
Read the section tag and check that it's the same as Expected.
static ErrorOr< std::unique_ptr< SampleProfileReaderItaniumRemapper > > create(StringRef Filename, vfs::FileSystem &FS, SampleProfileReader &Reader, LLVMContext &C)
Create a remapper from the given remapping file.
void applyRemapping(LLVMContext &Ctx)
Apply remappings to the profile read by Reader.
std::optional< StringRef > lookUpNameInProfile(StringRef FunctionName)
Return the equivalent name in the profile for FunctionName if it exists.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
std::error_code readImpl() override
Read sample profiles from the associated file.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if Buffer is in the format supported by this class.
Sample-based profile reader.
bool ProfileIsPreInlined
Whether function profile contains ShouldBeInlined contexts.
SampleProfileMap & getProfiles()
Return all the profiles.
uint32_t CSProfileCount
Number of context-sensitive profiles.
static ErrorOr< std::unique_ptr< SampleProfileReader > > create(StringRef Filename, LLVMContext &C, vfs::FileSystem &FS, FSDiscriminatorPass P=FSDiscriminatorPass::Base, StringRef RemapFilename="")
Create a sample profile reader appropriate to the file format.
void dump(raw_ostream &OS=dbgs())
Print all the profiles on stream OS.
bool useMD5() const
Return whether names in the profile are all MD5 numbers.
const Module * M
The current module being compiled if SampleProfileReader is used by compiler.
std::unique_ptr< MemoryBuffer > Buffer
Memory buffer holding the profile file.
std::unique_ptr< SampleProfileReaderItaniumRemapper > Remapper
bool ProfileIsCS
Whether function profiles are context-sensitive flat profiles.
bool ProfileIsMD5
Whether the profile uses MD5 for Sample Contexts and function names.
std::unique_ptr< ProfileSummary > Summary
Profile summary information.
void computeSummary()
Compute summary for this profile.
uint32_t getDiscriminatorMask() const
Get the bitmask the discriminators: For FS profiles, return the bit mask for this pass.
bool ProfileIsFS
Whether the function profiles use FS discriminators.
void dumpJson(raw_ostream &OS=dbgs())
Print all the profiles on stream OS in the JSON format.
SampleProfileMap Profiles
Map every function to its associated profile.
void dumpFunctionProfile(const FunctionSamples &FS, raw_ostream &OS=dbgs())
Print the profile for FunctionSamples on stream OS.
bool ProfileIsProbeBased
Whether samples are collected based on pseudo probes.
void reportError(int64_t LineNumber, const Twine &Msg) const
Report a parse error message.
Representation of a single sample record.
Definition: SampleProf.h:325
uint64_t getSamples() const
Definition: SampleProf.h:392
const SortedCallTargetSet getSortedCallTargets() const
Definition: SampleProf.h:394
The virtual file system interface.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
GCOVVersion
Definition: GCOV.h:42
@ V407
Definition: GCOV.h:42
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)
void sortFuncProfiles(const SampleProfileMap &ProfileMap, std::vector< NameFunctionSamples > &SortedProfiles)
Definition: SampleProf.cpp:202
static uint64_t SPMagic(SampleProfileFormat Format=SPF_Binary)
Definition: SampleProf.h:106
static bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag)
Definition: SampleProf.h:264
std::map< LineLocation, FunctionSamplesMap > CallsiteSampleMap
Definition: SampleProf.h:735
uint64_t MD5Hash(const FunctionId &Obj)
Definition: FunctionId.h:167
std::map< LineLocation, SampleRecord > BodySampleMap
Definition: SampleProf.h:731
@ SecFlagIsPreInlined
SecFlagIsPreInlined means this profile contains ShouldBeInlined contexts thus this is CS preinliner c...
@ SecFlagPartial
SecFlagPartial means the profile is for common/shared code.
@ SecFlagFSDiscriminator
SecFlagFSDiscriminator means this profile uses flow-sensitive discriminators.
@ SecFlagFullContext
SecFlagContext means this is context-sensitive flat profile for CSSPGO.
SmallVector< SampleContextFrame, 1 > SampleContextFrameVector
Definition: SampleProf.h:503
static std::string getSecName(SecType Type)
Definition: SampleProf.h:131
static uint64_t SPVersion()
Definition: SampleProf.h:113
uint64_t read64le(const void *P)
Definition: Endian.h:428
void write64le(void *P, uint64_t V)
Definition: Endian.h:471
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
Definition: Error.h:977
uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)
Utility function to decode a ULEB128 value.
Definition: LEB128.h:131
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2067
sampleprof_error
Definition: SampleProf.h:47
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
sampleprof_error MergeResult(sampleprof_error &Accumulator, sampleprof_error Result)
Definition: SampleProf.h:69
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1914
@ DS_Warning
Represents the relative location of an instruction.
Definition: SampleProf.h:280