LLVM  9.0.0svn
InstrProfReader.cpp
Go to the documentation of this file.
1 //===- InstrProfReader.cpp - Instrumented profiling reader ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains support for reading profiling data for clang's
10 // instrumentation based PGO and coverage.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/DenseMap.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/IR/ProfileSummary.h"
22 #include "llvm/Support/Endian.h"
23 #include "llvm/Support/Error.h"
24 #include "llvm/Support/ErrorOr.h"
28 #include <algorithm>
29 #include <cctype>
30 #include <cstddef>
31 #include <cstdint>
32 #include <limits>
33 #include <memory>
34 #include <system_error>
35 #include <utility>
36 #include <vector>
37 
38 using namespace llvm;
39 
41 setupMemoryBuffer(const Twine &Path) {
44  if (std::error_code EC = BufferOrErr.getError())
45  return errorCodeToError(EC);
46  return std::move(BufferOrErr.get());
47 }
48 
50  return Reader.readHeader();
51 }
52 
55  // Set up the buffer to read.
56  auto BufferOrError = setupMemoryBuffer(Path);
57  if (Error E = BufferOrError.takeError())
58  return std::move(E);
59  return InstrProfReader::create(std::move(BufferOrError.get()));
60 }
61 
63 InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer) {
64  // Sanity check the buffer.
65  if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<unsigned>::max())
66  return make_error<InstrProfError>(instrprof_error::too_large);
67 
68  if (Buffer->getBufferSize() == 0)
69  return make_error<InstrProfError>(instrprof_error::empty_raw_profile);
70 
71  std::unique_ptr<InstrProfReader> Result;
72  // Create the reader.
74  Result.reset(new IndexedInstrProfReader(std::move(Buffer)));
75  else if (RawInstrProfReader64::hasFormat(*Buffer))
76  Result.reset(new RawInstrProfReader64(std::move(Buffer)));
77  else if (RawInstrProfReader32::hasFormat(*Buffer))
78  Result.reset(new RawInstrProfReader32(std::move(Buffer)));
79  else if (TextInstrProfReader::hasFormat(*Buffer))
80  Result.reset(new TextInstrProfReader(std::move(Buffer)));
81  else
82  return make_error<InstrProfError>(instrprof_error::unrecognized_format);
83 
84  // Initialize the reader and return the result.
85  if (Error E = initializeReader(*Result))
86  return std::move(E);
87 
88  return std::move(Result);
89 }
90 
92 IndexedInstrProfReader::create(const Twine &Path, const Twine &RemappingPath) {
93  // Set up the buffer to read.
94  auto BufferOrError = setupMemoryBuffer(Path);
95  if (Error E = BufferOrError.takeError())
96  return std::move(E);
97 
98  // Set up the remapping buffer if requested.
99  std::unique_ptr<MemoryBuffer> RemappingBuffer;
100  std::string RemappingPathStr = RemappingPath.str();
101  if (!RemappingPathStr.empty()) {
102  auto RemappingBufferOrError = setupMemoryBuffer(RemappingPathStr);
103  if (Error E = RemappingBufferOrError.takeError())
104  return std::move(E);
105  RemappingBuffer = std::move(RemappingBufferOrError.get());
106  }
107 
108  return IndexedInstrProfReader::create(std::move(BufferOrError.get()),
109  std::move(RemappingBuffer));
110 }
111 
113 IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
114  std::unique_ptr<MemoryBuffer> RemappingBuffer) {
115  // Sanity check the buffer.
116  if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<unsigned>::max())
117  return make_error<InstrProfError>(instrprof_error::too_large);
118 
119  // Create the reader.
120  if (!IndexedInstrProfReader::hasFormat(*Buffer))
121  return make_error<InstrProfError>(instrprof_error::bad_magic);
122  auto Result = llvm::make_unique<IndexedInstrProfReader>(
123  std::move(Buffer), std::move(RemappingBuffer));
124 
125  // Initialize the reader and return the result.
126  if (Error E = initializeReader(*Result))
127  return std::move(E);
128 
129  return std::move(Result);
130 }
131 
132 void InstrProfIterator::Increment() {
133  if (auto E = Reader->readNextRecord(Record)) {
134  // Handle errors in the reader.
135  InstrProfError::take(std::move(E));
136  *this = InstrProfIterator();
137  }
138 }
139 
141  // Verify that this really looks like plain ASCII text by checking a
142  // 'reasonable' number of characters (up to profile magic size).
143  size_t count = std::min(Buffer.getBufferSize(), sizeof(uint64_t));
144  StringRef buffer = Buffer.getBufferStart();
145  return count == 0 ||
146  std::all_of(buffer.begin(), buffer.begin() + count,
147  [](char c) { return isPrint(c) || ::isspace(c); });
148 }
149 
150 // Read the profile variant flag from the header: ":FE" means this is a FE
151 // generated profile. ":IR" means this is an IR level profile. Other strings
152 // with a leading ':' will be reported an error format.
154  Symtab.reset(new InstrProfSymtab());
155  bool IsIRInstr = false;
156  if (!Line->startswith(":")) {
157  IsIRLevelProfile = false;
158  return success();
159  }
160  StringRef Str = (Line)->substr(1);
161  if (Str.equals_lower("ir"))
162  IsIRInstr = true;
163  else if (Str.equals_lower("fe"))
164  IsIRInstr = false;
165  else if (Str.equals_lower("csir")) {
166  IsIRInstr = true;
167  HasCSIRLevelProfile = true;
168  } else
170 
171  ++Line;
172  IsIRLevelProfile = IsIRInstr;
173  return success();
174 }
175 
176 Error
177 TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
178 
179 #define CHECK_LINE_END(Line) \
180  if (Line.is_at_end()) \
181  return error(instrprof_error::truncated);
182 #define READ_NUM(Str, Dst) \
183  if ((Str).getAsInteger(10, (Dst))) \
184  return error(instrprof_error::malformed);
185 #define VP_READ_ADVANCE(Val) \
186  CHECK_LINE_END(Line); \
187  uint32_t Val; \
188  READ_NUM((*Line), (Val)); \
189  Line++;
190 
191  if (Line.is_at_end())
192  return success();
193 
194  uint32_t NumValueKinds;
195  if (Line->getAsInteger(10, NumValueKinds)) {
196  // No value profile data
197  return success();
198  }
199  if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
201  Line++;
202 
203  for (uint32_t VK = 0; VK < NumValueKinds; VK++) {
205  if (ValueKind > IPVK_Last)
207  VP_READ_ADVANCE(NumValueSites);
208  if (!NumValueSites)
209  continue;
210 
211  Record.reserveSites(VK, NumValueSites);
212  for (uint32_t S = 0; S < NumValueSites; S++) {
213  VP_READ_ADVANCE(NumValueData);
214 
215  std::vector<InstrProfValueData> CurrentValues;
216  for (uint32_t V = 0; V < NumValueData; V++) {
217  CHECK_LINE_END(Line);
218  std::pair<StringRef, StringRef> VD = Line->rsplit(':');
219  uint64_t TakenCount, Value;
220  if (ValueKind == IPVK_IndirectCallTarget) {
221  if (InstrProfSymtab::isExternalSymbol(VD.first)) {
222  Value = 0;
223  } else {
224  if (Error E = Symtab->addFuncName(VD.first))
225  return E;
226  Value = IndexedInstrProf::ComputeHash(VD.first);
227  }
228  } else {
229  READ_NUM(VD.first, Value);
230  }
231  READ_NUM(VD.second, TakenCount);
232  CurrentValues.push_back({Value, TakenCount});
233  Line++;
234  }
235  Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData,
236  nullptr);
237  }
238  }
239  return success();
240 
241 #undef CHECK_LINE_END
242 #undef READ_NUM
243 #undef VP_READ_ADVANCE
244 }
245 
247  // Skip empty lines and comments.
248  while (!Line.is_at_end() && (Line->empty() || Line->startswith("#")))
249  ++Line;
250  // If we hit EOF while looking for a name, we're done.
251  if (Line.is_at_end()) {
252  return error(instrprof_error::eof);
253  }
254 
255  // Read the function name.
256  Record.Name = *Line++;
257  if (Error E = Symtab->addFuncName(Record.Name))
258  return error(std::move(E));
259 
260  // Read the function hash.
261  if (Line.is_at_end())
263  if ((Line++)->getAsInteger(0, Record.Hash))
265 
266  // Read the number of counters.
267  uint64_t NumCounters;
268  if (Line.is_at_end())
270  if ((Line++)->getAsInteger(10, NumCounters))
272  if (NumCounters == 0)
274 
275  // Read each counter and fill our internal storage with the values.
276  Record.Clear();
277  Record.Counts.reserve(NumCounters);
278  for (uint64_t I = 0; I < NumCounters; ++I) {
279  if (Line.is_at_end())
281  uint64_t Count;
282  if ((Line++)->getAsInteger(10, Count))
284  Record.Counts.push_back(Count);
285  }
286 
287  // Check if value profile data exists and read it if so.
288  if (Error E = readValueProfileData(Record))
289  return error(std::move(E));
290 
291  return success();
292 }
293 
294 template <class IntPtrT>
296  if (DataBuffer.getBufferSize() < sizeof(uint64_t))
297  return false;
298  uint64_t Magic =
299  *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
300  return RawInstrProf::getMagic<IntPtrT>() == Magic ||
301  sys::getSwappedBytes(RawInstrProf::getMagic<IntPtrT>()) == Magic;
302 }
303 
304 template <class IntPtrT>
306  if (!hasFormat(*DataBuffer))
308  if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header))
310  auto *Header = reinterpret_cast<const RawInstrProf::Header *>(
311  DataBuffer->getBufferStart());
312  ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
313  return readHeader(*Header);
314 }
315 
316 template <class IntPtrT>
317 Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
318  const char *End = DataBuffer->getBufferEnd();
319  // Skip zero padding between profiles.
320  while (CurrentPos != End && *CurrentPos == 0)
321  ++CurrentPos;
322  // If there's nothing left, we're done.
323  if (CurrentPos == End)
324  return make_error<InstrProfError>(instrprof_error::eof);
325  // If there isn't enough space for another header, this is probably just
326  // garbage at the end of the file.
327  if (CurrentPos + sizeof(RawInstrProf::Header) > End)
328  return make_error<InstrProfError>(instrprof_error::malformed);
329  // The writer ensures each profile is padded to start at an aligned address.
330  if (reinterpret_cast<size_t>(CurrentPos) % alignof(uint64_t))
331  return make_error<InstrProfError>(instrprof_error::malformed);
332  // The magic should have the same byte order as in the previous header.
333  uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos);
334  if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
335  return make_error<InstrProfError>(instrprof_error::bad_magic);
336 
337  // There's another profile to read, so we need to process the header.
338  auto *Header = reinterpret_cast<const RawInstrProf::Header *>(CurrentPos);
339  return readHeader(*Header);
340 }
341 
342 template <class IntPtrT>
344  if (Error E = Symtab.create(StringRef(NamesStart, NamesSize)))
345  return error(std::move(E));
346  for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
347  const IntPtrT FPtr = swap(I->FunctionPointer);
348  if (!FPtr)
349  continue;
350  Symtab.mapAddress(FPtr, I->NameRef);
351  }
352  return success();
353 }
354 
355 template <class IntPtrT>
357  const RawInstrProf::Header &Header) {
358  Version = swap(Header.Version);
359  if (GET_VERSION(Version) != RawInstrProf::Version)
361 
362  CountersDelta = swap(Header.CountersDelta);
363  NamesDelta = swap(Header.NamesDelta);
364  auto DataSize = swap(Header.DataSize);
365  auto CountersSize = swap(Header.CountersSize);
366  NamesSize = swap(Header.NamesSize);
367  ValueKindLast = swap(Header.ValueKindLast);
368 
369  auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData<IntPtrT>);
370  auto PaddingSize = getNumPaddingBytes(NamesSize);
371 
372  ptrdiff_t DataOffset = sizeof(RawInstrProf::Header);
373  ptrdiff_t CountersOffset = DataOffset + DataSizeInBytes;
374  ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize;
375  ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize;
376 
377  auto *Start = reinterpret_cast<const char *>(&Header);
378  if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
380 
381  Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
382  Start + DataOffset);
383  DataEnd = Data + DataSize;
384  CountersStart = reinterpret_cast<const uint64_t *>(Start + CountersOffset);
385  NamesStart = Start + NamesOffset;
386  ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
387 
388  std::unique_ptr<InstrProfSymtab> NewSymtab = make_unique<InstrProfSymtab>();
389  if (Error E = createSymtab(*NewSymtab.get()))
390  return E;
391 
392  Symtab = std::move(NewSymtab);
393  return success();
394 }
395 
396 template <class IntPtrT>
398  Record.Name = getName(Data->NameRef);
399  return success();
400 }
401 
402 template <class IntPtrT>
404  Record.Hash = swap(Data->FuncHash);
405  return success();
406 }
407 
408 template <class IntPtrT>
410  InstrProfRecord &Record) {
411  uint32_t NumCounters = swap(Data->NumCounters);
412  IntPtrT CounterPtr = Data->CounterPtr;
413  if (NumCounters == 0)
415 
416  auto RawCounts = makeArrayRef(getCounter(CounterPtr), NumCounters);
417  auto *NamesStartAsCounter = reinterpret_cast<const uint64_t *>(NamesStart);
418 
419  // Check bounds.
420  if (RawCounts.data() < CountersStart ||
421  RawCounts.data() + RawCounts.size() > NamesStartAsCounter)
423 
424  if (ShouldSwapBytes) {
425  Record.Counts.clear();
426  Record.Counts.reserve(RawCounts.size());
427  for (uint64_t Count : RawCounts)
428  Record.Counts.push_back(swap(Count));
429  } else
430  Record.Counts = RawCounts;
431 
432  return success();
433 }
434 
435 template <class IntPtrT>
437  InstrProfRecord &Record) {
438  Record.clearValueData();
439  CurValueDataSize = 0;
440  // Need to match the logic in value profile dumper code in compiler-rt:
441  uint32_t NumValueKinds = 0;
442  for (uint32_t I = 0; I < IPVK_Last + 1; I++)
443  NumValueKinds += (Data->NumValueSites[I] != 0);
444 
445  if (!NumValueKinds)
446  return success();
447 
449  ValueProfData::getValueProfData(
450  ValueDataStart, (const unsigned char *)DataBuffer->getBufferEnd(),
451  getDataEndianness());
452 
453  if (Error E = VDataPtrOrErr.takeError())
454  return E;
455 
456  // Note that besides deserialization, this also performs the conversion for
457  // indirect call targets. The function pointers from the raw profile are
458  // remapped into function name hashes.
459  VDataPtrOrErr.get()->deserializeTo(Record, Symtab.get());
460  CurValueDataSize = VDataPtrOrErr.get()->getSize();
461  return success();
462 }
463 
464 template <class IntPtrT>
466  if (atEnd())
467  // At this point, ValueDataStart field points to the next header.
468  if (Error E = readNextHeader(getNextHeaderPos()))
469  return error(std::move(E));
470 
471  // Read name ad set it in Record.
472  if (Error E = readName(Record))
473  return error(std::move(E));
474 
475  // Read FuncHash and set it in Record.
476  if (Error E = readFuncHash(Record))
477  return error(std::move(E));
478 
479  // Read raw counts and set Record.
480  if (Error E = readRawCounts(Record))
481  return error(std::move(E));
482 
483  // Read value data and set Record.
484  if (Error E = readValueProfilingData(Record))
485  return error(std::move(E));
486 
487  // Iterate.
488  advanceData();
489  return success();
490 }
491 
492 namespace llvm {
493 
494 template class RawInstrProfReader<uint32_t>;
495 template class RawInstrProfReader<uint64_t>;
496 
497 } // end namespace llvm
498 
502 }
503 
506 
508  const unsigned char *&D, const unsigned char *const End) {
510  ValueProfData::getValueProfData(D, End, ValueProfDataEndianness);
511 
512  if (VDataPtrOrErr.takeError())
513  return false;
514 
515  VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), nullptr);
516  D += VDataPtrOrErr.get()->TotalSize;
517 
518  return true;
519 }
520 
522  offset_type N) {
523  using namespace support;
524 
525  // Check if the data is corrupt. If so, don't try to read it.
526  if (N % sizeof(uint64_t))
527  return data_type();
528 
529  DataBuffer.clear();
530  std::vector<uint64_t> CounterBuffer;
531 
532  const unsigned char *End = D + N;
533  while (D < End) {
534  // Read hash.
535  if (D + sizeof(uint64_t) >= End)
536  return data_type();
537  uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D);
538 
539  // Initialize number of counters for GET_VERSION(FormatVersion) == 1.
540  uint64_t CountsSize = N / sizeof(uint64_t) - 1;
541  // If format version is different then read the number of counters.
542  if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) {
543  if (D + sizeof(uint64_t) > End)
544  return data_type();
545  CountsSize = endian::readNext<uint64_t, little, unaligned>(D);
546  }
547  // Read counter values.
548  if (D + CountsSize * sizeof(uint64_t) > End)
549  return data_type();
550 
551  CounterBuffer.clear();
552  CounterBuffer.reserve(CountsSize);
553  for (uint64_t J = 0; J < CountsSize; ++J)
554  CounterBuffer.push_back(endian::readNext<uint64_t, little, unaligned>(D));
555 
556  DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer));
557 
558  // Read value profiling data.
559  if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 &&
560  !readValueProfilingData(D, End)) {
561  DataBuffer.clear();
562  return data_type();
563  }
564  }
565  return DataBuffer;
566 }
567 
568 template <typename HashTableImpl>
571  auto Iter = HashTable->find(FuncName);
572  if (Iter == HashTable->end())
573  return make_error<InstrProfError>(instrprof_error::unknown_function);
574 
575  Data = (*Iter);
576  if (Data.empty())
577  return make_error<InstrProfError>(instrprof_error::malformed);
578 
579  return Error::success();
580 }
581 
582 template <typename HashTableImpl>
585  if (atEnd())
586  return make_error<InstrProfError>(instrprof_error::eof);
587 
588  Data = *RecordIterator;
589 
590  if (Data.empty())
591  return make_error<InstrProfError>(instrprof_error::malformed);
592 
593  return Error::success();
594 }
595 
596 template <typename HashTableImpl>
598  const unsigned char *Buckets, const unsigned char *const Payload,
599  const unsigned char *const Base, IndexedInstrProf::HashT HashType,
600  uint64_t Version) {
601  FormatVersion = Version;
602  HashTable.reset(HashTableImpl::Create(
603  Buckets, Payload, Base,
604  typename HashTableImpl::InfoType(HashType, Version)));
605  RecordIterator = HashTable->data_begin();
606 }
607 
608 namespace {
609 /// A remapper that does not apply any remappings.
610 class InstrProfReaderNullRemapper : public InstrProfReaderRemapper {
612 
613 public:
614  InstrProfReaderNullRemapper(InstrProfReaderIndexBase &Underlying)
615  : Underlying(Underlying) {}
616 
617  Error getRecords(StringRef FuncName,
619  return Underlying.getRecords(FuncName, Data);
620  }
621 };
622 }
623 
624 /// A remapper that applies remappings based on a symbol remapping file.
625 template <typename HashTableImpl>
627  : public InstrProfReaderRemapper {
628 public:
630  std::unique_ptr<MemoryBuffer> RemapBuffer,
632  : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {
633  }
634 
635  /// Extract the original function name from a PGO function name.
637  // We can have multiple :-separated pieces; there can be pieces both
638  // before and after the mangled name. Find the first part that starts
639  // with '_Z'; we'll assume that's the mangled name we want.
640  std::pair<StringRef, StringRef> Parts = {StringRef(), Name};
641  while (true) {
642  Parts = Parts.second.split(':');
643  if (Parts.first.startswith("_Z"))
644  return Parts.first;
645  if (Parts.second.empty())
646  return Name;
647  }
648  }
649 
650  /// Given a mangled name extracted from a PGO function name, and a new
651  /// form for that mangled name, reconstitute the name.
652  static void reconstituteName(StringRef OrigName, StringRef ExtractedName,
653  StringRef Replacement,
654  SmallVectorImpl<char> &Out) {
655  Out.reserve(OrigName.size() + Replacement.size() - ExtractedName.size());
656  Out.insert(Out.end(), OrigName.begin(), ExtractedName.begin());
657  Out.insert(Out.end(), Replacement.begin(), Replacement.end());
658  Out.insert(Out.end(), ExtractedName.end(), OrigName.end());
659  }
660 
662  if (Error E = Remappings.read(*RemapBuffer))
663  return E;
664  for (StringRef Name : Underlying.HashTable->keys()) {
665  StringRef RealName = extractName(Name);
666  if (auto Key = Remappings.insert(RealName)) {
667  // FIXME: We could theoretically map the same equivalence class to
668  // multiple names in the profile data. If that happens, we should
669  // return NamedInstrProfRecords from all of them.
670  MappedNames.insert({Key, RealName});
671  }
672  }
673  return Error::success();
674  }
675 
678  StringRef RealName = extractName(FuncName);
679  if (auto Key = Remappings.lookup(RealName)) {
680  StringRef Remapped = MappedNames.lookup(Key);
681  if (!Remapped.empty()) {
682  if (RealName.begin() == FuncName.begin() &&
683  RealName.end() == FuncName.end())
684  FuncName = Remapped;
685  else {
686  // Try rebuilding the name from the given remapping.
687  SmallString<256> Reconstituted;
688  reconstituteName(FuncName, RealName, Remapped, Reconstituted);
689  Error E = Underlying.getRecords(Reconstituted, Data);
690  if (!E)
691  return E;
692 
693  // If we failed because the name doesn't exist, fall back to asking
694  // about the original name.
695  if (Error Unhandled = handleErrors(
696  std::move(E), [](std::unique_ptr<InstrProfError> Err) {
697  return Err->get() == instrprof_error::unknown_function
698  ? Error::success()
699  : Error(std::move(Err));
700  }))
701  return Unhandled;
702  }
703  }
704  }
705  return Underlying.getRecords(FuncName, Data);
706  }
707 
708 private:
709  /// The memory buffer containing the remapping configuration. Remappings
710  /// holds pointers into this buffer.
711  std::unique_ptr<MemoryBuffer> RemapBuffer;
712 
713  /// The mangling remapper.
714  SymbolRemappingReader Remappings;
715 
716  /// Mapping from mangled name keys to the name used for the key in the
717  /// profile data.
718  /// FIXME: Can we store a location within the on-disk hash table instead of
719  /// redoing lookup?
721 
722  /// The real profile data reader.
724 };
725 
727  using namespace support;
728 
729  if (DataBuffer.getBufferSize() < 8)
730  return false;
731  uint64_t Magic =
732  endian::read<uint64_t, little, aligned>(DataBuffer.getBufferStart());
733  // Verify that it's magical.
734  return Magic == IndexedInstrProf::Magic;
735 }
736 
737 const unsigned char *
738 IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
739  const unsigned char *Cur, bool UseCS) {
740  using namespace IndexedInstrProf;
741  using namespace support;
742 
744  const IndexedInstrProf::Summary *SummaryInLE =
745  reinterpret_cast<const IndexedInstrProf::Summary *>(Cur);
746  uint64_t NFields =
747  endian::byte_swap<uint64_t, little>(SummaryInLE->NumSummaryFields);
748  uint64_t NEntries =
749  endian::byte_swap<uint64_t, little>(SummaryInLE->NumCutoffEntries);
750  uint32_t SummarySize =
751  IndexedInstrProf::Summary::getSize(NFields, NEntries);
752  std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
753  IndexedInstrProf::allocSummary(SummarySize);
754 
755  const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE);
756  uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get());
757  for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
758  Dst[I] = endian::byte_swap<uint64_t, little>(Src[I]);
759 
760  SummaryEntryVector DetailedSummary;
761  for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) {
762  const IndexedInstrProf::Summary::Entry &Ent = SummaryData->getEntry(I);
763  DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount,
764  Ent.NumBlocks);
765  }
766  std::unique_ptr<llvm::ProfileSummary> &Summary =
767  UseCS ? this->CS_Summary : this->Summary;
768 
769  // initialize InstrProfSummary using the SummaryData from disk.
770  Summary = llvm::make_unique<ProfileSummary>(
772  DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
773  SummaryData->get(Summary::MaxBlockCount),
774  SummaryData->get(Summary::MaxInternalBlockCount),
775  SummaryData->get(Summary::MaxFunctionCount),
776  SummaryData->get(Summary::TotalNumBlocks),
777  SummaryData->get(Summary::TotalNumFunctions));
778  return Cur + SummarySize;
779  } else {
780  // For older version of profile data, we need to compute on the fly:
781  using namespace IndexedInstrProf;
782 
784  // FIXME: This only computes an empty summary. Need to call addRecord for
785  // all NamedInstrProfRecords to get the correct summary.
786  this->Summary = Builder.getSummary();
787  return Cur;
788  }
789 }
790 
792  using namespace support;
793 
794  const unsigned char *Start =
795  (const unsigned char *)DataBuffer->getBufferStart();
796  const unsigned char *Cur = Start;
797  if ((const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
799 
800  auto *Header = reinterpret_cast<const IndexedInstrProf::Header *>(Cur);
801  Cur += sizeof(IndexedInstrProf::Header);
802 
803  // Check the magic number.
804  uint64_t Magic = endian::byte_swap<uint64_t, little>(Header->Magic);
805  if (Magic != IndexedInstrProf::Magic)
807 
808  // Read the version.
809  uint64_t FormatVersion = endian::byte_swap<uint64_t, little>(Header->Version);
810  if (GET_VERSION(FormatVersion) >
813 
814  Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur,
815  /* UseCS */ false);
816  if (FormatVersion & VARIANT_MASK_CSIR_PROF)
817  Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur,
818  /* UseCS */ true);
819 
820  // Read the hash type and start offset.
822  endian::byte_swap<uint64_t, little>(Header->HashType));
823  if (HashType > IndexedInstrProf::HashT::Last)
825 
826  uint64_t HashOffset = endian::byte_swap<uint64_t, little>(Header->HashOffset);
827 
828  // The rest of the file is an on disk hash table.
829  auto IndexPtr =
830  llvm::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
831  Start + HashOffset, Cur, Start, HashType, FormatVersion);
832 
833  // Load the remapping table now if requested.
834  if (RemappingBuffer) {
835  Remapper = llvm::make_unique<
837  std::move(RemappingBuffer), *IndexPtr);
838  if (Error E = Remapper->populateRemappings())
839  return E;
840  } else {
841  Remapper = llvm::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
842  }
843  Index = std::move(IndexPtr);
844 
845  return success();
846 }
847 
849  if (Symtab.get())
850  return *Symtab.get();
851 
852  std::unique_ptr<InstrProfSymtab> NewSymtab = make_unique<InstrProfSymtab>();
853  if (Error E = Index->populateSymtab(*NewSymtab.get())) {
855  }
856 
857  Symtab = std::move(NewSymtab);
858  return *Symtab.get();
859 }
860 
863  uint64_t FuncHash) {
865  Error Err = Remapper->getRecords(FuncName, Data);
866  if (Err)
867  return std::move(Err);
868  // Found it. Look for counters with the right hash.
869  for (unsigned I = 0, E = Data.size(); I < E; ++I) {
870  // Check for a match and fill the vector if there is one.
871  if (Data[I].Hash == FuncHash) {
872  return std::move(Data[I]);
873  }
874  }
876 }
877 
879  uint64_t FuncHash,
880  std::vector<uint64_t> &Counts) {
881  Expected<InstrProfRecord> Record = getInstrProfRecord(FuncName, FuncHash);
882  if (Error E = Record.takeError())
883  return error(std::move(E));
884 
885  Counts = Record.get().Counts;
886  return success();
887 }
888 
891 
892  Error E = Index->getRecords(Data);
893  if (E)
894  return error(std::move(E));
895 
896  Record = Data[RecordIndex++];
897  if (RecordIndex >= Data.size()) {
898  Index->advanceToNextKey();
899  RecordIndex = 0;
900  }
901  return success();
902 }
903 
905  uint64_t NumFuncs = 0;
906  for (const auto &Func : *this) {
907  if (isIRLevelProfile()) {
908  bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(Func.Hash);
909  if (FuncIsCS != IsCS)
910  continue;
911  }
912  Func.accumuateCounts(Sum);
913  ++NumFuncs;
914  }
915  Sum.NumEntries = NumFuncs;
916 }
#define VP_READ_ADVANCE(Val)
Represents either an error or a value T.
Definition: ErrorOr.h:56
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:410
Error readHeader() override
Read the header. Required before reading first record.
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
Definition: InstrProf.h:1041
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
void accumuateCounts(CountSumOrPercent &Sum, bool IsCS)
Compute the sum of counts and return in Sum.
bool isPrint(char C)
Checks whether character C is printable.
Definition: StringExtras.h:105
This class represents lattice values for constants.
Definition: AllocatorList.h:23
uint64_t MinBlockCount
The minimum execution count for this percentile.
Definition: InstrProf.h:1007
LLVM_NODISCARD bool equals_lower(StringRef RHS) const
equals_lower - Check for string equality, ignoring case.
Definition: StringRef.h:167
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
Definition: InstrProf.cpp:708
static bool hasFormat(const MemoryBuffer &DataBuffer)
Return true if the given buffer is in an indexed instrprof format.
static instrprof_error take(Error E)
Consume an Error and return the raw enum value contained within it.
Definition: InstrProf.h:323
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
Definition: InstrProf.h:507
A remapper that applies remappings based on a symbol remapping file.
Error error(instrprof_error Err)
Set the current error and return same.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1081
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:64
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1185
InstrProfSymtab & getSymtab() override
Return the PGO symtab.
size_t getBufferSize() const
Definition: MemoryBuffer.h:61
std::enable_if<!std::is_array< T >::value, std::unique_ptr< T > >::type make_unique(Args &&... args)
Constructs a new T() with the given args and returns a unique_ptr<T> which owns the object...
Definition: STLExtras.h:1403
Error takeError()
Take ownership of the stored error.
Definition: Error.h:552
void reserve(size_type N)
Definition: SmallVector.h:369
InstrProfLookupTrait::offset_type offset_type
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
Definition: InstrProf.h:920
Error readHeader() override
Read the header.
data_type ReadData(StringRef K, const unsigned char *D, offset_type N)
const uint64_t Magic
Definition: InstrProf.h:960
Definition: BitVector.h:937
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path)
Factory method to create an appropriately typed reader for the given instrprof file.
const HashT HashType
Definition: InstrProf.h:986
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
static StringRef getName(Value *V)
Tagged union holding either a T or a Error.
Definition: CachePruning.h:22
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:126
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name&#39;s MD5 hash.
Definition: InstrProf.h:485
Key
PAL metadata keys.
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if the given buffer is in text instrprof format.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:130
static StringRef extractName(StringRef Name)
Extract the original function name from a PGO function name.
const uint64_t Version
Definition: InstrProf.h:1099
Reader for the simple text based instrprof format.
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
Definition: InstrProf.h:1008
Error getRecords(StringRef FuncName, ArrayRef< NamedInstrProfRecord > &Data) override
static bool hasFormat(const MemoryBuffer &DataBuffer)
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(adl_begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1251
InstrProfLookupTrait::data_type data_type
hash_value_type ComputeHash(StringRef K)
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
ArrayRef< NamedInstrProfRecord > data_type
Base class and interface for reading profiling data of any known instrprof format.
uint64_t Cutoff
The required percentile of total execution count.
Definition: InstrProf.h:1005
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:87
ValueKind
Value kinds.
std::error_code getError() const
Definition: ErrorOr.h:159
bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)
std::unique_ptr< InstrProfSymtab > Symtab
std::underlying_type< E >::type Underlying(E Val)
Check that Val is in range for E, and return Val cast to E&#39;s underlying type.
Definition: BitmaskEnum.h:90
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:858
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:988
virtual Error getRecords(ArrayRef< NamedInstrProfRecord > &Data)=0
Error getRecords(ArrayRef< NamedInstrProfRecord > &Data) override
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:981
#define CHECK_LINE_END(Line)
#define READ_NUM(Str, Dst)
void clearValueData()
Clear value data entries.
Definition: InstrProf.h:772
static const char *const Magic
Definition: Archive.cpp:41
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Path)
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Name matcher supporting fuzzy matching of symbol names to names in profiles.
static ErrorSuccess success()
Create a success value.
Definition: Error.h:326
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
RawInstrProfReader< uint32_t > RawInstrProfReader32
std::unique_ptr< ProfileSummary > getSummary()
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:696
A file format agnostic iterator over profiling data.
Reader for the raw instrprof binary format from runtime.
reference get()
Returns a reference to the stored T value.
Definition: Error.h:532
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
RawInstrProfReader< uint64_t > RawInstrProfReader64
This interface provides simple read-only access to a block of memory, and provides simple methods for...
Definition: MemoryBuffer.h:41
InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)
InstrProfReaderItaniumRemapper(std::unique_ptr< MemoryBuffer > RemapBuffer, InstrProfReaderIndex< HashTableImpl > &Underlying)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:471
iterator begin() const
Definition: StringRef.h:101
unsigned char getSwappedBytes(unsigned char C)
Definition: SwapByteOrder.h:72
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileOrSTDIN(const Twine &Filename, int64_t FileSize=-1, bool RequiresNullTerminator=true)
Open the specified file as a MemoryBuffer, or open stdin if the Filename is "-".
std::vector< uint64_t > Counts
Definition: InstrProf.h:690
Error readHeader() override
Read the file header.
virtual Error readHeader()=0
Read the header. Required before reading first record.
Profiling information for a single function.
Definition: InstrProf.h:689
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
Reader for symbol remapping files.
static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl< char > &Out)
Given a mangled name extracted from a PGO function name, and a new form for that mangled name...
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
const char * getBufferStart() const
Definition: MemoryBuffer.h:59
virtual bool isIRLevelProfile() const =0
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
Provides ErrorOr<T> smart pointer.
LLVM Value Representation.
Definition: Value.h:72
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
Definition: Error.h:881
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
Reader for the indexed binary instrprof format.
std::vector< ProfileSummaryEntry > SummaryEntryVector
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, const Twine &RemappingPath="")
Factory method to create an indexed reader.
iterator end() const
Definition: StringRef.h:103
const uint64_t Version
Definition: InstrProf.h:984
void Clear()
Clear value data entries and edge counters.
Definition: InstrProf.h:766
reference get()
Definition: ErrorOr.h:156
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:143
Error success()
Clear the current error and return a successful one.
Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)
Fill Counts with the profile data for the given function name.
static Error initializeReader(InstrProfReader &Reader)