LLVM  16.0.0git
InstrProfReader.cpp
Go to the documentation of this file.
1 //===- InstrProfReader.cpp - Instrumented profiling reader ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains support for reading profiling data for clang's
10 // instrumentation based PGO and coverage.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/DenseMap.h"
17 #include "llvm/ADT/StringExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/IR/ProfileSummary.h"
23 #include "llvm/Support/Endian.h"
24 #include "llvm/Support/Error.h"
25 #include "llvm/Support/ErrorOr.h"
29 #include <algorithm>
30 #include <cstddef>
31 #include <cstdint>
32 #include <limits>
33 #include <memory>
34 #include <system_error>
35 #include <utility>
36 #include <vector>
37 
38 using namespace llvm;
39 
40 // Extracts the variant information from the top 8 bits in the version and
41 // returns an enum specifying the variants present.
44  if (Version & VARIANT_MASK_IR_PROF) {
46  }
47  if (Version & VARIANT_MASK_CSIR_PROF) {
49  }
50  if (Version & VARIANT_MASK_INSTR_ENTRY) {
52  }
53  if (Version & VARIANT_MASK_BYTE_COVERAGE) {
55  }
56  if (Version & VARIANT_MASK_FUNCTION_ENTRY_ONLY) {
58  }
59  if (Version & VARIANT_MASK_MEMPROF) {
61  }
62  return ProfileKind;
63 }
64 
66 setupMemoryBuffer(const Twine &Path) {
68  MemoryBuffer::getFileOrSTDIN(Path, /*IsText=*/true);
69  if (std::error_code EC = BufferOrErr.getError())
70  return errorCodeToError(EC);
71  return std::move(BufferOrErr.get());
72 }
73 
75  return Reader.readHeader();
76 }
77 
80  const InstrProfCorrelator *Correlator) {
81  // Set up the buffer to read.
82  auto BufferOrError = setupMemoryBuffer(Path);
83  if (Error E = BufferOrError.takeError())
84  return std::move(E);
85  return InstrProfReader::create(std::move(BufferOrError.get()), Correlator);
86 }
87 
89 InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
90  const InstrProfCorrelator *Correlator) {
91  // Sanity check the buffer.
92  if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<uint64_t>::max())
93  return make_error<InstrProfError>(instrprof_error::too_large);
94 
95  if (Buffer->getBufferSize() == 0)
96  return make_error<InstrProfError>(instrprof_error::empty_raw_profile);
97 
98  std::unique_ptr<InstrProfReader> Result;
99  // Create the reader.
101  Result.reset(new IndexedInstrProfReader(std::move(Buffer)));
102  else if (RawInstrProfReader64::hasFormat(*Buffer))
103  Result.reset(new RawInstrProfReader64(std::move(Buffer), Correlator));
104  else if (RawInstrProfReader32::hasFormat(*Buffer))
105  Result.reset(new RawInstrProfReader32(std::move(Buffer), Correlator));
106  else if (TextInstrProfReader::hasFormat(*Buffer))
107  Result.reset(new TextInstrProfReader(std::move(Buffer)));
108  else
109  return make_error<InstrProfError>(instrprof_error::unrecognized_format);
110 
111  // Initialize the reader and return the result.
112  if (Error E = initializeReader(*Result))
113  return std::move(E);
114 
115  return std::move(Result);
116 }
117 
119 IndexedInstrProfReader::create(const Twine &Path, const Twine &RemappingPath) {
120  // Set up the buffer to read.
121  auto BufferOrError = setupMemoryBuffer(Path);
122  if (Error E = BufferOrError.takeError())
123  return std::move(E);
124 
125  // Set up the remapping buffer if requested.
126  std::unique_ptr<MemoryBuffer> RemappingBuffer;
127  std::string RemappingPathStr = RemappingPath.str();
128  if (!RemappingPathStr.empty()) {
129  auto RemappingBufferOrError = setupMemoryBuffer(RemappingPathStr);
130  if (Error E = RemappingBufferOrError.takeError())
131  return std::move(E);
132  RemappingBuffer = std::move(RemappingBufferOrError.get());
133  }
134 
135  return IndexedInstrProfReader::create(std::move(BufferOrError.get()),
136  std::move(RemappingBuffer));
137 }
138 
140 IndexedInstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
141  std::unique_ptr<MemoryBuffer> RemappingBuffer) {
142  if (uint64_t(Buffer->getBufferSize()) > std::numeric_limits<uint64_t>::max())
143  return make_error<InstrProfError>(instrprof_error::too_large);
144 
145  // Create the reader.
146  if (!IndexedInstrProfReader::hasFormat(*Buffer))
147  return make_error<InstrProfError>(instrprof_error::bad_magic);
148  auto Result = std::make_unique<IndexedInstrProfReader>(
149  std::move(Buffer), std::move(RemappingBuffer));
150 
151  // Initialize the reader and return the result.
152  if (Error E = initializeReader(*Result))
153  return std::move(E);
154 
155  return std::move(Result);
156 }
157 
159  // Verify that this really looks like plain ASCII text by checking a
160  // 'reasonable' number of characters (up to profile magic size).
161  size_t count = std::min(Buffer.getBufferSize(), sizeof(uint64_t));
162  StringRef buffer = Buffer.getBufferStart();
163  return count == 0 ||
164  std::all_of(buffer.begin(), buffer.begin() + count,
165  [](char c) { return isPrint(c) || isSpace(c); });
166 }
167 
168 // Read the profile variant flag from the header: ":FE" means this is a FE
169 // generated profile. ":IR" means this is an IR level profile. Other strings
170 // with a leading ':' will be reported an error format.
172  Symtab.reset(new InstrProfSymtab());
173 
174  while (Line->startswith(":")) {
175  StringRef Str = Line->substr(1);
176  if (Str.equals_insensitive("ir"))
177  ProfileKind |= InstrProfKind::IRInstrumentation;
178  else if (Str.equals_insensitive("fe"))
180  else if (Str.equals_insensitive("csir")) {
181  ProfileKind |= InstrProfKind::IRInstrumentation;
182  ProfileKind |= InstrProfKind::ContextSensitive;
183  } else if (Str.equals_insensitive("entry_first"))
185  else if (Str.equals_insensitive("not_entry_first"))
186  ProfileKind &= ~InstrProfKind::FunctionEntryInstrumentation;
187  else
189  ++Line;
190  }
191  return success();
192 }
193 
194 Error
195 TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) {
196 
197 #define CHECK_LINE_END(Line) \
198  if (Line.is_at_end()) \
199  return error(instrprof_error::truncated);
200 #define READ_NUM(Str, Dst) \
201  if ((Str).getAsInteger(10, (Dst))) \
202  return error(instrprof_error::malformed);
203 #define VP_READ_ADVANCE(Val) \
204  CHECK_LINE_END(Line); \
205  uint32_t Val; \
206  READ_NUM((*Line), (Val)); \
207  Line++;
208 
209  if (Line.is_at_end())
210  return success();
211 
212  uint32_t NumValueKinds;
213  if (Line->getAsInteger(10, NumValueKinds)) {
214  // No value profile data
215  return success();
216  }
217  if (NumValueKinds == 0 || NumValueKinds > IPVK_Last + 1)
219  "number of value kinds is invalid");
220  Line++;
221 
222  for (uint32_t VK = 0; VK < NumValueKinds; VK++) {
224  if (ValueKind > IPVK_Last)
225  return error(instrprof_error::malformed, "value kind is invalid");
226  ;
227  VP_READ_ADVANCE(NumValueSites);
228  if (!NumValueSites)
229  continue;
230 
231  Record.reserveSites(VK, NumValueSites);
232  for (uint32_t S = 0; S < NumValueSites; S++) {
233  VP_READ_ADVANCE(NumValueData);
234 
235  std::vector<InstrProfValueData> CurrentValues;
236  for (uint32_t V = 0; V < NumValueData; V++) {
237  CHECK_LINE_END(Line);
238  std::pair<StringRef, StringRef> VD = Line->rsplit(':');
239  uint64_t TakenCount, Value;
240  if (ValueKind == IPVK_IndirectCallTarget) {
241  if (InstrProfSymtab::isExternalSymbol(VD.first)) {
242  Value = 0;
243  } else {
244  if (Error E = Symtab->addFuncName(VD.first))
245  return E;
247  }
248  } else {
249  READ_NUM(VD.first, Value);
250  }
251  READ_NUM(VD.second, TakenCount);
252  CurrentValues.push_back({Value, TakenCount});
253  Line++;
254  }
255  Record.addValueData(ValueKind, S, CurrentValues.data(), NumValueData,
256  nullptr);
257  }
258  }
259  return success();
260 
261 #undef CHECK_LINE_END
262 #undef READ_NUM
263 #undef VP_READ_ADVANCE
264 }
265 
267  // Skip empty lines and comments.
268  while (!Line.is_at_end() && (Line->empty() || Line->startswith("#")))
269  ++Line;
270  // If we hit EOF while looking for a name, we're done.
271  if (Line.is_at_end()) {
272  return error(instrprof_error::eof);
273  }
274 
275  // Read the function name.
276  Record.Name = *Line++;
277  if (Error E = Symtab->addFuncName(Record.Name))
278  return error(std::move(E));
279 
280  // Read the function hash.
281  if (Line.is_at_end())
283  if ((Line++)->getAsInteger(0, Record.Hash))
285  "function hash is not a valid integer");
286 
287  // Read the number of counters.
288  uint64_t NumCounters;
289  if (Line.is_at_end())
291  if ((Line++)->getAsInteger(10, NumCounters))
293  "number of counters is not a valid integer");
294  if (NumCounters == 0)
295  return error(instrprof_error::malformed, "number of counters is zero");
296 
297  // Read each counter and fill our internal storage with the values.
298  Record.Clear();
299  Record.Counts.reserve(NumCounters);
300  for (uint64_t I = 0; I < NumCounters; ++I) {
301  if (Line.is_at_end())
303  uint64_t Count;
304  if ((Line++)->getAsInteger(10, Count))
305  return error(instrprof_error::malformed, "count is invalid");
306  Record.Counts.push_back(Count);
307  }
308 
309  // Check if value profile data exists and read it if so.
310  if (Error E = readValueProfileData(Record))
311  return error(std::move(E));
312 
313  return success();
314 }
315 
316 template <class IntPtrT>
319 }
320 
321 template <class IntPtrT>
323  if (DataBuffer.getBufferSize() < sizeof(uint64_t))
324  return false;
325  uint64_t Magic =
326  *reinterpret_cast<const uint64_t *>(DataBuffer.getBufferStart());
327  return RawInstrProf::getMagic<IntPtrT>() == Magic ||
328  sys::getSwappedBytes(RawInstrProf::getMagic<IntPtrT>()) == Magic;
329 }
330 
331 template <class IntPtrT>
333  if (!hasFormat(*DataBuffer))
335  if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header))
337  auto *Header = reinterpret_cast<const RawInstrProf::Header *>(
338  DataBuffer->getBufferStart());
339  ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic<IntPtrT>();
340  return readHeader(*Header);
341 }
342 
343 template <class IntPtrT>
344 Error RawInstrProfReader<IntPtrT>::readNextHeader(const char *CurrentPos) {
345  const char *End = DataBuffer->getBufferEnd();
346  // Skip zero padding between profiles.
347  while (CurrentPos != End && *CurrentPos == 0)
348  ++CurrentPos;
349  // If there's nothing left, we're done.
350  if (CurrentPos == End)
351  return make_error<InstrProfError>(instrprof_error::eof);
352  // If there isn't enough space for another header, this is probably just
353  // garbage at the end of the file.
354  if (CurrentPos + sizeof(RawInstrProf::Header) > End)
355  return make_error<InstrProfError>(instrprof_error::malformed,
356  "not enough space for another header");
357  // The writer ensures each profile is padded to start at an aligned address.
358  if (reinterpret_cast<size_t>(CurrentPos) % alignof(uint64_t))
359  return make_error<InstrProfError>(instrprof_error::malformed,
360  "insufficient padding");
361  // The magic should have the same byte order as in the previous header.
362  uint64_t Magic = *reinterpret_cast<const uint64_t *>(CurrentPos);
363  if (Magic != swap(RawInstrProf::getMagic<IntPtrT>()))
364  return make_error<InstrProfError>(instrprof_error::bad_magic);
365 
366  // There's another profile to read, so we need to process the header.
367  auto *Header = reinterpret_cast<const RawInstrProf::Header *>(CurrentPos);
368  return readHeader(*Header);
369 }
370 
371 template <class IntPtrT>
373  if (Error E = Symtab.create(StringRef(NamesStart, NamesEnd - NamesStart)))
374  return error(std::move(E));
375  for (const RawInstrProf::ProfileData<IntPtrT> *I = Data; I != DataEnd; ++I) {
376  const IntPtrT FPtr = swap(I->FunctionPointer);
377  if (!FPtr)
378  continue;
379  Symtab.mapAddress(FPtr, I->NameRef);
380  }
381  return success();
382 }
383 
384 template <class IntPtrT>
386  const RawInstrProf::Header &Header) {
387  Version = swap(Header.Version);
388  if (GET_VERSION(Version) != RawInstrProf::Version)
390  if (useDebugInfoCorrelate() && !Correlator)
392  if (!useDebugInfoCorrelate() && Correlator)
394 
395  BinaryIdsSize = swap(Header.BinaryIdsSize);
396  if (BinaryIdsSize % sizeof(uint64_t))
398 
399  CountersDelta = swap(Header.CountersDelta);
400  NamesDelta = swap(Header.NamesDelta);
401  auto NumData = swap(Header.DataSize);
402  auto PaddingBytesBeforeCounters = swap(Header.PaddingBytesBeforeCounters);
403  auto CountersSize = swap(Header.CountersSize) * getCounterTypeSize();
404  auto PaddingBytesAfterCounters = swap(Header.PaddingBytesAfterCounters);
405  auto NamesSize = swap(Header.NamesSize);
406  ValueKindLast = swap(Header.ValueKindLast);
407 
408  auto DataSize = NumData * sizeof(RawInstrProf::ProfileData<IntPtrT>);
409  auto PaddingSize = getNumPaddingBytes(NamesSize);
410 
411  // Profile data starts after profile header and binary ids if exist.
412  ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdsSize;
413  ptrdiff_t CountersOffset = DataOffset + DataSize + PaddingBytesBeforeCounters;
414  ptrdiff_t NamesOffset =
415  CountersOffset + CountersSize + PaddingBytesAfterCounters;
416  ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize;
417 
418  auto *Start = reinterpret_cast<const char *>(&Header);
419  if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
421 
422  if (Correlator) {
423  // These sizes in the raw file are zero because we constructed them in the
424  // Correlator.
425  assert(DataSize == 0 && NamesSize == 0);
426  assert(CountersDelta == 0 && NamesDelta == 0);
427  Data = Correlator->getDataPointer();
428  DataEnd = Data + Correlator->getDataSize();
429  NamesStart = Correlator->getNamesPointer();
430  NamesEnd = NamesStart + Correlator->getNamesSize();
431  } else {
432  Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
433  Start + DataOffset);
434  DataEnd = Data + NumData;
435  NamesStart = Start + NamesOffset;
436  NamesEnd = NamesStart + NamesSize;
437  }
438 
439  // Binary ids start just after the header.
440  BinaryIdsStart =
441  reinterpret_cast<const uint8_t *>(&Header) + sizeof(RawInstrProf::Header);
442  CountersStart = Start + CountersOffset;
443  CountersEnd = CountersStart + CountersSize;
444  ValueDataStart = reinterpret_cast<const uint8_t *>(Start + ValueDataOffset);
445 
446  const uint8_t *BufferEnd = (const uint8_t *)DataBuffer->getBufferEnd();
447  if (BinaryIdsStart + BinaryIdsSize > BufferEnd)
449 
450  std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
451  if (Error E = createSymtab(*NewSymtab))
452  return E;
453 
454  Symtab = std::move(NewSymtab);
455  return success();
456 }
457 
458 template <class IntPtrT>
460  Record.Name = getName(Data->NameRef);
461  return success();
462 }
463 
464 template <class IntPtrT>
466  Record.Hash = swap(Data->FuncHash);
467  return success();
468 }
469 
470 template <class IntPtrT>
473  uint32_t NumCounters = swap(Data->NumCounters);
474  if (NumCounters == 0)
475  return error(instrprof_error::malformed, "number of counters is zero");
476 
477  ptrdiff_t CounterBaseOffset = swap(Data->CounterPtr) - CountersDelta;
478  if (CounterBaseOffset < 0)
479  return error(
481  ("counter offset " + Twine(CounterBaseOffset) + " is negative").str());
482 
483  if (CounterBaseOffset >= CountersEnd - CountersStart)
485  ("counter offset " + Twine(CounterBaseOffset) +
486  " is greater than the maximum counter offset " +
487  Twine(CountersEnd - CountersStart - 1))
488  .str());
489 
490  uint64_t MaxNumCounters =
491  (CountersEnd - (CountersStart + CounterBaseOffset)) /
492  getCounterTypeSize();
493  if (NumCounters > MaxNumCounters)
495  ("number of counters " + Twine(NumCounters) +
496  " is greater than the maximum number of counters " +
497  Twine(MaxNumCounters))
498  .str());
499 
500  Record.Counts.clear();
501  Record.Counts.reserve(NumCounters);
502  for (uint32_t I = 0; I < NumCounters; I++) {
503  const char *Ptr =
504  CountersStart + CounterBaseOffset + I * getCounterTypeSize();
505  if (hasSingleByteCoverage()) {
506  // A value of zero signifies the block is covered.
507  Record.Counts.push_back(*Ptr == 0 ? 1 : 0);
508  } else {
509  const auto *CounterValue = reinterpret_cast<const uint64_t *>(Ptr);
510  Record.Counts.push_back(swap(*CounterValue));
511  }
512  }
513 
514  return success();
515 }
516 
517 template <class IntPtrT>
520  Record.clearValueData();
521  CurValueDataSize = 0;
522  // Need to match the logic in value profile dumper code in compiler-rt:
523  uint32_t NumValueKinds = 0;
524  for (uint32_t I = 0; I < IPVK_Last + 1; I++)
525  NumValueKinds += (Data->NumValueSites[I] != 0);
526 
527  if (!NumValueKinds)
528  return success();
529 
531  ValueProfData::getValueProfData(
532  ValueDataStart, (const unsigned char *)DataBuffer->getBufferEnd(),
533  getDataEndianness());
534 
535  if (Error E = VDataPtrOrErr.takeError())
536  return E;
537 
538  // Note that besides deserialization, this also performs the conversion for
539  // indirect call targets. The function pointers from the raw profile are
540  // remapped into function name hashes.
541  VDataPtrOrErr.get()->deserializeTo(Record, Symtab.get());
542  CurValueDataSize = VDataPtrOrErr.get()->getSize();
543  return success();
544 }
545 
546 template <class IntPtrT>
548  // Keep reading profiles that consist of only headers and no profile data and
549  // counters.
550  while (atEnd())
551  // At this point, ValueDataStart field points to the next header.
552  if (Error E = readNextHeader(getNextHeaderPos()))
553  return error(std::move(E));
554 
555  // Read name ad set it in Record.
556  if (Error E = readName(Record))
557  return error(std::move(E));
558 
559  // Read FuncHash and set it in Record.
560  if (Error E = readFuncHash(Record))
561  return error(std::move(E));
562 
563  // Read raw counts and set Record.
564  if (Error E = readRawCounts(Record))
565  return error(std::move(E));
566 
567  // Read value data and set Record.
568  if (Error E = readValueProfilingData(Record))
569  return error(std::move(E));
570 
571  // Iterate.
572  advanceData();
573  return success();
574 }
575 
576 static size_t RoundUp(size_t size, size_t align) {
577  return (size + align - 1) & ~(align - 1);
578 }
579 
580 template <class IntPtrT>
582  if (BinaryIdsSize == 0)
583  return success();
584 
585  OS << "Binary IDs: \n";
586  const uint8_t *BI = BinaryIdsStart;
587  const uint8_t *BIEnd = BinaryIdsStart + BinaryIdsSize;
588  while (BI < BIEnd) {
589  size_t Remaining = BIEnd - BI;
590 
591  // There should be enough left to read the binary ID size field.
592  if (Remaining < sizeof(uint64_t))
593  return make_error<InstrProfError>(
595  "not enough data to read binary id length");
596 
597  uint64_t BinaryIdLen = swap(*reinterpret_cast<const uint64_t *>(BI));
598 
599  // There should be enough left to read the binary ID size field, and the
600  // binary ID.
601  if (Remaining < sizeof(BinaryIdLen) + BinaryIdLen)
602  return make_error<InstrProfError>(
603  instrprof_error::malformed, "not enough data to read binary id data");
604 
605  // Increment by binary id length data type size.
606  BI += sizeof(BinaryIdLen);
607  if (BI > (const uint8_t *)DataBuffer->getBufferEnd())
608  return make_error<InstrProfError>(
610  "binary id that is read is bigger than buffer size");
611 
612  for (uint64_t I = 0; I < BinaryIdLen; I++)
613  OS << format("%02x", BI[I]);
614  OS << "\n";
615 
616  // Increment by binary id data length, rounded to the next 8 bytes. This
617  // accounts for the zero-padding after each build ID.
618  BI += RoundUp(BinaryIdLen, sizeof(uint64_t));
619  if (BI > (const uint8_t *)DataBuffer->getBufferEnd())
620  return make_error<InstrProfError>(instrprof_error::malformed);
621  }
622 
623  return success();
624 }
625 
626 namespace llvm {
627 
628 template class RawInstrProfReader<uint32_t>;
629 template class RawInstrProfReader<uint64_t>;
630 
631 } // end namespace llvm
632 
635  return IndexedInstrProf::ComputeHash(HashType, K);
636 }
637 
640 
642  const unsigned char *&D, const unsigned char *const End) {
644  ValueProfData::getValueProfData(D, End, ValueProfDataEndianness);
645 
646  if (VDataPtrOrErr.takeError())
647  return false;
648 
649  VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), nullptr);
650  D += VDataPtrOrErr.get()->TotalSize;
651 
652  return true;
653 }
654 
656  offset_type N) {
657  using namespace support;
658 
659  // Check if the data is corrupt. If so, don't try to read it.
660  if (N % sizeof(uint64_t))
661  return data_type();
662 
663  DataBuffer.clear();
664  std::vector<uint64_t> CounterBuffer;
665 
666  const unsigned char *End = D + N;
667  while (D < End) {
668  // Read hash.
669  if (D + sizeof(uint64_t) >= End)
670  return data_type();
671  uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D);
672 
673  // Initialize number of counters for GET_VERSION(FormatVersion) == 1.
674  uint64_t CountsSize = N / sizeof(uint64_t) - 1;
675  // If format version is different then read the number of counters.
676  if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) {
677  if (D + sizeof(uint64_t) > End)
678  return data_type();
679  CountsSize = endian::readNext<uint64_t, little, unaligned>(D);
680  }
681  // Read counter values.
682  if (D + CountsSize * sizeof(uint64_t) > End)
683  return data_type();
684 
685  CounterBuffer.clear();
686  CounterBuffer.reserve(CountsSize);
687  for (uint64_t J = 0; J < CountsSize; ++J)
688  CounterBuffer.push_back(endian::readNext<uint64_t, little, unaligned>(D));
689 
690  DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer));
691 
692  // Read value profiling data.
693  if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 &&
694  !readValueProfilingData(D, End)) {
695  DataBuffer.clear();
696  return data_type();
697  }
698  }
699  return DataBuffer;
700 }
701 
702 template <typename HashTableImpl>
705  auto Iter = HashTable->find(FuncName);
706  if (Iter == HashTable->end())
707  return make_error<InstrProfError>(instrprof_error::unknown_function);
708 
709  Data = (*Iter);
710  if (Data.empty())
711  return make_error<InstrProfError>(instrprof_error::malformed,
712  "profile data is empty");
713 
714  return Error::success();
715 }
716 
717 template <typename HashTableImpl>
720  if (atEnd())
721  return make_error<InstrProfError>(instrprof_error::eof);
722 
723  Data = *RecordIterator;
724 
725  if (Data.empty())
726  return make_error<InstrProfError>(instrprof_error::malformed,
727  "profile data is empty");
728 
729  return Error::success();
730 }
731 
732 template <typename HashTableImpl>
734  const unsigned char *Buckets, const unsigned char *const Payload,
735  const unsigned char *const Base, IndexedInstrProf::HashT HashType,
736  uint64_t Version) {
737  FormatVersion = Version;
738  HashTable.reset(HashTableImpl::Create(
739  Buckets, Payload, Base,
740  typename HashTableImpl::InfoType(HashType, Version)));
741  RecordIterator = HashTable->data_begin();
742 }
743 
744 template <typename HashTableImpl>
746  return getProfileKindFromVersion(FormatVersion);
747 }
748 
749 namespace {
750 /// A remapper that does not apply any remappings.
751 class InstrProfReaderNullRemapper : public InstrProfReaderRemapper {
753 
754 public:
755  InstrProfReaderNullRemapper(InstrProfReaderIndexBase &Underlying)
756  : Underlying(Underlying) {}
757 
758  Error getRecords(StringRef FuncName,
760  return Underlying.getRecords(FuncName, Data);
761  }
762 };
763 } // namespace
764 
765 /// A remapper that applies remappings based on a symbol remapping file.
766 template <typename HashTableImpl>
768  : public InstrProfReaderRemapper {
769 public:
771  std::unique_ptr<MemoryBuffer> RemapBuffer,
773  : RemapBuffer(std::move(RemapBuffer)), Underlying(Underlying) {
774  }
775 
776  /// Extract the original function name from a PGO function name.
778  // We can have multiple :-separated pieces; there can be pieces both
779  // before and after the mangled name. Find the first part that starts
780  // with '_Z'; we'll assume that's the mangled name we want.
781  std::pair<StringRef, StringRef> Parts = {StringRef(), Name};
782  while (true) {
783  Parts = Parts.second.split(':');
784  if (Parts.first.startswith("_Z"))
785  return Parts.first;
786  if (Parts.second.empty())
787  return Name;
788  }
789  }
790 
791  /// Given a mangled name extracted from a PGO function name, and a new
792  /// form for that mangled name, reconstitute the name.
793  static void reconstituteName(StringRef OrigName, StringRef ExtractedName,
794  StringRef Replacement,
795  SmallVectorImpl<char> &Out) {
796  Out.reserve(OrigName.size() + Replacement.size() - ExtractedName.size());
797  Out.insert(Out.end(), OrigName.begin(), ExtractedName.begin());
798  Out.insert(Out.end(), Replacement.begin(), Replacement.end());
799  Out.insert(Out.end(), ExtractedName.end(), OrigName.end());
800  }
801 
803  if (Error E = Remappings.read(*RemapBuffer))
804  return E;
805  for (StringRef Name : Underlying.HashTable->keys()) {
806  StringRef RealName = extractName(Name);
807  if (auto Key = Remappings.insert(RealName)) {
808  // FIXME: We could theoretically map the same equivalence class to
809  // multiple names in the profile data. If that happens, we should
810  // return NamedInstrProfRecords from all of them.
811  MappedNames.insert({Key, RealName});
812  }
813  }
814  return Error::success();
815  }
816 
819  StringRef RealName = extractName(FuncName);
820  if (auto Key = Remappings.lookup(RealName)) {
821  StringRef Remapped = MappedNames.lookup(Key);
822  if (!Remapped.empty()) {
823  if (RealName.begin() == FuncName.begin() &&
824  RealName.end() == FuncName.end())
825  FuncName = Remapped;
826  else {
827  // Try rebuilding the name from the given remapping.
828  SmallString<256> Reconstituted;
829  reconstituteName(FuncName, RealName, Remapped, Reconstituted);
830  Error E = Underlying.getRecords(Reconstituted, Data);
831  if (!E)
832  return E;
833 
834  // If we failed because the name doesn't exist, fall back to asking
835  // about the original name.
836  if (Error Unhandled = handleErrors(
837  std::move(E), [](std::unique_ptr<InstrProfError> Err) {
838  return Err->get() == instrprof_error::unknown_function
839  ? Error::success()
840  : Error(std::move(Err));
841  }))
842  return Unhandled;
843  }
844  }
845  }
846  return Underlying.getRecords(FuncName, Data);
847  }
848 
849 private:
850  /// The memory buffer containing the remapping configuration. Remappings
851  /// holds pointers into this buffer.
852  std::unique_ptr<MemoryBuffer> RemapBuffer;
853 
854  /// The mangling remapper.
855  SymbolRemappingReader Remappings;
856 
857  /// Mapping from mangled name keys to the name used for the key in the
858  /// profile data.
859  /// FIXME: Can we store a location within the on-disk hash table instead of
860  /// redoing lookup?
862 
863  /// The real profile data reader.
865 };
866 
868  using namespace support;
869 
870  if (DataBuffer.getBufferSize() < 8)
871  return false;
872  uint64_t Magic =
873  endian::read<uint64_t, little, aligned>(DataBuffer.getBufferStart());
874  // Verify that it's magical.
875  return Magic == IndexedInstrProf::Magic;
876 }
877 
878 const unsigned char *
879 IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
880  const unsigned char *Cur, bool UseCS) {
881  using namespace IndexedInstrProf;
882  using namespace support;
883 
885  const IndexedInstrProf::Summary *SummaryInLE =
886  reinterpret_cast<const IndexedInstrProf::Summary *>(Cur);
887  uint64_t NFields =
888  endian::byte_swap<uint64_t, little>(SummaryInLE->NumSummaryFields);
889  uint64_t NEntries =
890  endian::byte_swap<uint64_t, little>(SummaryInLE->NumCutoffEntries);
891  uint32_t SummarySize =
892  IndexedInstrProf::Summary::getSize(NFields, NEntries);
893  std::unique_ptr<IndexedInstrProf::Summary> SummaryData =
894  IndexedInstrProf::allocSummary(SummarySize);
895 
896  const uint64_t *Src = reinterpret_cast<const uint64_t *>(SummaryInLE);
897  uint64_t *Dst = reinterpret_cast<uint64_t *>(SummaryData.get());
898  for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
899  Dst[I] = endian::byte_swap<uint64_t, little>(Src[I]);
900 
901  SummaryEntryVector DetailedSummary;
902  for (unsigned I = 0; I < SummaryData->NumCutoffEntries; I++) {
903  const IndexedInstrProf::Summary::Entry &Ent = SummaryData->getEntry(I);
904  DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount,
905  Ent.NumBlocks);
906  }
907  std::unique_ptr<llvm::ProfileSummary> &Summary =
908  UseCS ? this->CS_Summary : this->Summary;
909 
910  // initialize InstrProfSummary using the SummaryData from disk.
911  Summary = std::make_unique<ProfileSummary>(
913  DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
914  SummaryData->get(Summary::MaxBlockCount),
915  SummaryData->get(Summary::MaxInternalBlockCount),
916  SummaryData->get(Summary::MaxFunctionCount),
917  SummaryData->get(Summary::TotalNumBlocks),
918  SummaryData->get(Summary::TotalNumFunctions));
919  return Cur + SummarySize;
920  } else {
921  // The older versions do not support a profile summary. This just computes
922  // an empty summary, which will not result in accurate hot/cold detection.
923  // We would need to call addRecord for all NamedInstrProfRecords to get the
924  // correct summary. However, this version is old (prior to early 2016) and
925  // has not been supporting an accurate summary for several years.
927  Summary = Builder.getSummary();
928  return Cur;
929  }
930 }
931 
933  using namespace support;
934 
935  const unsigned char *Start =
936  (const unsigned char *)DataBuffer->getBufferStart();
937  const unsigned char *Cur = Start;
938  if ((const unsigned char *)DataBuffer->getBufferEnd() - Cur < 24)
940 
941  auto HeaderOr = IndexedInstrProf::Header::readFromBuffer(Start);
942  if (!HeaderOr)
943  return HeaderOr.takeError();
944 
945  const IndexedInstrProf::Header *Header = &HeaderOr.get();
946  Cur += Header->size();
947 
948  Cur = readSummary((IndexedInstrProf::ProfVersion)Header->formatVersion(), Cur,
949  /* UseCS */ false);
950  if (Header->formatVersion() & VARIANT_MASK_CSIR_PROF)
951  Cur = readSummary((IndexedInstrProf::ProfVersion)Header->formatVersion(), Cur,
952  /* UseCS */ true);
953 
954  // Read the hash type and start offset.
955  IndexedInstrProf::HashT HashType = static_cast<IndexedInstrProf::HashT>(
956  endian::byte_swap<uint64_t, little>(Header->HashType));
957  if (HashType > IndexedInstrProf::HashT::Last)
959 
960  uint64_t HashOffset = endian::byte_swap<uint64_t, little>(Header->HashOffset);
961 
962  // The hash table with profile counts comes next.
963  auto IndexPtr = std::make_unique<InstrProfReaderIndex<OnDiskHashTableImplV3>>(
964  Start + HashOffset, Cur, Start, HashType, Header->formatVersion());
965 
966  // The MemProfOffset field in the header is only valid when the format version
967  // is higher than 8 (when it was introduced).
968  if (GET_VERSION(Header->formatVersion()) >= 8 &&
969  Header->formatVersion() & VARIANT_MASK_MEMPROF) {
970  uint64_t MemProfOffset =
971  endian::byte_swap<uint64_t, little>(Header->MemProfOffset);
972 
973  const unsigned char *Ptr = Start + MemProfOffset;
974  // The value returned from RecordTableGenerator.Emit.
975  const uint64_t RecordTableOffset =
976  support::endian::readNext<uint64_t, little, unaligned>(Ptr);
977  // The offset in the stream right before invoking FrameTableGenerator.Emit.
978  const uint64_t FramePayloadOffset =
979  support::endian::readNext<uint64_t, little, unaligned>(Ptr);
980  // The value returned from FrameTableGenerator.Emit.
981  const uint64_t FrameTableOffset =
982  support::endian::readNext<uint64_t, little, unaligned>(Ptr);
983 
984  // Read the schema.
985  auto SchemaOr = memprof::readMemProfSchema(Ptr);
986  if (!SchemaOr)
987  return SchemaOr.takeError();
988  Schema = SchemaOr.get();
989 
990  // Now initialize the table reader with a pointer into data buffer.
991  MemProfRecordTable.reset(MemProfRecordHashTable::Create(
992  /*Buckets=*/Start + RecordTableOffset,
993  /*Payload=*/Ptr,
994  /*Base=*/Start, memprof::RecordLookupTrait(Schema)));
995 
996  // Initialize the frame table reader with the payload and bucket offsets.
997  MemProfFrameTable.reset(MemProfFrameHashTable::Create(
998  /*Buckets=*/Start + FrameTableOffset,
999  /*Payload=*/Start + FramePayloadOffset,
1000  /*Base=*/Start, memprof::FrameLookupTrait()));
1001  }
1002 
1003  // Load the remapping table now if requested.
1004  if (RemappingBuffer) {
1005  Remapper = std::make_unique<
1007  std::move(RemappingBuffer), *IndexPtr);
1008  if (Error E = Remapper->populateRemappings())
1009  return E;
1010  } else {
1011  Remapper = std::make_unique<InstrProfReaderNullRemapper>(*IndexPtr);
1012  }
1013  Index = std::move(IndexPtr);
1014 
1015  return success();
1016 }
1017 
1019  if (Symtab)
1020  return *Symtab;
1021 
1022  std::unique_ptr<InstrProfSymtab> NewSymtab = std::make_unique<InstrProfSymtab>();
1023  if (Error E = Index->populateSymtab(*NewSymtab)) {
1025  }
1026 
1027  Symtab = std::move(NewSymtab);
1028  return *Symtab;
1029 }
1030 
1032  StringRef FuncName, uint64_t FuncHash, uint64_t *MismatchedFuncSum) {
1034  uint64_t FuncSum = 0;
1035  Error Err = Remapper->getRecords(FuncName, Data);
1036  if (Err)
1037  return std::move(Err);
1038  // Found it. Look for counters with the right hash.
1039 
1040  // A flag to indicate if the records are from the same type
1041  // of profile (i.e cs vs nocs).
1042  bool CSBitMatch = false;
1043  auto getFuncSum = [](const std::vector<uint64_t> &Counts) {
1044  uint64_t ValueSum = 0;
1045  for (uint64_t CountValue : Counts) {
1046  if (CountValue == (uint64_t)-1)
1047  continue;
1048  // Handle overflow -- if that happens, return max.
1049  if (std::numeric_limits<uint64_t>::max() - CountValue <= ValueSum)
1051  ValueSum += CountValue;
1052  }
1053  return ValueSum;
1054  };
1055 
1056  for (const NamedInstrProfRecord &I : Data) {
1057  // Check for a match and fill the vector if there is one.
1058  if (I.Hash == FuncHash)
1059  return std::move(I);
1062  CSBitMatch = true;
1063  if (MismatchedFuncSum == nullptr)
1064  continue;
1065  FuncSum = std::max(FuncSum, getFuncSum(I.Counts));
1066  }
1067  }
1068  if (CSBitMatch) {
1069  if (MismatchedFuncSum != nullptr)
1070  *MismatchedFuncSum = FuncSum;
1072  }
1074 }
1075 
1078  // TODO: Add memprof specific errors.
1079  if (MemProfRecordTable == nullptr)
1080  return make_error<InstrProfError>(instrprof_error::invalid_prof,
1081  "no memprof data available in profile");
1082  auto Iter = MemProfRecordTable->find(FuncNameHash);
1083  if (Iter == MemProfRecordTable->end())
1084  return make_error<InstrProfError>(
1086  "memprof record not found for function hash " + Twine(FuncNameHash));
1087 
1088  // Setup a callback to convert from frame ids to frame using the on-disk
1089  // FrameData hash table.
1090  memprof::FrameId LastUnmappedFrameId = 0;
1091  bool HasFrameMappingError = false;
1092  auto IdToFrameCallback = [&](const memprof::FrameId Id) {
1093  auto FrIter = MemProfFrameTable->find(Id);
1094  if (FrIter == MemProfFrameTable->end()) {
1095  LastUnmappedFrameId = Id;
1096  HasFrameMappingError = true;
1097  return memprof::Frame(0, 0, 0, false);
1098  }
1099  return *FrIter;
1100  };
1101 
1102  memprof::MemProfRecord Record(*Iter, IdToFrameCallback);
1103 
1104  // Check that all frame ids were successfully converted to frames.
1105  if (HasFrameMappingError) {
1106  return make_error<InstrProfError>(instrprof_error::hash_mismatch,
1107  "memprof frame not found for frame id " +
1108  Twine(LastUnmappedFrameId));
1109  }
1110  return Record;
1111 }
1112 
1114  uint64_t FuncHash,
1115  std::vector<uint64_t> &Counts) {
1116  Expected<InstrProfRecord> Record = getInstrProfRecord(FuncName, FuncHash);
1117  if (Error E = Record.takeError())
1118  return error(std::move(E));
1119 
1120  Counts = Record.get().Counts;
1121  return success();
1122 }
1123 
1126 
1127  Error E = Index->getRecords(Data);
1128  if (E)
1129  return error(std::move(E));
1130 
1131  Record = Data[RecordIndex++];
1132  if (RecordIndex >= Data.size()) {
1133  Index->advanceToNextKey();
1134  RecordIndex = 0;
1135  }
1136  return success();
1137 }
1138 
1140  uint64_t NumFuncs = 0;
1141  for (const auto &Func : *this) {
1142  if (isIRLevelProfile()) {
1143  bool FuncIsCS = NamedInstrProfRecord::hasCSFlagInHash(Func.Hash);
1144  if (FuncIsCS != IsCS)
1145  continue;
1146  }
1147  Func.accumulateCounts(Sum);
1148  ++NumFuncs;
1149  }
1150  Sum.NumEntries = NumFuncs;
1151 }
llvm::coverage::coveragemap_error::success
@ success
MemoryBuffer.h
llvm::IndexedInstrProfReader::getSymtab
InstrProfSymtab & getSymtab() override
Return the PGO symtab.
Definition: InstrProfReader.cpp:1018
llvm::instrprof_error::bad_header
@ bad_header
llvm::RawInstrProfReader
Reader for the raw instrprof binary format from runtime.
Definition: InstrProfReader.h:258
getName
static StringRef getName(Value *V)
Definition: ProvenanceAnalysisEvaluator.cpp:42
llvm::InstrProfReader::success
Error success()
Clear the current error and return a successful one.
Definition: InstrProfReader.h:161
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::IndexedInstrProf::ProfVersion
ProfVersion
Definition: InstrProf.h:1027
llvm::handleErrors
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
Definition: Error.h:943
llvm::InstrProfKind::FunctionEntryInstrumentation
@ FunctionEntryInstrumentation
llvm::InstrProfKind::FunctionEntryOnly
@ FunctionEntryOnly
offset_type
InstrProfLookupTrait::offset_type offset_type
Definition: InstrProfReader.cpp:639
llvm::IndexedInstrProfReader::hasFormat
static bool hasFormat(const MemoryBuffer &DataBuffer)
Return true if the given buffer is in an indexed instrprof format.
Definition: InstrProfReader.cpp:867
llvm::InstrProfLookupTrait::hash_value_type
uint64_t hash_value_type
Definition: InstrProfReader.h:418
llvm::RawInstrProfReader::getProfileKind
InstrProfKind getProfileKind() const override
Returns a BitsetEnum describing the attributes of the raw instr profile.
Definition: InstrProfReader.cpp:317
ProfileSummary.h
StringRef.h
llvm::IndexedInstrProf::Summary::Entry::NumBlocks
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
Definition: InstrProf.h:1095
llvm::IndexedInstrProf::ComputeHash
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1060
llvm::InstrProfReaderIndex::InstrProfReaderIndex
InstrProfReaderIndex(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, IndexedInstrProf::HashT HashType, uint64_t Version)
Definition: InstrProfReader.cpp:733
llvm::InstrProfKind::ContextSensitive
@ ContextSensitive
llvm::RawInstrProfReader64
RawInstrProfReader< uint64_t > RawInstrProfReader64
Definition: InstrProfReader.h:391
llvm::InstrProfSymtab::create
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
llvm::Error::success
static ErrorSuccess success()
Create a success value.
Definition: Error.h:329
llvm::IndexedInstrProf::Header::size
size_t size() const
Definition: InstrProf.cpp:1388
Error.h
SwapByteOrder.h
llvm::ARM::ProfileKind
ProfileKind
Definition: ARMTargetParser.h:169
llvm::IndexedInstrProf::Summary::Entry
Definition: InstrProf.h:1091
llvm::InstrProfLookupTrait::data_type
ArrayRef< NamedInstrProfRecord > data_type
Definition: InstrProfReader.h:414
DenseMap.h
llvm::InstrProfReaderIndex::getProfileKind
InstrProfKind getProfileKind() const override
Definition: InstrProfReader.cpp:745
llvm::InstrProfLookupTrait::ComputeHash
hash_value_type ComputeHash(StringRef K)
Definition: InstrProfReader.cpp:634
error
#define error(X)
Definition: SymbolRecordMapping.cpp:14
llvm::StringRef::substr
StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:561
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::Expected
Tagged union holding either a T or a Error.
Definition: APFloat.h:41
llvm::instrprof_error::invalid_prof
@ invalid_prof
ProfileCommon.h
llvm::IndexedInstrProf::Header::MemProfOffset
uint64_t MemProfOffset
Definition: InstrProf.h:1070
llvm::memprof::FrameLookupTrait
Definition: MemProf.h:575
llvm::consumeError
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1043
llvm::Data
@ Data
Definition: SIMachineScheduler.h:55
llvm::MemoryBuffer
This interface provides simple read-only access to a block of memory, and provides simple methods for...
Definition: MemoryBuffer.h:50
llvm::InstrProfRecord
Profiling information for a single function.
Definition: InstrProf.h:730
llvm::instrprof_error::unexpected_debug_info_for_correlation
@ unexpected_debug_info_for_correlation
llvm::TextInstrProfReader::readHeader
Error readHeader() override
Read the header.
Definition: InstrProfReader.cpp:171
llvm::IndexedInstrProf::Version4
@ Version4
Definition: InstrProf.h:1043
llvm::IndexedInstrProfReader::getInstrProfRecord
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
Definition: InstrProfReader.cpp:1031
llvm::AMDGPU::HSAMD::ValueKind
ValueKind
Value kinds.
Definition: AMDGPUMetadata.h:78
llvm::coverage::Version1
@ Version1
Definition: CoverageMapping.h:993
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1590
llvm::SymbolRemappingReader
Reader for symbol remapping files.
Definition: SymbolRemappingReader.h:98
llvm::StringRef::startswith
bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:256
ptrdiff_t
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::InstrProfKind::IRInstrumentation
@ IRInstrumentation
llvm::MemoryBuffer::getBufferSize
size_t getBufferSize() const
Definition: MemoryBuffer.h:67
llvm::InstrProfKind::SingleByteCoverage
@ SingleByteCoverage
llvm::memprof::RecordLookupTrait
Definition: MemProf.h:447
llvm::errorCodeToError
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:92
InfoType
InfoType
FunctionInfo information type that is used to encode the optional data that is associated with a Func...
Definition: FunctionInfo.cpp:21
setupMemoryBuffer
static Expected< std::unique_ptr< MemoryBuffer > > setupMemoryBuffer(const Twine &Path)
Definition: InstrProfReader.cpp:66
llvm::AMDGPU::PALMD::Key
Key
PAL metadata keys.
Definition: AMDGPUMetadata.h:486
llvm::ErrorOr::getError
std::error_code getError() const
Definition: ErrorOr.h:153
llvm::InstrProfReaderItaniumRemapper::InstrProfReaderItaniumRemapper
InstrProfReaderItaniumRemapper(std::unique_ptr< MemoryBuffer > RemapBuffer, InstrProfReaderIndex< HashTableImpl > &Underlying)
Definition: InstrProfReader.cpp:770
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
CHECK_LINE_END
#define CHECK_LINE_END(Line)
InstrProf.h
llvm::instrprof_error::truncated
@ truncated
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
llvm::IndexedInstrProfReader::getFunctionCounts
Error getFunctionCounts(StringRef FuncName, uint64_t FuncHash, std::vector< uint64_t > &Counts)
Fill Counts with the profile data for the given function name.
Definition: InstrProfReader.cpp:1113
llvm::IndexedInstrProf::Version
const uint64_t Version
Definition: InstrProf.h:1056
llvm::TextInstrProfReader::readNextRecord
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Definition: InstrProfReader.cpp:266
c
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical int int c
Definition: README.txt:418
llvm::IndexedInstrProf::Header
Definition: InstrProf.h:1064
llvm::instrprof_error::bad_magic
@ bad_magic
llvm::IndexedInstrProf::Header::readFromBuffer
static Expected< Header > readFromBuffer(const unsigned char *Buffer)
Definition: InstrProf.cpp:1351
llvm::instrprof_error::unsupported_version
@ unsupported_version
llvm::StringRef::getAsInteger
std::enable_if_t< std::numeric_limits< T >::is_signed, bool > getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:463
llvm::InstrProfReader::Symtab
std::unique_ptr< InstrProfSymtab > Symtab
Definition: InstrProfReader.h:141
RoundUp
static size_t RoundUp(size_t size, size_t align)
Definition: InstrProfReader.cpp:576
llvm::IndexedInstrProfReader::readHeader
Error readHeader() override
Read the file header.
Definition: InstrProfReader.cpp:932
llvm::instrprof_error::too_large
@ too_large
llvm::InstrProfError::take
static instrprof_error take(Error E)
Consume an Error and return the raw enum value contained within it.
Definition: InstrProf.h:358
llvm::SmallString< 256 >
llvm::memprof::readMemProfSchema
Expected< MemProfSchema > readMemProfSchema(const unsigned char *&Buffer)
Definition: MemProf.cpp:84
llvm::Twine::str
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
llvm::sys::getSwappedBytes
unsigned char getSwappedBytes(unsigned char C)
Definition: SwapByteOrder.h:103
llvm::StringRef::empty
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
llvm::instrprof_error::unsupported_hash_type
@ unsupported_hash_type
llvm::InstrProfReaderItaniumRemapper::extractName
static StringRef extractName(StringRef Name)
Extract the original function name from a PGO function name.
Definition: InstrProfReader.cpp:777
InstrProfReader.h
llvm::CountSumOrPercent::NumEntries
uint64_t NumEntries
Definition: InstrProf.h:637
llvm::InstrProfReaderItaniumRemapper::getRecords
Error getRecords(StringRef FuncName, ArrayRef< NamedInstrProfRecord > &Data) override
Definition: InstrProfReader.cpp:817
SymbolRemappingReader.h
llvm::memprof::Frame
Definition: MemProf.h:141
initializeReader
static Error initializeReader(InstrProfReader &Reader)
Definition: InstrProfReader.cpp:74
llvm::count
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1700
uint64_t
D
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
llvm::StringRef::end
iterator end() const
Definition: StringRef.h:113
llvm::BitmaskEnumDetail::Underlying
constexpr std::underlying_type_t< E > Underlying(E Val)
Check that Val is in range for E, and return Val cast to E's underlying type.
Definition: BitmaskEnum.h:90
llvm::IndexedInstrProfReader
Reader for the indexed binary instrprof format.
Definition: InstrProfReader.h:552
llvm::IndexedInstrProf::Summary::getSize
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
Definition: InstrProf.h:1128
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::DenseMap
Definition: DenseMap.h:714
ErrorOr.h
llvm::InstrProfSummaryBuilder
Definition: ProfileCommon.h:74
llvm::MemoryBuffer::getFileOrSTDIN
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileOrSTDIN(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true)
Open the specified file as a MemoryBuffer, or open stdin if the Filename is "-".
Definition: MemoryBuffer.cpp:146
llvm::CountSumOrPercent
Definition: InstrProf.h:636
llvm::TextInstrProfReader
Reader for the simple text based instrprof format.
Definition: InstrProfReader.h:195
llvm::instrprof_error::empty_raw_profile
@ empty_raw_profile
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::line_iterator::is_at_end
bool is_at_end() const
Return true if we're an "end" iterator or have reached EOF.
Definition: LineIterator.h:63
StringExtras.h
ArrayRef.h
llvm::InstrProfKind::FrontendInstrumentation
@ FrontendInstrumentation
llvm::IndexedInstrProf::Header::HashType
uint64_t HashType
Definition: InstrProf.h:1068
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::move
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1666
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
llvm::instrprof_error::unknown_function
@ unknown_function
llvm::IndexedInstrProf::HashT::Last
@ Last
llvm::instrprof_error::unrecognized_format
@ unrecognized_format
Ptr
@ Ptr
Definition: TargetLibraryInfo.cpp:60
llvm::IndexedInstrProf::allocSummary
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1168
llvm::Record
Definition: Record.h:1543
llvm::IndexedInstrProfReader::create
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Definition: InstrProfReader.cpp:119
llvm::InstrProfKind::MemProf
@ MemProf
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1571
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::InstrProfSymtab::isExternalSymbol
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
Definition: InstrProf.h:544
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::InstrProfReaderIndex
Definition: InstrProfReader.h:485
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::IndexedInstrProf::Summary::Entry::Cutoff
uint64_t Cutoff
The required percentile of total execution count.
Definition: InstrProf.h:1092
llvm::msf::Magic
static const char Magic[]
Definition: MSFCommon.h:23
llvm::IndexedInstrProf::Magic
const uint64_t Magic
Definition: InstrProf.h:1025
llvm::StringRef::rsplit
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Definition: StringRef.h:722
llvm::IndexedInstrProf::HashT
HashT
Definition: InstrProf.h:1012
llvm::NamedInstrProfRecord::hasCSFlagInHash
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:923
llvm::Expected::get
reference get()
Returns a reference to the stored T value.
Definition: Error.h:567
uint32_t
llvm::instrprof_error::eof
@ eof
llvm::IndexedInstrProf::Summary::NumCutoffEntries
uint64_t NumCutoffEntries
Definition: InstrProf.h:1121
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::RawInstrProfReader::readHeader
Error readHeader() override
Read the header. Required before reading first record.
Definition: InstrProfReader.cpp:332
llvm::format
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:124
llvm::ProfileSummary::PSK_CSInstr
@ PSK_CSInstr
Definition: ProfileSummary.h:47
llvm::RawInstrProfReader::hasFormat
static bool hasFormat(const MemoryBuffer &DataBuffer)
Definition: InstrProfReader.cpp:322
llvm::instrprof_error::hash_mismatch
@ hash_mismatch
llvm::SummaryEntryVector
std::vector< ProfileSummaryEntry > SummaryEntryVector
Definition: ProfileSummary.h:43
llvm::InstrProfReaderItaniumRemapper
A remapper that applies remappings based on a symbol remapping file.
Definition: InstrProfReader.h:482
VP_READ_ADVANCE
#define VP_READ_ADVANCE(Val)
llvm::StringRef::size
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
MemProf.h
llvm::InstrProfLookupTrait::ReadData
data_type ReadData(StringRef K, const unsigned char *D, offset_type N)
Definition: InstrProfReader.cpp:655
llvm::IndexedInstrProf::Header::formatVersion
uint64_t formatVersion() const
Definition: InstrProf.cpp:1346
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::memprof::MemProfRecord
Definition: MemProf.h:397
llvm::MemoryBuffer::getBufferStart
const char * getBufferStart() const
Definition: MemoryBuffer.h:65
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
std
Definition: BitVector.h:851
llvm::InstrProfSymtab
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:446
llvm::InstrProfReader::accumulateCounts
void accumulateCounts(CountSumOrPercent &Sum, bool IsCS)
Compute the sum of counts and return in Sum.
Definition: InstrProfReader.cpp:1139
llvm::Error
Lightweight error class with error context and mandatory checking.
Definition: Error.h:155
llvm::InstrProfReaderItaniumRemapper::reconstituteName
static void reconstituteName(StringRef OrigName, StringRef ExtractedName, StringRef Replacement, SmallVectorImpl< char > &Out)
Given a mangled name extracted from a PGO function name, and a new form for that mangled name,...
Definition: InstrProfReader.cpp:793
llvm::IndexedInstrProf::Summary
Definition: InstrProf.h:1090
READ_NUM
#define READ_NUM(Str, Dst)
llvm::IndexedInstrProfReader::getMemProfRecord
Expected< memprof::MemProfRecord > getMemProfRecord(uint64_t FuncNameHash)
Return the memprof record for the function identified by llvm::md5(Name).
Definition: InstrProfReader.cpp:1077
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::InstrProfReaderRemapper
Name matcher supporting fuzzy matching of symbol names to names in profiles.
Definition: InstrProfReader.h:543
llvm::InstrProfReader::create
static Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, const InstrProfCorrelator *Correlator=nullptr)
Factory method to create an appropriately typed reader for the given instrprof file.
Definition: InstrProfReader.cpp:79
llvm::InstrProfReader::readHeader
virtual Error readHeader()=0
Read the header. Required before reading first record.
llvm::RawInstrProfReader::readNextRecord
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Definition: InstrProfReader.cpp:547
llvm::InstrProfReaderIndexBase
Definition: InstrProfReader.h:450
llvm::RawInstrProf::ProfileData
Definition: InstrProf.h:1205
llvm::IndexedInstrProfReader::readNextRecord
Error readNextRecord(NamedInstrProfRecord &Record) override
Read a single record.
Definition: InstrProfReader.cpp:1124
llvm::IndexedInstrProf::Header::HashOffset
uint64_t HashOffset
Definition: InstrProf.h:1069
llvm::Expected::takeError
Error takeError()
Take ownership of the stored error.
Definition: Error.h:597
llvm::RawInstrProfReader::printBinaryIds
Error printBinaryIds(raw_ostream &OS) override
Print binary ids on stream OS.
Definition: InstrProfReader.cpp:581
llvm::InstrProfKind::Unknown
@ Unknown
align
mov r0 ldr L5 sub r0 lr needed for prologue ldmia ip add bx lr r2 The last stmia stores r2 into the address passed in there is one additional stmia that stores and r2 to some stack location The store is dead The llvm gcc generated code looks like align
Definition: README.txt:236
llvm::OnDiskIterableChainedHashTable::Create
static OnDiskIterableChainedHashTable * Create(const unsigned char *Buckets, const unsigned char *const Payload, const unsigned char *const Base, const Info &InfoObj=Info())
Create the hash table.
Definition: OnDiskHashTable.h:603
llvm::InstrProfSymtab::mapAddress
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Definition: InstrProf.h:522
llvm::ErrorOr::get
reference get()
Definition: ErrorOr.h:150
llvm::TextInstrProfReader::hasFormat
static bool hasFormat(const MemoryBuffer &Buffer)
Return true if the given buffer is in text instrprof format.
Definition: InstrProfReader.cpp:158
llvm::ProfileSummary::PSK_Instr
@ PSK_Instr
Definition: ProfileSummary.h:47
N
#define N
llvm::ErrorOr
Represents either an error or a value T.
Definition: ErrorOr.h:56
llvm::InstrProfKind
InstrProfKind
An enum describing the attributes of an instrumented profile.
Definition: InstrProf.h:287
llvm::coverage::Version2
@ Version2
Definition: CoverageMapping.h:997
support
Reimplement select in terms of SEL *We would really like to support but we need to prove that the add doesn t need to overflow between the two bit chunks *Implement pre post increment support(e.g. PR935) *Implement smarter const ant generation for binops with large immediates. A few ARMv6T2 ops should be pattern matched
Definition: README.txt:10
llvm::InstrProfCorrelator
InstrProfCorrelator - A base class used to create raw instrumentation data to their functions.
Definition: InstrProfCorrelator.h:30
llvm::NamedInstrProfRecord
Definition: InstrProf.h:911
llvm::SmallVectorImpl< char >
llvm::instrprof_error::malformed
@ malformed
llvm::RawInstrProf::Version
const uint64_t Version
Definition: InstrProf.h:1189
llvm::RawInstrProf::Header
Definition: InstrProf.h:1214
llvm::InstrProfLookupTrait::offset_type
uint64_t offset_type
Definition: InstrProfReader.h:419
llvm::AMDGPU::VGPRIndexMode::Id
Id
Definition: SIDefines.h:241
llvm::InstrProfReaderItaniumRemapper::populateRemappings
Error populateRemappings() override
Definition: InstrProfReader.cpp:802
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition: SmallVector.h:650
getProfileKindFromVersion
static InstrProfKind getProfileKindFromVersion(uint64_t Version)
Definition: InstrProfReader.cpp:42
Endian.h
llvm::StringRef::begin
iterator begin() const
Definition: StringRef.h:111
llvm::IndexedInstrProf::Summary::NumSummaryFields
uint64_t NumSummaryFields
Definition: InstrProf.h:1119
llvm::IndexedInstrProf::Summary::Entry::MinBlockCount
uint64_t MinBlockCount
The minimum execution count for this percentile.
Definition: InstrProf.h:1094
llvm::ProfileSummaryBuilder::DefaultCutoffs
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:65
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::InstrProfReader
Base class and interface for reading profiling data of any known instrprof format.
Definition: InstrProfReader.h:85
llvm::RawInstrProfReader32
RawInstrProfReader< uint32_t > RawInstrProfReader32
Definition: InstrProfReader.h:390
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::instrprof_error::missing_debug_info_for_correlation
@ missing_debug_info_for_correlation
llvm::InstrProfLookupTrait::readValueProfilingData
bool readValueProfilingData(const unsigned char *&D, const unsigned char *const End)
Definition: InstrProfReader.cpp:641
llvm::InstrProfReader::error
Error error(instrprof_error Err, const std::string &ErrMsg="")
Set the current error and return same.
Definition: InstrProfReader.h:144
llvm::InstrProfReaderIndex::getRecords
Error getRecords(ArrayRef< NamedInstrProfRecord > &Data) override
Definition: InstrProfReader.cpp:718
llvm::SmallVectorImpl::insert
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:792