LLVM 19.0.0git
InstrProfWriter.cpp
Go to the documentation of this file.
1//===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for writing profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/StringRef.h"
22#include "llvm/Support/Endian.h"
24#include "llvm/Support/Error.h"
28#include <cstdint>
29#include <memory>
30#include <string>
31#include <tuple>
32#include <utility>
33#include <vector>
34
35using namespace llvm;
36
37// A struct to define how the data stream should be patched. For Indexed
38// profiling, only uint64_t data type is needed.
39struct PatchItem {
40 uint64_t Pos; // Where to patch.
41 uint64_t *D; // Pointer to an array of source data.
42 int N; // Number of elements in \c D array.
43};
44
45namespace llvm {
46
47// A wrapper class to abstract writer stream with support of bytes
48// back patching.
50public:
52 : IsFDOStream(true), OS(FD), LE(FD, llvm::endianness::little) {}
54 : IsFDOStream(false), OS(STR), LE(STR, llvm::endianness::little) {}
55
56 uint64_t tell() { return OS.tell(); }
57 void write(uint64_t V) { LE.write<uint64_t>(V); }
58 void writeByte(uint8_t V) { LE.write<uint8_t>(V); }
59
60 // \c patch can only be called when all data is written and flushed.
61 // For raw_string_ostream, the patch is done on the target string
62 // directly and it won't be reflected in the stream's internal buffer.
64 using namespace support;
65
66 if (IsFDOStream) {
67 raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
68 const uint64_t LastPos = FDOStream.tell();
69 for (const auto &K : P) {
70 FDOStream.seek(K.Pos);
71 for (int I = 0; I < K.N; I++)
72 write(K.D[I]);
73 }
74 // Reset the stream to the last position after patching so that users
75 // don't accidentally overwrite data. This makes it consistent with
76 // the string stream below which replaces the data directly.
77 FDOStream.seek(LastPos);
78 } else {
79 raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
80 std::string &Data = SOStream.str(); // with flush
81 for (const auto &K : P) {
82 for (int I = 0; I < K.N; I++) {
83 uint64_t Bytes =
84 endian::byte_swap<uint64_t, llvm::endianness::little>(K.D[I]);
85 Data.replace(K.Pos + I * sizeof(uint64_t), sizeof(uint64_t),
86 (const char *)&Bytes, sizeof(uint64_t));
87 }
88 }
89 }
90 }
91
92 // If \c OS is an instance of \c raw_fd_ostream, this field will be
93 // true. Otherwise, \c OS will be an raw_string_ostream.
97};
98
100public:
103
106
109
113
115
118 }
119
120 static std::pair<offset_type, offset_type>
122 using namespace support;
123
125
126 offset_type N = K.size();
127 LE.write<offset_type>(N);
128
129 offset_type M = 0;
130 for (const auto &ProfileData : *V) {
131 const InstrProfRecord &ProfRecord = ProfileData.second;
132 M += sizeof(uint64_t); // The function hash
133 M += sizeof(uint64_t); // The size of the Counts vector
134 M += ProfRecord.Counts.size() * sizeof(uint64_t);
135 M += sizeof(uint64_t); // The size of the Bitmap vector
136 M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t);
137
138 // Value data
139 M += ValueProfData::getSize(ProfileData.second);
140 }
141 LE.write<offset_type>(M);
142
143 return std::make_pair(N, M);
144 }
145
147 Out.write(K.data(), N);
148 }
149
151 using namespace support;
152
154 for (const auto &ProfileData : *V) {
155 const InstrProfRecord &ProfRecord = ProfileData.second;
156 if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))
157 CSSummaryBuilder->addRecord(ProfRecord);
158 else
159 SummaryBuilder->addRecord(ProfRecord);
160
161 LE.write<uint64_t>(ProfileData.first); // Function hash
162 LE.write<uint64_t>(ProfRecord.Counts.size());
163 for (uint64_t I : ProfRecord.Counts)
164 LE.write<uint64_t>(I);
165
166 LE.write<uint64_t>(ProfRecord.BitmapBytes.size());
167 for (uint64_t I : ProfRecord.BitmapBytes)
168 LE.write<uint64_t>(I);
169
170 // Write value data
171 std::unique_ptr<ValueProfData> VDataPtr =
172 ValueProfData::serializeFrom(ProfileData.second);
173 uint32_t S = VDataPtr->getSize();
174 VDataPtr->swapBytesFromHost(ValueProfDataEndianness);
175 Out.write((const char *)VDataPtr.get(), S);
176 }
177 }
178};
179
180} // end namespace llvm
181
183 uint64_t TemporalProfTraceReservoirSize,
184 uint64_t MaxTemporalProfTraceLength,
185 bool WritePrevVersion)
186 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
187 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
188 InfoObj(new InstrProfRecordWriterTrait()),
189 WritePrevVersion(WritePrevVersion) {}
190
192
193// Internal interface for testing purpose only.
195 InfoObj->ValueProfDataEndianness = Endianness;
196}
197
199 this->Sparse = Sparse;
200}
201
203 function_ref<void(Error)> Warn) {
204 auto Name = I.Name;
205 auto Hash = I.Hash;
206 addRecord(Name, Hash, std::move(I), Weight, Warn);
207}
208
210 OverlapStats &Overlap,
211 OverlapStats &FuncLevelOverlap,
212 const OverlapFuncFilters &FuncFilter) {
213 auto Name = Other.Name;
214 auto Hash = Other.Hash;
215 Other.accumulateCounts(FuncLevelOverlap.Test);
216 if (!FunctionData.contains(Name)) {
217 Overlap.addOneUnique(FuncLevelOverlap.Test);
218 return;
219 }
220 if (FuncLevelOverlap.Test.CountSum < 1.0f) {
221 Overlap.Overlap.NumEntries += 1;
222 return;
223 }
224 auto &ProfileDataMap = FunctionData[Name];
225 bool NewFunc;
227 std::tie(Where, NewFunc) =
228 ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
229 if (NewFunc) {
230 Overlap.addOneMismatch(FuncLevelOverlap.Test);
231 return;
232 }
233 InstrProfRecord &Dest = Where->second;
234
235 uint64_t ValueCutoff = FuncFilter.ValueCutoff;
236 if (!FuncFilter.NameFilter.empty() && Name.contains(FuncFilter.NameFilter))
237 ValueCutoff = 0;
238
239 Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
240}
241
243 InstrProfRecord &&I, uint64_t Weight,
244 function_ref<void(Error)> Warn) {
245 auto &ProfileDataMap = FunctionData[Name];
246
247 bool NewFunc;
249 std::tie(Where, NewFunc) =
250 ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
251 InstrProfRecord &Dest = Where->second;
252
253 auto MapWarn = [&](instrprof_error E) {
254 Warn(make_error<InstrProfError>(E));
255 };
256
257 if (NewFunc) {
258 // We've never seen a function with this name and hash, add it.
259 Dest = std::move(I);
260 if (Weight > 1)
261 Dest.scale(Weight, 1, MapWarn);
262 } else {
263 // We're updating a function we've seen before.
264 Dest.merge(I, Weight, MapWarn);
265 }
266
267 Dest.sortValueData();
268}
269
272 auto Result = MemProfRecordData.insert({Id, Record});
273 // If we inserted a new record then we are done.
274 if (Result.second) {
275 return;
276 }
277 memprof::IndexedMemProfRecord &Existing = Result.first->second;
278 Existing.merge(Record);
279}
280
282 const memprof::Frame &Frame,
283 function_ref<void(Error)> Warn) {
284 auto Result = MemProfFrameData.insert({Id, Frame});
285 // If a mapping already exists for the current frame id and it does not
286 // match the new mapping provided then reset the existing contents and bail
287 // out. We don't support the merging of memprof data whose Frame -> Id
288 // mapping across profiles is inconsistent.
289 if (!Result.second && Result.first->second != Frame) {
290 Warn(make_error<InstrProfError>(instrprof_error::malformed,
291 "frame to id mapping mismatch"));
292 return false;
293 }
294 return true;
295}
296
298 llvm::append_range(BinaryIds, BIs);
299}
300
301void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) {
302 if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
303 Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);
304 if (Trace.FunctionNameRefs.empty())
305 return;
306
307 if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
308 // Simply append the trace if we have not yet hit our reservoir size limit.
309 TemporalProfTraces.push_back(std::move(Trace));
310 } else {
311 // Otherwise, replace a random trace in the stream.
312 std::uniform_int_distribution<uint64_t> Distribution(
313 0, TemporalProfTraceStreamSize);
314 uint64_t RandomIndex = Distribution(RNG);
315 if (RandomIndex < TemporalProfTraces.size())
316 TemporalProfTraces[RandomIndex] = std::move(Trace);
317 }
318 ++TemporalProfTraceStreamSize;
319}
320
322 SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
323 // Assume that the source has the same reservoir size as the destination to
324 // avoid needing to record it in the indexed profile format.
325 bool IsDestSampled =
326 (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize);
327 bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize);
328 if (!IsDestSampled && IsSrcSampled) {
329 // If one of the traces are sampled, ensure that it belongs to Dest.
330 std::swap(TemporalProfTraces, SrcTraces);
331 std::swap(TemporalProfTraceStreamSize, SrcStreamSize);
332 std::swap(IsDestSampled, IsSrcSampled);
333 }
334 if (!IsSrcSampled) {
335 // If the source stream is not sampled, we add each source trace normally.
336 for (auto &Trace : SrcTraces)
337 addTemporalProfileTrace(std::move(Trace));
338 return;
339 }
340 // Otherwise, we find the traces that would have been removed if we added
341 // the whole source stream.
342 SmallSetVector<uint64_t, 8> IndicesToReplace;
343 for (uint64_t I = 0; I < SrcStreamSize; I++) {
344 std::uniform_int_distribution<uint64_t> Distribution(
345 0, TemporalProfTraceStreamSize);
346 uint64_t RandomIndex = Distribution(RNG);
347 if (RandomIndex < TemporalProfTraces.size())
348 IndicesToReplace.insert(RandomIndex);
349 ++TemporalProfTraceStreamSize;
350 }
351 // Then we insert a random sample of the source traces.
352 llvm::shuffle(SrcTraces.begin(), SrcTraces.end(), RNG);
353 for (const auto &[Index, Trace] : llvm::zip(IndicesToReplace, SrcTraces))
354 TemporalProfTraces[Index] = std::move(Trace);
355}
356
358 function_ref<void(Error)> Warn) {
359 for (auto &I : IPW.FunctionData)
360 for (auto &Func : I.getValue())
361 addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn);
362
363 BinaryIds.reserve(BinaryIds.size() + IPW.BinaryIds.size());
364 for (auto &I : IPW.BinaryIds)
366
367 addTemporalProfileTraces(IPW.TemporalProfTraces,
368 IPW.TemporalProfTraceStreamSize);
369
370 MemProfFrameData.reserve(IPW.MemProfFrameData.size());
371 for (auto &I : IPW.MemProfFrameData) {
372 // If we weren't able to add the frame mappings then it doesn't make sense
373 // to try to merge the records from this profile.
374 if (!addMemProfFrame(I.first, I.second, Warn))
375 return;
376 }
377
378 MemProfRecordData.reserve(IPW.MemProfRecordData.size());
379 for (auto &I : IPW.MemProfRecordData) {
380 addMemProfRecord(I.first, I.second);
381 }
382}
383
384bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
385 if (!Sparse)
386 return true;
387 for (const auto &Func : PD) {
388 const InstrProfRecord &IPR = Func.second;
389 if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; }))
390 return true;
391 if (llvm::any_of(IPR.BitmapBytes, [](uint8_t Byte) { return Byte > 0; }))
392 return true;
393 }
394 return false;
395}
396
397static void setSummary(IndexedInstrProf::Summary *TheSummary,
398 ProfileSummary &PS) {
399 using namespace IndexedInstrProf;
400
401 const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
402 TheSummary->NumSummaryFields = Summary::NumKinds;
403 TheSummary->NumCutoffEntries = Res.size();
404 TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());
405 TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());
406 TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());
407 TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());
408 TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());
409 TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());
410 for (unsigned I = 0; I < Res.size(); I++)
411 TheSummary->setEntry(I, Res[I]);
412}
413
414Error InstrProfWriter::writeImpl(ProfOStream &OS) {
415 using namespace IndexedInstrProf;
416 using namespace support;
417
419
421 InfoObj->SummaryBuilder = &ISB;
423 InfoObj->CSSummaryBuilder = &CSISB;
424
425 // Populate the hash table generator.
427 for (const auto &I : FunctionData)
428 if (shouldEncodeData(I.getValue()))
429 OrderedData.emplace_back((I.getKey()), &I.getValue());
430 llvm::sort(OrderedData, less_first());
431 for (const auto &I : OrderedData)
432 Generator.insert(I.first, I.second);
433
434 // Write the header.
436 Header.Magic = IndexedInstrProf::Magic;
437 Header.Version = WritePrevVersion
440 // The WritePrevVersion handling will either need to be removed or updated
441 // if the version is advanced beyond 12.
444 if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
445 Header.Version |= VARIANT_MASK_IR_PROF;
446 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
447 Header.Version |= VARIANT_MASK_CSIR_PROF;
448 if (static_cast<bool>(ProfileKind &
450 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
451 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
452 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
453 if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
454 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
455 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
456 Header.Version |= VARIANT_MASK_MEMPROF;
457 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
458 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
459
460 Header.Unused = 0;
461 Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
462 Header.HashOffset = 0;
463 Header.MemProfOffset = 0;
464 Header.BinaryIdOffset = 0;
465 Header.TemporalProfTracesOffset = 0;
466 Header.VTableNamesOffset = 0;
467
468 // Only write out the first four fields. We need to remember the offset of the
469 // remaining fields to allow back patching later.
470 for (int I = 0; I < 4; I++)
471 OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
472
473 // Save the location of Header.HashOffset field in \c OS.
474 uint64_t HashTableStartFieldOffset = OS.tell();
475 // Reserve the space for HashOffset field.
476 OS.write(0);
477
478 // Save the location of MemProf profile data. This is stored in two parts as
479 // the schema and as a separate on-disk chained hashtable.
480 uint64_t MemProfSectionOffset = OS.tell();
481 // Reserve space for the MemProf table field to be patched later if this
482 // profile contains memory profile information.
483 OS.write(0);
484
485 // Save the location of binary ids section.
486 uint64_t BinaryIdSectionOffset = OS.tell();
487 // Reserve space for the BinaryIdOffset field to be patched later if this
488 // profile contains binary ids.
489 OS.write(0);
490
491 uint64_t TemporalProfTracesOffset = OS.tell();
492 OS.write(0);
493
494 uint64_t VTableNamesOffset = OS.tell();
495 if (!WritePrevVersion)
496 OS.write(0);
497
498 // Reserve space to write profile summary data.
500 uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
501 // Remember the summary offset.
502 uint64_t SummaryOffset = OS.tell();
503 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
504 OS.write(0);
505 uint64_t CSSummaryOffset = 0;
506 uint64_t CSSummarySize = 0;
507 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
508 CSSummaryOffset = OS.tell();
509 CSSummarySize = SummarySize / sizeof(uint64_t);
510 for (unsigned I = 0; I < CSSummarySize; I++)
511 OS.write(0);
512 }
513
514 // Write the hash table.
515 uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
516
517 // Write the MemProf profile data if we have it. This includes a simple schema
518 // with the format described below followed by the hashtable:
519 // uint64_t RecordTableOffset = RecordTableGenerator.Emit
520 // uint64_t FramePayloadOffset = Stream offset before emitting the frame table
521 // uint64_t FrameTableOffset = FrameTableGenerator.Emit
522 // uint64_t Num schema entries
523 // uint64_t Schema entry 0
524 // uint64_t Schema entry 1
525 // ....
526 // uint64_t Schema entry N - 1
527 // OnDiskChainedHashTable MemProfRecordData
528 // OnDiskChainedHashTable MemProfFrameData
529 uint64_t MemProfSectionStart = 0;
530 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
531 MemProfSectionStart = OS.tell();
532 OS.write(0ULL); // Reserve space for the memprof record table offset.
533 OS.write(0ULL); // Reserve space for the memprof frame payload offset.
534 OS.write(0ULL); // Reserve space for the memprof frame table offset.
535
537 OS.write(static_cast<uint64_t>(Schema.size()));
538 for (const auto Id : Schema) {
539 OS.write(static_cast<uint64_t>(Id));
540 }
541
542 auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
543 RecordWriter->Schema = &Schema;
545 RecordTableGenerator;
546 for (auto &I : MemProfRecordData) {
547 // Insert the key (func hash) and value (memprof record).
548 RecordTableGenerator.insert(I.first, I.second);
549 }
550 // Release the memory of this MapVector as it is no longer needed.
551 MemProfRecordData.clear();
552
553 // The call to Emit invokes RecordWriterTrait::EmitData which destructs
554 // the memprof record copies owned by the RecordTableGenerator. This works
555 // because the RecordTableGenerator is not used after this point.
556 uint64_t RecordTableOffset =
557 RecordTableGenerator.Emit(OS.OS, *RecordWriter);
558
559 uint64_t FramePayloadOffset = OS.tell();
560
561 auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
563 FrameTableGenerator;
564 for (auto &I : MemProfFrameData) {
565 // Insert the key (frame id) and value (frame contents).
566 FrameTableGenerator.insert(I.first, I.second);
567 }
568 // Release the memory of this MapVector as it is no longer needed.
569 MemProfFrameData.clear();
570
571 uint64_t FrameTableOffset = FrameTableGenerator.Emit(OS.OS, *FrameWriter);
572
573 PatchItem PatchItems[] = {
574 {MemProfSectionStart, &RecordTableOffset, 1},
575 {MemProfSectionStart + sizeof(uint64_t), &FramePayloadOffset, 1},
576 {MemProfSectionStart + 2 * sizeof(uint64_t), &FrameTableOffset, 1},
577 };
578 OS.patch(PatchItems);
579 }
580
581 // BinaryIdSection has two parts:
582 // 1. uint64_t BinaryIdsSectionSize
583 // 2. list of binary ids that consist of:
584 // a. uint64_t BinaryIdLength
585 // b. uint8_t BinaryIdData
586 // c. uint8_t Padding (if necessary)
587 uint64_t BinaryIdSectionStart = OS.tell();
588 // Calculate size of binary section.
589 uint64_t BinaryIdsSectionSize = 0;
590
591 // Remove duplicate binary ids.
592 llvm::sort(BinaryIds);
593 BinaryIds.erase(std::unique(BinaryIds.begin(), BinaryIds.end()),
594 BinaryIds.end());
595
596 for (auto BI : BinaryIds) {
597 // Increment by binary id length data type size.
598 BinaryIdsSectionSize += sizeof(uint64_t);
599 // Increment by binary id data length, aligned to 8 bytes.
600 BinaryIdsSectionSize += alignToPowerOf2(BI.size(), sizeof(uint64_t));
601 }
602 // Write binary ids section size.
603 OS.write(BinaryIdsSectionSize);
604
605 for (auto BI : BinaryIds) {
606 uint64_t BILen = BI.size();
607 // Write binary id length.
608 OS.write(BILen);
609 // Write binary id data.
610 for (unsigned K = 0; K < BILen; K++)
611 OS.writeByte(BI[K]);
612 // Write padding if necessary.
613 uint64_t PaddingSize = alignToPowerOf2(BILen, sizeof(uint64_t)) - BILen;
614 for (unsigned K = 0; K < PaddingSize; K++)
615 OS.writeByte(0);
616 }
617
618 uint64_t VTableNamesSectionStart = OS.tell();
619
620 if (!WritePrevVersion) {
621 // Use a dummy (and uncompressed) string as compressed vtable names and get
622 // the necessary profile format change in place for version 12.
623 // TODO: Store the list of vtable names in InstrProfWriter and use the
624 // real compressed name.
625 std::string CompressedVTableNames = "VTableNames";
626
627 uint64_t CompressedStringLen = CompressedVTableNames.length();
628
629 // Record the length of compressed string.
630 OS.write(CompressedStringLen);
631
632 // Write the chars in compressed strings.
633 for (auto &c : CompressedVTableNames)
634 OS.writeByte(static_cast<uint8_t>(c));
635
636 // Pad up to a multiple of 8.
637 // InstrProfReader would read bytes according to 'CompressedStringLen'.
638 uint64_t PaddedLength = alignTo(CompressedStringLen, 8);
639
640 for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) {
641 OS.writeByte(0);
642 }
643 }
644
645 uint64_t TemporalProfTracesSectionStart = 0;
646 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
647 TemporalProfTracesSectionStart = OS.tell();
648 OS.write(TemporalProfTraces.size());
649 OS.write(TemporalProfTraceStreamSize);
650 for (auto &Trace : TemporalProfTraces) {
651 OS.write(Trace.Weight);
652 OS.write(Trace.FunctionNameRefs.size());
653 for (auto &NameRef : Trace.FunctionNameRefs)
654 OS.write(NameRef);
655 }
656 }
657
658 // Allocate space for data to be serialized out.
659 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
661 // Compute the Summary and copy the data to the data
662 // structure to be serialized out (to disk or buffer).
663 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
664 setSummary(TheSummary.get(), *PS);
665 InfoObj->SummaryBuilder = nullptr;
666
667 // For Context Sensitive summary.
668 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
669 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
670 TheCSSummary = IndexedInstrProf::allocSummary(SummarySize);
671 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
672 setSummary(TheCSSummary.get(), *CSPS);
673 }
674 InfoObj->CSSummaryBuilder = nullptr;
675
676 if (!WritePrevVersion) {
677 // Now do the final patch:
678 PatchItem PatchItems[] = {
679 // Patch the Header.HashOffset field.
680 {HashTableStartFieldOffset, &HashTableStart, 1},
681 // Patch the Header.MemProfOffset (=0 for profiles without MemProf
682 // data).
683 {MemProfSectionOffset, &MemProfSectionStart, 1},
684 // Patch the Header.BinaryIdSectionOffset.
685 {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
686 // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
687 // traces).
688 {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
689 {VTableNamesOffset, &VTableNamesSectionStart, 1},
690 // Patch the summary data.
691 {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
692 (int)(SummarySize / sizeof(uint64_t))},
693 {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()),
694 (int)CSSummarySize}};
695
696 OS.patch(PatchItems);
697 } else {
698 // Now do the final patch:
699 PatchItem PatchItems[] = {
700 // Patch the Header.HashOffset field.
701 {HashTableStartFieldOffset, &HashTableStart, 1},
702 // Patch the Header.MemProfOffset (=0 for profiles without MemProf
703 // data).
704 {MemProfSectionOffset, &MemProfSectionStart, 1},
705 // Patch the Header.BinaryIdSectionOffset.
706 {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
707 // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
708 // traces).
709 {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
710 // Patch the summary data.
711 {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
712 (int)(SummarySize / sizeof(uint64_t))},
713 {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()),
714 (int)CSSummarySize}};
715
716 OS.patch(PatchItems);
717 }
718
719 for (const auto &I : FunctionData)
720 for (const auto &F : I.getValue())
721 if (Error E = validateRecord(F.second))
722 return E;
723
724 return Error::success();
725}
726
728 // Write the hash table.
729 ProfOStream POS(OS);
730 return writeImpl(POS);
731}
732
734 ProfOStream POS(OS);
735 return writeImpl(POS);
736}
737
738std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
739 std::string Data;
741 // Write the hash table.
742 if (Error E = write(OS))
743 return nullptr;
744 // Return this in an aligned memory buffer.
746}
747
748static const char *ValueProfKindStr[] = {
749#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
751};
752
754 for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
755 uint32_t NS = Func.getNumValueSites(VK);
756 if (!NS)
757 continue;
758 for (uint32_t S = 0; S < NS; S++) {
759 uint32_t ND = Func.getNumValueDataForSite(VK, S);
760 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
761 DenseSet<uint64_t> SeenValues;
762 for (uint32_t I = 0; I < ND; I++)
763 if ((VK != IPVK_IndirectCallTarget && VK != IPVK_VTableTarget) &&
764 !SeenValues.insert(VD[I].Value).second)
765 return make_error<InstrProfError>(instrprof_error::invalid_prof);
766 }
767 }
768
769 return Error::success();
770}
771
773 const InstrProfRecord &Func,
774 InstrProfSymtab &Symtab,
776 OS << Name << "\n";
777 OS << "# Func Hash:\n" << Hash << "\n";
778 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
779 OS << "# Counter Values:\n";
780 for (uint64_t Count : Func.Counts)
781 OS << Count << "\n";
782
783 if (Func.BitmapBytes.size() > 0) {
784 OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n";
785 OS << "# Bitmap Byte Values:\n";
786 for (uint8_t Byte : Func.BitmapBytes) {
787 OS << "0x";
788 OS.write_hex(Byte);
789 OS << "\n";
790 }
791 OS << "\n";
792 }
793
794 uint32_t NumValueKinds = Func.getNumValueKinds();
795 if (!NumValueKinds) {
796 OS << "\n";
797 return;
798 }
799
800 OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
801 for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
802 uint32_t NS = Func.getNumValueSites(VK);
803 if (!NS)
804 continue;
805 OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
806 OS << "# NumValueSites:\n" << NS << "\n";
807 for (uint32_t S = 0; S < NS; S++) {
808 uint32_t ND = Func.getNumValueDataForSite(VK, S);
809 OS << ND << "\n";
810 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
811 for (uint32_t I = 0; I < ND; I++) {
812 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
813 OS << Symtab.getFuncOrVarNameIfDefined(VD[I].Value) << ":"
814 << VD[I].Count << "\n";
815 else
816 OS << VD[I].Value << ":" << VD[I].Count << "\n";
817 }
818 }
819 }
820
821 OS << "\n";
822}
823
825 // Check CS first since it implies an IR level profile.
826 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
827 OS << "# CSIR level Instrumentation Flag\n:csir\n";
828 else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
829 OS << "# IR level Instrumentation Flag\n:ir\n";
830
831 if (static_cast<bool>(ProfileKind &
833 OS << "# Always instrument the function entry block\n:entry_first\n";
834 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
835 OS << "# Instrument block coverage\n:single_byte_coverage\n";
836 InstrProfSymtab Symtab;
837
839 using RecordType = std::pair<StringRef, FuncPair>;
840 SmallVector<RecordType, 4> OrderedFuncData;
841
842 for (const auto &I : FunctionData) {
843 if (shouldEncodeData(I.getValue())) {
844 if (Error E = Symtab.addFuncName(I.getKey()))
845 return E;
846 for (const auto &Func : I.getValue())
847 OrderedFuncData.push_back(std::make_pair(I.getKey(), Func));
848 }
849 }
850
851 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
853
854 llvm::sort(OrderedFuncData, [](const RecordType &A, const RecordType &B) {
855 return std::tie(A.first, A.second.first) <
856 std::tie(B.first, B.second.first);
857 });
858
859 for (const auto &record : OrderedFuncData) {
860 const StringRef &Name = record.first;
861 const FuncPair &Func = record.second;
862 writeRecordInText(Name, Func.first, Func.second, Symtab, OS);
863 }
864
865 for (const auto &record : OrderedFuncData) {
866 const FuncPair &Func = record.second;
867 if (Error E = validateRecord(Func.second))
868 return E;
869 }
870
871 return Error::success();
872}
873
875 InstrProfSymtab &Symtab) {
876 OS << ":temporal_prof_traces\n";
877 OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";
878 OS << "# Temporal Profile Trace Stream Size:\n"
879 << TemporalProfTraceStreamSize << "\n";
880 for (auto &Trace : TemporalProfTraces) {
881 OS << "# Weight:\n" << Trace.Weight << "\n";
882 for (auto &NameRef : Trace.FunctionNameRefs)
883 OS << Symtab.getFuncOrVarName(NameRef) << ",";
884 OS << "\n";
885 }
886 OS << "\n";
887}
basic Basic Alias true
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
std::string Name
static void setSummary(IndexedInstrProf::Summary *TheSummary, ProfileSummary &PS)
static const char * ValueProfKindStr[]
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Defines facilities for reading and writing on-disk hash tables.
#define P(N)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file implements a set that has insertion order iteration characteristics.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
Definition: DenseMap.h:71
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
static std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
const InstrProfWriter::ProfilingData *const data_type_ref
InstrProfSummaryBuilder * SummaryBuilder
static hash_value_type ComputeHash(key_type_ref K)
InstrProfSummaryBuilder * CSSummaryBuilder
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N)
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
const InstrProfWriter::ProfilingData *const data_type
void addRecord(const InstrProfRecord &)
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
Definition: InstrProf.h:429
StringRef getFuncOrVarName(uint64_t ValMD5Hash)
Return name of functions or global variables from the name's md5 hash value.
Definition: InstrProf.h:588
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:503
StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash)
Just like getFuncOrVarName, except that it will return literal string 'External Symbol' if the functi...
Definition: InstrProf.h:581
Error write(raw_fd_ostream &OS)
Write the profile to OS.
void addTemporalProfileTraces(SmallVectorImpl< TemporalProfTraceTy > &SrcTraces, uint64_t SrcStreamSize)
Add SrcTraces using reservoir sampling where SrcStreamSize is the total number of temporal profiling ...
void overlapRecord(NamedInstrProfRecord &&Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, const OverlapFuncFilters &FuncFilter)
Error writeText(raw_fd_ostream &OS)
Write the profile in text format to OS.
InstrProfWriter(bool Sparse=false, uint64_t TemporalProfTraceReservoirSize=0, uint64_t MaxTemporalProfTraceLength=0, bool WritePrevVersion=false)
void addBinaryIds(ArrayRef< llvm::object::BuildID > BIs)
void addMemProfRecord(const GlobalValue::GUID Id, const memprof::IndexedMemProfRecord &Record)
Add a memprof record for a function identified by its Id.
static void writeRecordInText(StringRef Name, uint64_t Hash, const InstrProfRecord &Counters, InstrProfSymtab &Symtab, raw_fd_ostream &OS)
Write Record in text format to OS.
void setValueProfDataEndianness(llvm::endianness Endianness)
void addRecord(NamedInstrProfRecord &&I, uint64_t Weight, function_ref< void(Error)> Warn)
Add function counts for the given function.
void mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref< void(Error)> Warn)
Merge existing function counts from the given writer.
void writeTextTemporalProfTraceData(raw_fd_ostream &OS, InstrProfSymtab &Symtab)
Write temporal profile trace data to the header in text format to OS.
std::unique_ptr< MemoryBuffer > writeBuffer()
Write the profile, returning the raw data. For testing.
bool addMemProfFrame(const memprof::FrameId, const memprof::Frame &F, function_ref< void(Error)> Warn)
Add a memprof frame identified by the hash of the contents of the frame in FrameId.
void setOutputSparse(bool Sparse)
Error validateRecord(const InstrProfRecord &Func)
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it.
Generates an on disk hash table.
offset_type Emit(raw_ostream &Out)
Emit the table to Out, which must not be at offset 0.
void writeByte(uint8_t V)
void patch(ArrayRef< PatchItem > P)
ProfOStream(raw_string_ostream &STR)
support::endian::Writer LE
ProfOStream(raw_fd_ostream &FD)
void write(uint64_t V)
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:70
uint64_t getTotalCount() const
uint64_t getMaxCount() const
const SummaryEntryVector & getDetailedSummary()
uint32_t getNumCounts() const
uint64_t getMaxInternalCount() const
uint64_t getMaxFunctionCount() const
uint32_t getNumFunctions() const
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool empty() const
Definition: Trace.h:96
unsigned size() const
Definition: Trace.h:95
LLVM Value Representation.
Definition: Value.h:74
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:470
uint64_t seek(uint64_t off)
Flushes the stream and repositions the underlying file descriptor position to the offset specified fr...
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
uint64_t tell() const
tell - Return the current offset with the file.
Definition: raw_ostream.h:150
raw_ostream & write_hex(unsigned long long N)
Output N in hexadecimal, without any prefix or padding.
raw_ostream & write(unsigned char C)
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:678
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1163
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1051
const uint64_t Magic
Definition: InstrProf.h:1008
const HashT HashType
Definition: InstrProf.h:1049
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition: STLExtras.h:862
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
Definition: MathExtras.h:382
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2082
void shuffle(Iterator first, Iterator last, RNG &&g)
Definition: STLExtras.h:1550
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
@ Other
Any other memory.
instrprof_error
Definition: InstrProf.h:324
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
endianness
Definition: bit.h:70
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
uint64_t Pos
uint64_t * D
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
void set(SummaryFieldKind K, uint64_t V)
Definition: InstrProf.h:1149
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
Definition: InstrProf.h:1155
Profiling information for a single function.
Definition: InstrProf.h:704
std::vector< uint64_t > Counts
Definition: InstrProf.h:705
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:812
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:708
void sortValueData()
Sort value profile data (per site) by count.
Definition: InstrProf.h:779
std::vector< uint8_t > BitmapBytes
Definition: InstrProf.h:706
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:875
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:910
const std::string NameFilter
Definition: InstrProf.h:669
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1407
CountSumOrPercent Overlap
Definition: InstrProf.h:631
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1417
CountSumOrPercent Test
Definition: InstrProf.h:629
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
Definition: InstrProf.h:355
Function object to check whether the first component of a container supported by std::get (like std::...
Definition: STLExtras.h:1459
void merge(const IndexedMemProfRecord &Other)
Definition: MemProf.h:343
static MemProfSchema getSchema()
Definition: MemProf.h:101
Adapter to write values to a stream in a particular byte order.
Definition: EndianStream.h:67
void write(ArrayRef< value_type > Val)
Definition: EndianStream.h:71