LLVM 17.0.0git
InstrProfWriter.cpp
Go to the documentation of this file.
1//===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for writing profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/StringRef.h"
22#include "llvm/Support/Endian.h"
24#include "llvm/Support/Error.h"
28#include <cstdint>
29#include <memory>
30#include <string>
31#include <tuple>
32#include <utility>
33#include <vector>
34
35using namespace llvm;
36
37// A struct to define how the data stream should be patched. For Indexed
38// profiling, only uint64_t data type is needed.
39struct PatchItem {
40 uint64_t Pos; // Where to patch.
41 uint64_t *D; // Pointer to an array of source data.
42 int N; // Number of elements in \c D array.
43};
44
45namespace llvm {
46
47// A wrapper class to abstract writer stream with support of bytes
48// back patching.
50public:
52 : IsFDOStream(true), OS(FD), LE(FD, support::little) {}
54 : IsFDOStream(false), OS(STR), LE(STR, support::little) {}
55
56 uint64_t tell() { return OS.tell(); }
57 void write(uint64_t V) { LE.write<uint64_t>(V); }
58 void writeByte(uint8_t V) { LE.write<uint8_t>(V); }
59
60 // \c patch can only be called when all data is written and flushed.
61 // For raw_string_ostream, the patch is done on the target string
62 // directly and it won't be reflected in the stream's internal buffer.
63 void patch(PatchItem *P, int NItems) {
64 using namespace support;
65
66 if (IsFDOStream) {
67 raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
68 const uint64_t LastPos = FDOStream.tell();
69 for (int K = 0; K < NItems; K++) {
70 FDOStream.seek(P[K].Pos);
71 for (int I = 0; I < P[K].N; I++)
72 write(P[K].D[I]);
73 }
74 // Reset the stream to the last position after patching so that users
75 // don't accidentally overwrite data. This makes it consistent with
76 // the string stream below which replaces the data directly.
77 FDOStream.seek(LastPos);
78 } else {
79 raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
80 std::string &Data = SOStream.str(); // with flush
81 for (int K = 0; K < NItems; K++) {
82 for (int I = 0; I < P[K].N; I++) {
83 uint64_t Bytes = endian::byte_swap<uint64_t, little>(P[K].D[I]);
84 Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t),
85 (const char *)&Bytes, sizeof(uint64_t));
86 }
87 }
88 }
89 }
90
91 // If \c OS is an instance of \c raw_fd_ostream, this field will be
92 // true. Otherwise, \c OS will be an raw_string_ostream.
96};
97
99public:
102
105
108
112
114
117 }
118
119 static std::pair<offset_type, offset_type>
121 using namespace support;
122
123 endian::Writer LE(Out, little);
124
125 offset_type N = K.size();
126 LE.write<offset_type>(N);
127
128 offset_type M = 0;
129 for (const auto &ProfileData : *V) {
130 const InstrProfRecord &ProfRecord = ProfileData.second;
131 M += sizeof(uint64_t); // The function hash
132 M += sizeof(uint64_t); // The size of the Counts vector
133 M += ProfRecord.Counts.size() * sizeof(uint64_t);
134
135 // Value data
136 M += ValueProfData::getSize(ProfileData.second);
137 }
138 LE.write<offset_type>(M);
139
140 return std::make_pair(N, M);
141 }
142
144 Out.write(K.data(), N);
145 }
146
148 using namespace support;
149
150 endian::Writer LE(Out, little);
151 for (const auto &ProfileData : *V) {
152 const InstrProfRecord &ProfRecord = ProfileData.second;
153 if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))
154 CSSummaryBuilder->addRecord(ProfRecord);
155 else
156 SummaryBuilder->addRecord(ProfRecord);
157
158 LE.write<uint64_t>(ProfileData.first); // Function hash
159 LE.write<uint64_t>(ProfRecord.Counts.size());
160 for (uint64_t I : ProfRecord.Counts)
161 LE.write<uint64_t>(I);
162
163 // Write value data
164 std::unique_ptr<ValueProfData> VDataPtr =
165 ValueProfData::serializeFrom(ProfileData.second);
166 uint32_t S = VDataPtr->getSize();
167 VDataPtr->swapBytesFromHost(ValueProfDataEndianness);
168 Out.write((const char *)VDataPtr.get(), S);
169 }
170 }
171};
172
173} // end namespace llvm
174
176 uint64_t TemporalProfTraceReservoirSize,
177 uint64_t MaxTemporalProfTraceLength)
178 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
179 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
180 InfoObj(new InstrProfRecordWriterTrait()) {}
181
183
184// Internal interface for testing purpose only.
186 support::endianness Endianness) {
187 InfoObj->ValueProfDataEndianness = Endianness;
188}
189
191 this->Sparse = Sparse;
192}
193
195 function_ref<void(Error)> Warn) {
196 auto Name = I.Name;
197 auto Hash = I.Hash;
198 addRecord(Name, Hash, std::move(I), Weight, Warn);
199}
200
202 OverlapStats &Overlap,
203 OverlapStats &FuncLevelOverlap,
204 const OverlapFuncFilters &FuncFilter) {
205 auto Name = Other.Name;
206 auto Hash = Other.Hash;
207 Other.accumulateCounts(FuncLevelOverlap.Test);
208 if (!FunctionData.contains(Name)) {
209 Overlap.addOneUnique(FuncLevelOverlap.Test);
210 return;
211 }
212 if (FuncLevelOverlap.Test.CountSum < 1.0f) {
213 Overlap.Overlap.NumEntries += 1;
214 return;
215 }
216 auto &ProfileDataMap = FunctionData[Name];
217 bool NewFunc;
219 std::tie(Where, NewFunc) =
220 ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
221 if (NewFunc) {
222 Overlap.addOneMismatch(FuncLevelOverlap.Test);
223 return;
224 }
225 InstrProfRecord &Dest = Where->second;
226
227 uint64_t ValueCutoff = FuncFilter.ValueCutoff;
228 if (!FuncFilter.NameFilter.empty() && Name.contains(FuncFilter.NameFilter))
229 ValueCutoff = 0;
230
231 Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
232}
233
235 InstrProfRecord &&I, uint64_t Weight,
236 function_ref<void(Error)> Warn) {
237 auto &ProfileDataMap = FunctionData[Name];
238
239 bool NewFunc;
241 std::tie(Where, NewFunc) =
242 ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
243 InstrProfRecord &Dest = Where->second;
244
245 auto MapWarn = [&](instrprof_error E) {
246 Warn(make_error<InstrProfError>(E));
247 };
248
249 if (NewFunc) {
250 // We've never seen a function with this name and hash, add it.
251 Dest = std::move(I);
252 if (Weight > 1)
253 Dest.scale(Weight, 1, MapWarn);
254 } else {
255 // We're updating a function we've seen before.
256 Dest.merge(I, Weight, MapWarn);
257 }
258
259 Dest.sortValueData();
260}
261
264 auto Result = MemProfRecordData.insert({Id, Record});
265 // If we inserted a new record then we are done.
266 if (Result.second) {
267 return;
268 }
269 memprof::IndexedMemProfRecord &Existing = Result.first->second;
270 Existing.merge(Record);
271}
272
274 const memprof::Frame &Frame,
275 function_ref<void(Error)> Warn) {
276 auto Result = MemProfFrameData.insert({Id, Frame});
277 // If a mapping already exists for the current frame id and it does not
278 // match the new mapping provided then reset the existing contents and bail
279 // out. We don't support the merging of memprof data whose Frame -> Id
280 // mapping across profiles is inconsistent.
281 if (!Result.second && Result.first->second != Frame) {
282 Warn(make_error<InstrProfError>(instrprof_error::malformed,
283 "frame to id mapping mismatch"));
284 return false;
285 }
286 return true;
287}
288
290 llvm::append_range(BinaryIds, BIs);
291}
292
293void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) {
294 if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
295 Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);
296 if (Trace.FunctionNameRefs.empty())
297 return;
298
299 if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
300 // Simply append the trace if we have not yet hit our reservoir size limit.
301 TemporalProfTraces.push_back(std::move(Trace));
302 } else {
303 // Otherwise, replace a random trace in the stream.
304 std::uniform_int_distribution<uint64_t> Distribution(
305 0, TemporalProfTraceStreamSize);
306 uint64_t RandomIndex = Distribution(RNG);
307 if (RandomIndex < TemporalProfTraces.size())
308 TemporalProfTraces[RandomIndex] = std::move(Trace);
309 }
310 ++TemporalProfTraceStreamSize;
311}
312
314 SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
315 // Assume that the source has the same reservoir size as the destination to
316 // avoid needing to record it in the indexed profile format.
317 bool IsDestSampled =
318 (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize);
319 bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize);
320 if (!IsDestSampled && IsSrcSampled) {
321 // If one of the traces are sampled, ensure that it belongs to Dest.
322 std::swap(TemporalProfTraces, SrcTraces);
323 std::swap(TemporalProfTraceStreamSize, SrcStreamSize);
324 std::swap(IsDestSampled, IsSrcSampled);
325 }
326 if (!IsSrcSampled) {
327 // If the source stream is not sampled, we add each source trace normally.
328 for (auto &Trace : SrcTraces)
329 addTemporalProfileTrace(std::move(Trace));
330 return;
331 }
332 // Otherwise, we find the traces that would have been removed if we added
333 // the whole source stream.
334 SmallSetVector<uint64_t, 8> IndicesToReplace;
335 for (uint64_t I = 0; I < SrcStreamSize; I++) {
336 std::uniform_int_distribution<uint64_t> Distribution(
337 0, TemporalProfTraceStreamSize);
338 uint64_t RandomIndex = Distribution(RNG);
339 if (RandomIndex < TemporalProfTraces.size())
340 IndicesToReplace.insert(RandomIndex);
341 ++TemporalProfTraceStreamSize;
342 }
343 // Then we insert a random sample of the source traces.
344 llvm::shuffle(SrcTraces.begin(), SrcTraces.end(), RNG);
345 for (const auto &[Index, Trace] : llvm::zip(IndicesToReplace, SrcTraces))
346 TemporalProfTraces[Index] = std::move(Trace);
347}
348
350 function_ref<void(Error)> Warn) {
351 for (auto &I : IPW.FunctionData)
352 for (auto &Func : I.getValue())
353 addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn);
354
355 BinaryIds.reserve(BinaryIds.size() + IPW.BinaryIds.size());
356 for (auto &I : IPW.BinaryIds)
358
359 addTemporalProfileTraces(IPW.TemporalProfTraces,
360 IPW.TemporalProfTraceStreamSize);
361
362 MemProfFrameData.reserve(IPW.MemProfFrameData.size());
363 for (auto &I : IPW.MemProfFrameData) {
364 // If we weren't able to add the frame mappings then it doesn't make sense
365 // to try to merge the records from this profile.
366 if (!addMemProfFrame(I.first, I.second, Warn))
367 return;
368 }
369
370 MemProfRecordData.reserve(IPW.MemProfRecordData.size());
371 for (auto &I : IPW.MemProfRecordData) {
372 addMemProfRecord(I.first, I.second);
373 }
374}
375
376bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
377 if (!Sparse)
378 return true;
379 for (const auto &Func : PD) {
380 const InstrProfRecord &IPR = Func.second;
381 if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; }))
382 return true;
383 }
384 return false;
385}
386
387static void setSummary(IndexedInstrProf::Summary *TheSummary,
388 ProfileSummary &PS) {
389 using namespace IndexedInstrProf;
390
391 const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
392 TheSummary->NumSummaryFields = Summary::NumKinds;
393 TheSummary->NumCutoffEntries = Res.size();
394 TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());
395 TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());
396 TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());
397 TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());
398 TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());
399 TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());
400 for (unsigned I = 0; I < Res.size(); I++)
401 TheSummary->setEntry(I, Res[I]);
402}
403
404Error InstrProfWriter::writeImpl(ProfOStream &OS) {
405 using namespace IndexedInstrProf;
406 using namespace support;
407
409
411 InfoObj->SummaryBuilder = &ISB;
413 InfoObj->CSSummaryBuilder = &CSISB;
414
415 // Populate the hash table generator.
416 for (const auto &I : FunctionData)
417 if (shouldEncodeData(I.getValue()))
418 Generator.insert(I.getKey(), &I.getValue());
419
420 // Write the header.
422 Header.Magic = IndexedInstrProf::Magic;
424 if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
425 Header.Version |= VARIANT_MASK_IR_PROF;
426 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
427 Header.Version |= VARIANT_MASK_CSIR_PROF;
428 if (static_cast<bool>(ProfileKind &
430 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
431 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
432 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
433 if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
434 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
435 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
436 Header.Version |= VARIANT_MASK_MEMPROF;
437 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
438 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
439
440 Header.Unused = 0;
441 Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
442 Header.HashOffset = 0;
443 Header.MemProfOffset = 0;
444 Header.BinaryIdOffset = 0;
445 Header.TemporalProfTracesOffset = 0;
446 int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);
447
448 // Only write out all the fields except 'HashOffset', 'MemProfOffset',
449 // 'BinaryIdOffset' and `TemporalProfTracesOffset`. We need to remember the
450 // offset of these fields to allow back patching later.
451 for (int I = 0; I < N - 4; I++)
452 OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
453
454 // Save the location of Header.HashOffset field in \c OS.
455 uint64_t HashTableStartFieldOffset = OS.tell();
456 // Reserve the space for HashOffset field.
457 OS.write(0);
458
459 // Save the location of MemProf profile data. This is stored in two parts as
460 // the schema and as a separate on-disk chained hashtable.
461 uint64_t MemProfSectionOffset = OS.tell();
462 // Reserve space for the MemProf table field to be patched later if this
463 // profile contains memory profile information.
464 OS.write(0);
465
466 // Save the location of binary ids section.
467 uint64_t BinaryIdSectionOffset = OS.tell();
468 // Reserve space for the BinaryIdOffset field to be patched later if this
469 // profile contains binary ids.
470 OS.write(0);
471
472 uint64_t TemporalProfTracesOffset = OS.tell();
473 OS.write(0);
474
475 // Reserve space to write profile summary data.
477 uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
478 // Remember the summary offset.
479 uint64_t SummaryOffset = OS.tell();
480 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
481 OS.write(0);
482 uint64_t CSSummaryOffset = 0;
483 uint64_t CSSummarySize = 0;
484 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
485 CSSummaryOffset = OS.tell();
486 CSSummarySize = SummarySize / sizeof(uint64_t);
487 for (unsigned I = 0; I < CSSummarySize; I++)
488 OS.write(0);
489 }
490
491 // Write the hash table.
492 uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
493
494 // Write the MemProf profile data if we have it. This includes a simple schema
495 // with the format described below followed by the hashtable:
496 // uint64_t RecordTableOffset = RecordTableGenerator.Emit
497 // uint64_t FramePayloadOffset = Stream offset before emitting the frame table
498 // uint64_t FrameTableOffset = FrameTableGenerator.Emit
499 // uint64_t Num schema entries
500 // uint64_t Schema entry 0
501 // uint64_t Schema entry 1
502 // ....
503 // uint64_t Schema entry N - 1
504 // OnDiskChainedHashTable MemProfRecordData
505 // OnDiskChainedHashTable MemProfFrameData
506 uint64_t MemProfSectionStart = 0;
507 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
508 MemProfSectionStart = OS.tell();
509 OS.write(0ULL); // Reserve space for the memprof record table offset.
510 OS.write(0ULL); // Reserve space for the memprof frame payload offset.
511 OS.write(0ULL); // Reserve space for the memprof frame table offset.
512
514 OS.write(static_cast<uint64_t>(Schema.size()));
515 for (const auto Id : Schema) {
516 OS.write(static_cast<uint64_t>(Id));
517 }
518
519 auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
520 RecordWriter->Schema = &Schema;
522 RecordTableGenerator;
523 for (auto &I : MemProfRecordData) {
524 // Insert the key (func hash) and value (memprof record).
525 RecordTableGenerator.insert(I.first, I.second);
526 }
527
528 uint64_t RecordTableOffset =
529 RecordTableGenerator.Emit(OS.OS, *RecordWriter);
530
531 uint64_t FramePayloadOffset = OS.tell();
532
533 auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
535 FrameTableGenerator;
536 for (auto &I : MemProfFrameData) {
537 // Insert the key (frame id) and value (frame contents).
538 FrameTableGenerator.insert(I.first, I.second);
539 }
540
541 uint64_t FrameTableOffset = FrameTableGenerator.Emit(OS.OS, *FrameWriter);
542
543 PatchItem PatchItems[] = {
544 {MemProfSectionStart, &RecordTableOffset, 1},
545 {MemProfSectionStart + sizeof(uint64_t), &FramePayloadOffset, 1},
546 {MemProfSectionStart + 2 * sizeof(uint64_t), &FrameTableOffset, 1},
547 };
548 OS.patch(PatchItems, 3);
549 }
550
551 // BinaryIdSection has two parts:
552 // 1. uint64_t BinaryIdsSectionSize
553 // 2. list of binary ids that consist of:
554 // a. uint64_t BinaryIdLength
555 // b. uint8_t BinaryIdData
556 // c. uint8_t Padding (if necessary)
557 uint64_t BinaryIdSectionStart = OS.tell();
558 // Calculate size of binary section.
559 uint64_t BinaryIdsSectionSize = 0;
560
561 // Remove duplicate binary ids.
562 llvm::sort(BinaryIds);
563 BinaryIds.erase(std::unique(BinaryIds.begin(), BinaryIds.end()),
564 BinaryIds.end());
565
566 for (auto BI : BinaryIds) {
567 // Increment by binary id length data type size.
568 BinaryIdsSectionSize += sizeof(uint64_t);
569 // Increment by binary id data length, aligned to 8 bytes.
570 BinaryIdsSectionSize += alignToPowerOf2(BI.size(), sizeof(uint64_t));
571 }
572 // Write binary ids section size.
573 OS.write(BinaryIdsSectionSize);
574
575 for (auto BI : BinaryIds) {
576 uint64_t BILen = BI.size();
577 // Write binary id length.
578 OS.write(BILen);
579 // Write binary id data.
580 for (unsigned K = 0; K < BILen; K++)
581 OS.writeByte(BI[K]);
582 // Write padding if necessary.
583 uint64_t PaddingSize = alignToPowerOf2(BILen, sizeof(uint64_t)) - BILen;
584 for (unsigned K = 0; K < PaddingSize; K++)
585 OS.writeByte(0);
586 }
587
588 uint64_t TemporalProfTracesSectionStart = 0;
589 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
590 TemporalProfTracesSectionStart = OS.tell();
591 OS.write(TemporalProfTraces.size());
592 OS.write(TemporalProfTraceStreamSize);
593 for (auto &Trace : TemporalProfTraces) {
594 OS.write(Trace.Weight);
595 OS.write(Trace.FunctionNameRefs.size());
596 for (auto &NameRef : Trace.FunctionNameRefs)
597 OS.write(NameRef);
598 }
599 }
600
601 // Allocate space for data to be serialized out.
602 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
604 // Compute the Summary and copy the data to the data
605 // structure to be serialized out (to disk or buffer).
606 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
607 setSummary(TheSummary.get(), *PS);
608 InfoObj->SummaryBuilder = nullptr;
609
610 // For Context Sensitive summary.
611 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
612 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
613 TheCSSummary = IndexedInstrProf::allocSummary(SummarySize);
614 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
615 setSummary(TheCSSummary.get(), *CSPS);
616 }
617 InfoObj->CSSummaryBuilder = nullptr;
618
619 // Now do the final patch:
620 PatchItem PatchItems[] = {
621 // Patch the Header.HashOffset field.
622 {HashTableStartFieldOffset, &HashTableStart, 1},
623 // Patch the Header.MemProfOffset (=0 for profiles without MemProf
624 // data).
625 {MemProfSectionOffset, &MemProfSectionStart, 1},
626 // Patch the Header.BinaryIdSectionOffset.
627 {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
628 // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
629 // traces).
630 {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
631 // Patch the summary data.
632 {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
633 (int)(SummarySize / sizeof(uint64_t))},
634 {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()),
635 (int)CSSummarySize}};
636
637 OS.patch(PatchItems, std::size(PatchItems));
638
639 for (const auto &I : FunctionData)
640 for (const auto &F : I.getValue())
641 if (Error E = validateRecord(F.second))
642 return E;
643
644 return Error::success();
645}
646
648 // Write the hash table.
649 ProfOStream POS(OS);
650 return writeImpl(POS);
651}
652
653std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
654 std::string Data;
656 ProfOStream POS(OS);
657 // Write the hash table.
658 if (Error E = writeImpl(POS))
659 return nullptr;
660 // Return this in an aligned memory buffer.
662}
663
664static const char *ValueProfKindStr[] = {
665#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
667};
668
670 for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
671 uint32_t NS = Func.getNumValueSites(VK);
672 if (!NS)
673 continue;
674 for (uint32_t S = 0; S < NS; S++) {
675 uint32_t ND = Func.getNumValueDataForSite(VK, S);
676 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
677 DenseSet<uint64_t> SeenValues;
678 for (uint32_t I = 0; I < ND; I++)
679 if ((VK != IPVK_IndirectCallTarget) && !SeenValues.insert(VD[I].Value).second)
680 return make_error<InstrProfError>(instrprof_error::invalid_prof);
681 }
682 }
683
684 return Error::success();
685}
686
688 const InstrProfRecord &Func,
689 InstrProfSymtab &Symtab,
691 OS << Name << "\n";
692 OS << "# Func Hash:\n" << Hash << "\n";
693 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
694 OS << "# Counter Values:\n";
695 for (uint64_t Count : Func.Counts)
696 OS << Count << "\n";
697
698 uint32_t NumValueKinds = Func.getNumValueKinds();
699 if (!NumValueKinds) {
700 OS << "\n";
701 return;
702 }
703
704 OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
705 for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
706 uint32_t NS = Func.getNumValueSites(VK);
707 if (!NS)
708 continue;
709 OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
710 OS << "# NumValueSites:\n" << NS << "\n";
711 for (uint32_t S = 0; S < NS; S++) {
712 uint32_t ND = Func.getNumValueDataForSite(VK, S);
713 OS << ND << "\n";
714 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
715 for (uint32_t I = 0; I < ND; I++) {
716 if (VK == IPVK_IndirectCallTarget)
717 OS << Symtab.getFuncNameOrExternalSymbol(VD[I].Value) << ":"
718 << VD[I].Count << "\n";
719 else
720 OS << VD[I].Value << ":" << VD[I].Count << "\n";
721 }
722 }
723 }
724
725 OS << "\n";
726}
727
729 // Check CS first since it implies an IR level profile.
730 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
731 OS << "# CSIR level Instrumentation Flag\n:csir\n";
732 else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
733 OS << "# IR level Instrumentation Flag\n:ir\n";
734
735 if (static_cast<bool>(ProfileKind &
737 OS << "# Always instrument the function entry block\n:entry_first\n";
738 InstrProfSymtab Symtab;
739
741 using RecordType = std::pair<StringRef, FuncPair>;
742 SmallVector<RecordType, 4> OrderedFuncData;
743
744 for (const auto &I : FunctionData) {
745 if (shouldEncodeData(I.getValue())) {
746 if (Error E = Symtab.addFuncName(I.getKey()))
747 return E;
748 for (const auto &Func : I.getValue())
749 OrderedFuncData.push_back(std::make_pair(I.getKey(), Func));
750 }
751 }
752
753 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
755
756 llvm::sort(OrderedFuncData, [](const RecordType &A, const RecordType &B) {
757 return std::tie(A.first, A.second.first) <
758 std::tie(B.first, B.second.first);
759 });
760
761 for (const auto &record : OrderedFuncData) {
762 const StringRef &Name = record.first;
763 const FuncPair &Func = record.second;
764 writeRecordInText(Name, Func.first, Func.second, Symtab, OS);
765 }
766
767 for (const auto &record : OrderedFuncData) {
768 const FuncPair &Func = record.second;
769 if (Error E = validateRecord(Func.second))
770 return E;
771 }
772
773 return Error::success();
774}
775
777 InstrProfSymtab &Symtab) {
778 OS << ":temporal_prof_traces\n";
779 OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";
780 OS << "# Temporal Profile Trace Stream Size:\n"
781 << TemporalProfTraceStreamSize << "\n";
782 for (auto &Trace : TemporalProfTraces) {
783 OS << "# Weight:\n" << Trace.Weight << "\n";
784 for (auto &NameRef : Trace.FunctionNameRefs)
785 OS << Symtab.getFuncName(NameRef) << ",";
786 OS << "\n";
787 }
788 OS << "\n";
789}
basic Basic Alias true
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
std::string Name
static void setSummary(IndexedInstrProf::Summary *TheSummary, ProfileSummary &PS)
static const char * ValueProfKindStr[]
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Defines facilities for reading and writing on-disk hash tables.
#define P(N)
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file implements a set that has insertion order iteration characteristics.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
Definition: DenseMap.h:71
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Lightweight error class with error context and mandatory checking.
Definition: Error.h:156
static ErrorSuccess success()
Create a success value.
Definition: Error.h:330
static std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
const InstrProfWriter::ProfilingData *const data_type_ref
InstrProfSummaryBuilder * SummaryBuilder
static hash_value_type ComputeHash(key_type_ref K)
support::endianness ValueProfDataEndianness
InstrProfSummaryBuilder * CSSummaryBuilder
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N)
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
const InstrProfWriter::ProfilingData *const data_type
void addRecord(const InstrProfRecord &)
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:459
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:520
StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash)
Just like getFuncName, except that it will return a non-empty StringRef if the function is external t...
Definition: InstrProf.h:610
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
void setValueProfDataEndianness(support::endianness Endianness)
Error write(raw_fd_ostream &OS)
Write the profile to OS.
void addTemporalProfileTraces(SmallVectorImpl< TemporalProfTraceTy > &SrcTraces, uint64_t SrcStreamSize)
Add SrcTraces using reservoir sampling where SrcStreamSize is the total number of temporal profiling ...
void overlapRecord(NamedInstrProfRecord &&Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, const OverlapFuncFilters &FuncFilter)
Error writeText(raw_fd_ostream &OS)
Write the profile in text format to OS.
InstrProfWriter(bool Sparse=false, uint64_t TemporalProfTraceReservoirSize=0, uint64_t MaxTemporalProfTraceLength=0)
void addBinaryIds(ArrayRef< llvm::object::BuildID > BIs)
void addMemProfRecord(const GlobalValue::GUID Id, const memprof::IndexedMemProfRecord &Record)
Add a memprof record for a function identified by its Id.
static void writeRecordInText(StringRef Name, uint64_t Hash, const InstrProfRecord &Counters, InstrProfSymtab &Symtab, raw_fd_ostream &OS)
Write Record in text format to OS.
void addRecord(NamedInstrProfRecord &&I, uint64_t Weight, function_ref< void(Error)> Warn)
Add function counts for the given function.
void mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref< void(Error)> Warn)
Merge existing function counts from the given writer.
void writeTextTemporalProfTraceData(raw_fd_ostream &OS, InstrProfSymtab &Symtab)
Write temporal profile trace data to the header in text format to OS.
std::unique_ptr< MemoryBuffer > writeBuffer()
Write the profile, returning the raw data. For testing.
bool addMemProfFrame(const memprof::FrameId, const memprof::Frame &F, function_ref< void(Error)> Warn)
Add a memprof frame identified by the hash of the contents of the frame in FrameId.
void setOutputSparse(bool Sparse)
Error validateRecord(const InstrProfRecord &Func)
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it.
Generates an on disk hash table.
offset_type Emit(raw_ostream &Out)
Emit the table to Out, which must not be at offset 0.
void patch(PatchItem *P, int NItems)
void writeByte(uint8_t V)
ProfOStream(raw_string_ostream &STR)
support::endian::Writer LE
ProfOStream(raw_fd_ostream &FD)
void write(uint64_t V)
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:65
uint64_t getTotalCount() const
uint64_t getMaxCount() const
const SummaryEntryVector & getDetailedSummary()
uint32_t getNumCounts() const
uint64_t getMaxInternalCount() const
uint64_t getMaxFunctionCount() const
uint32_t getNumFunctions() const
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:152
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:312
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool empty() const
Definition: Trace.h:96
unsigned size() const
Definition: Trace.h:95
LLVM Value Representation.
Definition: Value.h:74
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:454
uint64_t seek(uint64_t off)
Flushes the stream and repositions the underlying file descriptor position to the offset specified fr...
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
uint64_t tell() const
tell - Return the current offset with the file.
Definition: raw_ostream.h:134
raw_ostream & write(unsigned char C)
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:660
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1187
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1077
const uint64_t Magic
Definition: InstrProf.h:1038
const HashT HashType
Definition: InstrProf.h:1075
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition: STLExtras.h:945
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
Definition: MathExtras.h:490
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:2129
void shuffle(Iterator first, Iterator last, RNG &&g)
Definition: STLExtras.h:1638
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1826
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1744
instrprof_error
Definition: InstrProf.h:310
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
uint64_t Pos
uint64_t * D
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
void set(SummaryFieldKind K, uint64_t V)
Definition: InstrProf.h:1173
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
Definition: InstrProf.h:1179
Profiling information for a single function.
Definition: InstrProf.h:743
std::vector< uint64_t > Counts
Definition: InstrProf.h:744
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:712
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:608
void sortValueData()
Sort value profile data (per site) by count.
Definition: InstrProf.h:813
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:763
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:936
const std::string NameFilter
Definition: InstrProf.h:708
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1254
CountSumOrPercent Overlap
Definition: InstrProf.h:670
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1264
CountSumOrPercent Test
Definition: InstrProf.h:668
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
Definition: InstrProf.h:339
void merge(const IndexedMemProfRecord &Other)
Definition: MemProf.h:340
static MemProfSchema getSchema()
Definition: MemProf.h:102
Adapter to write values to a stream in a particular byte order.
Definition: EndianStream.h:59
void write(ArrayRef< value_type > Val)
Definition: EndianStream.h:63