LLVM 18.0.0git
InstrProfWriter.cpp
Go to the documentation of this file.
1//===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for writing profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/StringRef.h"
22#include "llvm/Support/Endian.h"
24#include "llvm/Support/Error.h"
28#include <cstdint>
29#include <memory>
30#include <string>
31#include <tuple>
32#include <utility>
33#include <vector>
34
35using namespace llvm;
36
37// A struct to define how the data stream should be patched. For Indexed
38// profiling, only uint64_t data type is needed.
39struct PatchItem {
40 uint64_t Pos; // Where to patch.
41 uint64_t *D; // Pointer to an array of source data.
42 int N; // Number of elements in \c D array.
43};
44
45namespace llvm {
46
47// A wrapper class to abstract writer stream with support of bytes
48// back patching.
50public:
52 : IsFDOStream(true), OS(FD), LE(FD, support::little) {}
54 : IsFDOStream(false), OS(STR), LE(STR, support::little) {}
55
56 uint64_t tell() { return OS.tell(); }
57 void write(uint64_t V) { LE.write<uint64_t>(V); }
58 void writeByte(uint8_t V) { LE.write<uint8_t>(V); }
59
60 // \c patch can only be called when all data is written and flushed.
61 // For raw_string_ostream, the patch is done on the target string
62 // directly and it won't be reflected in the stream's internal buffer.
63 void patch(PatchItem *P, int NItems) {
64 using namespace support;
65
66 if (IsFDOStream) {
67 raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
68 const uint64_t LastPos = FDOStream.tell();
69 for (int K = 0; K < NItems; K++) {
70 FDOStream.seek(P[K].Pos);
71 for (int I = 0; I < P[K].N; I++)
72 write(P[K].D[I]);
73 }
74 // Reset the stream to the last position after patching so that users
75 // don't accidentally overwrite data. This makes it consistent with
76 // the string stream below which replaces the data directly.
77 FDOStream.seek(LastPos);
78 } else {
79 raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
80 std::string &Data = SOStream.str(); // with flush
81 for (int K = 0; K < NItems; K++) {
82 for (int I = 0; I < P[K].N; I++) {
83 uint64_t Bytes = endian::byte_swap<uint64_t, little>(P[K].D[I]);
84 Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t),
85 (const char *)&Bytes, sizeof(uint64_t));
86 }
87 }
88 }
89 }
90
91 // If \c OS is an instance of \c raw_fd_ostream, this field will be
92 // true. Otherwise, \c OS will be an raw_string_ostream.
96};
97
99public:
102
105
108
112
114
117 }
118
119 static std::pair<offset_type, offset_type>
121 using namespace support;
122
123 endian::Writer LE(Out, little);
124
125 offset_type N = K.size();
126 LE.write<offset_type>(N);
127
128 offset_type M = 0;
129 for (const auto &ProfileData : *V) {
130 const InstrProfRecord &ProfRecord = ProfileData.second;
131 M += sizeof(uint64_t); // The function hash
132 M += sizeof(uint64_t); // The size of the Counts vector
133 M += ProfRecord.Counts.size() * sizeof(uint64_t);
134
135 // Value data
136 M += ValueProfData::getSize(ProfileData.second);
137 }
138 LE.write<offset_type>(M);
139
140 return std::make_pair(N, M);
141 }
142
144 Out.write(K.data(), N);
145 }
146
148 using namespace support;
149
150 endian::Writer LE(Out, little);
151 for (const auto &ProfileData : *V) {
152 const InstrProfRecord &ProfRecord = ProfileData.second;
153 if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))
154 CSSummaryBuilder->addRecord(ProfRecord);
155 else
156 SummaryBuilder->addRecord(ProfRecord);
157
158 LE.write<uint64_t>(ProfileData.first); // Function hash
159 LE.write<uint64_t>(ProfRecord.Counts.size());
160 for (uint64_t I : ProfRecord.Counts)
161 LE.write<uint64_t>(I);
162
163 // Write value data
164 std::unique_ptr<ValueProfData> VDataPtr =
165 ValueProfData::serializeFrom(ProfileData.second);
166 uint32_t S = VDataPtr->getSize();
167 VDataPtr->swapBytesFromHost(ValueProfDataEndianness);
168 Out.write((const char *)VDataPtr.get(), S);
169 }
170 }
171};
172
173} // end namespace llvm
174
176 uint64_t TemporalProfTraceReservoirSize,
177 uint64_t MaxTemporalProfTraceLength)
178 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
179 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
180 InfoObj(new InstrProfRecordWriterTrait()) {}
181
183
184// Internal interface for testing purpose only.
186 support::endianness Endianness) {
187 InfoObj->ValueProfDataEndianness = Endianness;
188}
189
191 this->Sparse = Sparse;
192}
193
195 function_ref<void(Error)> Warn) {
196 auto Name = I.Name;
197 auto Hash = I.Hash;
198 addRecord(Name, Hash, std::move(I), Weight, Warn);
199}
200
202 OverlapStats &Overlap,
203 OverlapStats &FuncLevelOverlap,
204 const OverlapFuncFilters &FuncFilter) {
205 auto Name = Other.Name;
206 auto Hash = Other.Hash;
207 Other.accumulateCounts(FuncLevelOverlap.Test);
208 if (!FunctionData.contains(Name)) {
209 Overlap.addOneUnique(FuncLevelOverlap.Test);
210 return;
211 }
212 if (FuncLevelOverlap.Test.CountSum < 1.0f) {
213 Overlap.Overlap.NumEntries += 1;
214 return;
215 }
216 auto &ProfileDataMap = FunctionData[Name];
217 bool NewFunc;
219 std::tie(Where, NewFunc) =
220 ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
221 if (NewFunc) {
222 Overlap.addOneMismatch(FuncLevelOverlap.Test);
223 return;
224 }
225 InstrProfRecord &Dest = Where->second;
226
227 uint64_t ValueCutoff = FuncFilter.ValueCutoff;
228 if (!FuncFilter.NameFilter.empty() && Name.contains(FuncFilter.NameFilter))
229 ValueCutoff = 0;
230
231 Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
232}
233
235 InstrProfRecord &&I, uint64_t Weight,
236 function_ref<void(Error)> Warn) {
237 auto &ProfileDataMap = FunctionData[Name];
238
239 bool NewFunc;
241 std::tie(Where, NewFunc) =
242 ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
243 InstrProfRecord &Dest = Where->second;
244
245 auto MapWarn = [&](instrprof_error E) {
246 Warn(make_error<InstrProfError>(E));
247 };
248
249 if (NewFunc) {
250 // We've never seen a function with this name and hash, add it.
251 Dest = std::move(I);
252 if (Weight > 1)
253 Dest.scale(Weight, 1, MapWarn);
254 } else {
255 // We're updating a function we've seen before.
256 Dest.merge(I, Weight, MapWarn);
257 }
258
259 Dest.sortValueData();
260}
261
264 auto Result = MemProfRecordData.insert({Id, Record});
265 // If we inserted a new record then we are done.
266 if (Result.second) {
267 return;
268 }
269 memprof::IndexedMemProfRecord &Existing = Result.first->second;
270 Existing.merge(Record);
271}
272
274 const memprof::Frame &Frame,
275 function_ref<void(Error)> Warn) {
276 auto Result = MemProfFrameData.insert({Id, Frame});
277 // If a mapping already exists for the current frame id and it does not
278 // match the new mapping provided then reset the existing contents and bail
279 // out. We don't support the merging of memprof data whose Frame -> Id
280 // mapping across profiles is inconsistent.
281 if (!Result.second && Result.first->second != Frame) {
282 Warn(make_error<InstrProfError>(instrprof_error::malformed,
283 "frame to id mapping mismatch"));
284 return false;
285 }
286 return true;
287}
288
290 llvm::append_range(BinaryIds, BIs);
291}
292
293void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) {
294 if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
295 Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);
296 if (Trace.FunctionNameRefs.empty())
297 return;
298
299 if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
300 // Simply append the trace if we have not yet hit our reservoir size limit.
301 TemporalProfTraces.push_back(std::move(Trace));
302 } else {
303 // Otherwise, replace a random trace in the stream.
304 std::uniform_int_distribution<uint64_t> Distribution(
305 0, TemporalProfTraceStreamSize);
306 uint64_t RandomIndex = Distribution(RNG);
307 if (RandomIndex < TemporalProfTraces.size())
308 TemporalProfTraces[RandomIndex] = std::move(Trace);
309 }
310 ++TemporalProfTraceStreamSize;
311}
312
314 SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
315 // Assume that the source has the same reservoir size as the destination to
316 // avoid needing to record it in the indexed profile format.
317 bool IsDestSampled =
318 (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize);
319 bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize);
320 if (!IsDestSampled && IsSrcSampled) {
321 // If one of the traces are sampled, ensure that it belongs to Dest.
322 std::swap(TemporalProfTraces, SrcTraces);
323 std::swap(TemporalProfTraceStreamSize, SrcStreamSize);
324 std::swap(IsDestSampled, IsSrcSampled);
325 }
326 if (!IsSrcSampled) {
327 // If the source stream is not sampled, we add each source trace normally.
328 for (auto &Trace : SrcTraces)
329 addTemporalProfileTrace(std::move(Trace));
330 return;
331 }
332 // Otherwise, we find the traces that would have been removed if we added
333 // the whole source stream.
334 SmallSetVector<uint64_t, 8> IndicesToReplace;
335 for (uint64_t I = 0; I < SrcStreamSize; I++) {
336 std::uniform_int_distribution<uint64_t> Distribution(
337 0, TemporalProfTraceStreamSize);
338 uint64_t RandomIndex = Distribution(RNG);
339 if (RandomIndex < TemporalProfTraces.size())
340 IndicesToReplace.insert(RandomIndex);
341 ++TemporalProfTraceStreamSize;
342 }
343 // Then we insert a random sample of the source traces.
344 llvm::shuffle(SrcTraces.begin(), SrcTraces.end(), RNG);
345 for (const auto &[Index, Trace] : llvm::zip(IndicesToReplace, SrcTraces))
346 TemporalProfTraces[Index] = std::move(Trace);
347}
348
350 function_ref<void(Error)> Warn) {
351 for (auto &I : IPW.FunctionData)
352 for (auto &Func : I.getValue())
353 addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn);
354
355 BinaryIds.reserve(BinaryIds.size() + IPW.BinaryIds.size());
356 for (auto &I : IPW.BinaryIds)
358
359 addTemporalProfileTraces(IPW.TemporalProfTraces,
360 IPW.TemporalProfTraceStreamSize);
361
362 MemProfFrameData.reserve(IPW.MemProfFrameData.size());
363 for (auto &I : IPW.MemProfFrameData) {
364 // If we weren't able to add the frame mappings then it doesn't make sense
365 // to try to merge the records from this profile.
366 if (!addMemProfFrame(I.first, I.second, Warn))
367 return;
368 }
369
370 MemProfRecordData.reserve(IPW.MemProfRecordData.size());
371 for (auto &I : IPW.MemProfRecordData) {
372 addMemProfRecord(I.first, I.second);
373 }
374}
375
376bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
377 if (!Sparse)
378 return true;
379 for (const auto &Func : PD) {
380 const InstrProfRecord &IPR = Func.second;
381 if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; }))
382 return true;
383 }
384 return false;
385}
386
387static void setSummary(IndexedInstrProf::Summary *TheSummary,
388 ProfileSummary &PS) {
389 using namespace IndexedInstrProf;
390
391 const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
392 TheSummary->NumSummaryFields = Summary::NumKinds;
393 TheSummary->NumCutoffEntries = Res.size();
394 TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());
395 TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());
396 TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());
397 TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());
398 TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());
399 TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());
400 for (unsigned I = 0; I < Res.size(); I++)
401 TheSummary->setEntry(I, Res[I]);
402}
403
404Error InstrProfWriter::writeImpl(ProfOStream &OS) {
405 using namespace IndexedInstrProf;
406 using namespace support;
407
409
411 InfoObj->SummaryBuilder = &ISB;
413 InfoObj->CSSummaryBuilder = &CSISB;
414
415 // Populate the hash table generator.
417 for (const auto &I : FunctionData)
418 if (shouldEncodeData(I.getValue()))
419 OrderedData.emplace_back((I.getKey()), &I.getValue());
420 llvm::sort(OrderedData, less_first());
421 for (const auto &I : OrderedData)
422 Generator.insert(I.first, I.second);
423
424 // Write the header.
426 Header.Magic = IndexedInstrProf::Magic;
428 if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
429 Header.Version |= VARIANT_MASK_IR_PROF;
430 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
431 Header.Version |= VARIANT_MASK_CSIR_PROF;
432 if (static_cast<bool>(ProfileKind &
434 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
435 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
436 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
437 if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
438 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
439 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
440 Header.Version |= VARIANT_MASK_MEMPROF;
441 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
442 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
443
444 Header.Unused = 0;
445 Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
446 Header.HashOffset = 0;
447 Header.MemProfOffset = 0;
448 Header.BinaryIdOffset = 0;
449 Header.TemporalProfTracesOffset = 0;
450 int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t);
451
452 // Only write out all the fields except 'HashOffset', 'MemProfOffset',
453 // 'BinaryIdOffset' and `TemporalProfTracesOffset`. We need to remember the
454 // offset of these fields to allow back patching later.
455 for (int I = 0; I < N - 4; I++)
456 OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
457
458 // Save the location of Header.HashOffset field in \c OS.
459 uint64_t HashTableStartFieldOffset = OS.tell();
460 // Reserve the space for HashOffset field.
461 OS.write(0);
462
463 // Save the location of MemProf profile data. This is stored in two parts as
464 // the schema and as a separate on-disk chained hashtable.
465 uint64_t MemProfSectionOffset = OS.tell();
466 // Reserve space for the MemProf table field to be patched later if this
467 // profile contains memory profile information.
468 OS.write(0);
469
470 // Save the location of binary ids section.
471 uint64_t BinaryIdSectionOffset = OS.tell();
472 // Reserve space for the BinaryIdOffset field to be patched later if this
473 // profile contains binary ids.
474 OS.write(0);
475
476 uint64_t TemporalProfTracesOffset = OS.tell();
477 OS.write(0);
478
479 // Reserve space to write profile summary data.
481 uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
482 // Remember the summary offset.
483 uint64_t SummaryOffset = OS.tell();
484 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
485 OS.write(0);
486 uint64_t CSSummaryOffset = 0;
487 uint64_t CSSummarySize = 0;
488 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
489 CSSummaryOffset = OS.tell();
490 CSSummarySize = SummarySize / sizeof(uint64_t);
491 for (unsigned I = 0; I < CSSummarySize; I++)
492 OS.write(0);
493 }
494
495 // Write the hash table.
496 uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
497
498 // Write the MemProf profile data if we have it. This includes a simple schema
499 // with the format described below followed by the hashtable:
500 // uint64_t RecordTableOffset = RecordTableGenerator.Emit
501 // uint64_t FramePayloadOffset = Stream offset before emitting the frame table
502 // uint64_t FrameTableOffset = FrameTableGenerator.Emit
503 // uint64_t Num schema entries
504 // uint64_t Schema entry 0
505 // uint64_t Schema entry 1
506 // ....
507 // uint64_t Schema entry N - 1
508 // OnDiskChainedHashTable MemProfRecordData
509 // OnDiskChainedHashTable MemProfFrameData
510 uint64_t MemProfSectionStart = 0;
511 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
512 MemProfSectionStart = OS.tell();
513 OS.write(0ULL); // Reserve space for the memprof record table offset.
514 OS.write(0ULL); // Reserve space for the memprof frame payload offset.
515 OS.write(0ULL); // Reserve space for the memprof frame table offset.
516
518 OS.write(static_cast<uint64_t>(Schema.size()));
519 for (const auto Id : Schema) {
520 OS.write(static_cast<uint64_t>(Id));
521 }
522
523 auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
524 RecordWriter->Schema = &Schema;
526 RecordTableGenerator;
527 for (auto &I : MemProfRecordData) {
528 // Insert the key (func hash) and value (memprof record).
529 RecordTableGenerator.insert(I.first, I.second);
530 }
531
532 uint64_t RecordTableOffset =
533 RecordTableGenerator.Emit(OS.OS, *RecordWriter);
534
535 uint64_t FramePayloadOffset = OS.tell();
536
537 auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
539 FrameTableGenerator;
540 for (auto &I : MemProfFrameData) {
541 // Insert the key (frame id) and value (frame contents).
542 FrameTableGenerator.insert(I.first, I.second);
543 }
544
545 uint64_t FrameTableOffset = FrameTableGenerator.Emit(OS.OS, *FrameWriter);
546
547 PatchItem PatchItems[] = {
548 {MemProfSectionStart, &RecordTableOffset, 1},
549 {MemProfSectionStart + sizeof(uint64_t), &FramePayloadOffset, 1},
550 {MemProfSectionStart + 2 * sizeof(uint64_t), &FrameTableOffset, 1},
551 };
552 OS.patch(PatchItems, 3);
553 }
554
555 // BinaryIdSection has two parts:
556 // 1. uint64_t BinaryIdsSectionSize
557 // 2. list of binary ids that consist of:
558 // a. uint64_t BinaryIdLength
559 // b. uint8_t BinaryIdData
560 // c. uint8_t Padding (if necessary)
561 uint64_t BinaryIdSectionStart = OS.tell();
562 // Calculate size of binary section.
563 uint64_t BinaryIdsSectionSize = 0;
564
565 // Remove duplicate binary ids.
566 llvm::sort(BinaryIds);
567 BinaryIds.erase(std::unique(BinaryIds.begin(), BinaryIds.end()),
568 BinaryIds.end());
569
570 for (auto BI : BinaryIds) {
571 // Increment by binary id length data type size.
572 BinaryIdsSectionSize += sizeof(uint64_t);
573 // Increment by binary id data length, aligned to 8 bytes.
574 BinaryIdsSectionSize += alignToPowerOf2(BI.size(), sizeof(uint64_t));
575 }
576 // Write binary ids section size.
577 OS.write(BinaryIdsSectionSize);
578
579 for (auto BI : BinaryIds) {
580 uint64_t BILen = BI.size();
581 // Write binary id length.
582 OS.write(BILen);
583 // Write binary id data.
584 for (unsigned K = 0; K < BILen; K++)
585 OS.writeByte(BI[K]);
586 // Write padding if necessary.
587 uint64_t PaddingSize = alignToPowerOf2(BILen, sizeof(uint64_t)) - BILen;
588 for (unsigned K = 0; K < PaddingSize; K++)
589 OS.writeByte(0);
590 }
591
592 uint64_t TemporalProfTracesSectionStart = 0;
593 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
594 TemporalProfTracesSectionStart = OS.tell();
595 OS.write(TemporalProfTraces.size());
596 OS.write(TemporalProfTraceStreamSize);
597 for (auto &Trace : TemporalProfTraces) {
598 OS.write(Trace.Weight);
599 OS.write(Trace.FunctionNameRefs.size());
600 for (auto &NameRef : Trace.FunctionNameRefs)
601 OS.write(NameRef);
602 }
603 }
604
605 // Allocate space for data to be serialized out.
606 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
608 // Compute the Summary and copy the data to the data
609 // structure to be serialized out (to disk or buffer).
610 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
611 setSummary(TheSummary.get(), *PS);
612 InfoObj->SummaryBuilder = nullptr;
613
614 // For Context Sensitive summary.
615 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
616 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
617 TheCSSummary = IndexedInstrProf::allocSummary(SummarySize);
618 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
619 setSummary(TheCSSummary.get(), *CSPS);
620 }
621 InfoObj->CSSummaryBuilder = nullptr;
622
623 // Now do the final patch:
624 PatchItem PatchItems[] = {
625 // Patch the Header.HashOffset field.
626 {HashTableStartFieldOffset, &HashTableStart, 1},
627 // Patch the Header.MemProfOffset (=0 for profiles without MemProf
628 // data).
629 {MemProfSectionOffset, &MemProfSectionStart, 1},
630 // Patch the Header.BinaryIdSectionOffset.
631 {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
632 // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
633 // traces).
634 {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
635 // Patch the summary data.
636 {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
637 (int)(SummarySize / sizeof(uint64_t))},
638 {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()),
639 (int)CSSummarySize}};
640
641 OS.patch(PatchItems, std::size(PatchItems));
642
643 for (const auto &I : FunctionData)
644 for (const auto &F : I.getValue())
645 if (Error E = validateRecord(F.second))
646 return E;
647
648 return Error::success();
649}
650
652 // Write the hash table.
653 ProfOStream POS(OS);
654 return writeImpl(POS);
655}
656
658 ProfOStream POS(OS);
659 return writeImpl(POS);
660}
661
662std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
663 std::string Data;
665 // Write the hash table.
666 if (Error E = write(OS))
667 return nullptr;
668 // Return this in an aligned memory buffer.
670}
671
672static const char *ValueProfKindStr[] = {
673#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
675};
676
678 for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
679 uint32_t NS = Func.getNumValueSites(VK);
680 if (!NS)
681 continue;
682 for (uint32_t S = 0; S < NS; S++) {
683 uint32_t ND = Func.getNumValueDataForSite(VK, S);
684 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
685 DenseSet<uint64_t> SeenValues;
686 for (uint32_t I = 0; I < ND; I++)
687 if ((VK != IPVK_IndirectCallTarget) && !SeenValues.insert(VD[I].Value).second)
688 return make_error<InstrProfError>(instrprof_error::invalid_prof);
689 }
690 }
691
692 return Error::success();
693}
694
696 const InstrProfRecord &Func,
697 InstrProfSymtab &Symtab,
699 OS << Name << "\n";
700 OS << "# Func Hash:\n" << Hash << "\n";
701 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
702 OS << "# Counter Values:\n";
703 for (uint64_t Count : Func.Counts)
704 OS << Count << "\n";
705
706 uint32_t NumValueKinds = Func.getNumValueKinds();
707 if (!NumValueKinds) {
708 OS << "\n";
709 return;
710 }
711
712 OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
713 for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
714 uint32_t NS = Func.getNumValueSites(VK);
715 if (!NS)
716 continue;
717 OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
718 OS << "# NumValueSites:\n" << NS << "\n";
719 for (uint32_t S = 0; S < NS; S++) {
720 uint32_t ND = Func.getNumValueDataForSite(VK, S);
721 OS << ND << "\n";
722 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
723 for (uint32_t I = 0; I < ND; I++) {
724 if (VK == IPVK_IndirectCallTarget)
725 OS << Symtab.getFuncNameOrExternalSymbol(VD[I].Value) << ":"
726 << VD[I].Count << "\n";
727 else
728 OS << VD[I].Value << ":" << VD[I].Count << "\n";
729 }
730 }
731 }
732
733 OS << "\n";
734}
735
737 // Check CS first since it implies an IR level profile.
738 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
739 OS << "# CSIR level Instrumentation Flag\n:csir\n";
740 else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
741 OS << "# IR level Instrumentation Flag\n:ir\n";
742
743 if (static_cast<bool>(ProfileKind &
745 OS << "# Always instrument the function entry block\n:entry_first\n";
746 InstrProfSymtab Symtab;
747
749 using RecordType = std::pair<StringRef, FuncPair>;
750 SmallVector<RecordType, 4> OrderedFuncData;
751
752 for (const auto &I : FunctionData) {
753 if (shouldEncodeData(I.getValue())) {
754 if (Error E = Symtab.addFuncName(I.getKey()))
755 return E;
756 for (const auto &Func : I.getValue())
757 OrderedFuncData.push_back(std::make_pair(I.getKey(), Func));
758 }
759 }
760
761 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
763
764 llvm::sort(OrderedFuncData, [](const RecordType &A, const RecordType &B) {
765 return std::tie(A.first, A.second.first) <
766 std::tie(B.first, B.second.first);
767 });
768
769 for (const auto &record : OrderedFuncData) {
770 const StringRef &Name = record.first;
771 const FuncPair &Func = record.second;
772 writeRecordInText(Name, Func.first, Func.second, Symtab, OS);
773 }
774
775 for (const auto &record : OrderedFuncData) {
776 const FuncPair &Func = record.second;
777 if (Error E = validateRecord(Func.second))
778 return E;
779 }
780
781 return Error::success();
782}
783
785 InstrProfSymtab &Symtab) {
786 OS << ":temporal_prof_traces\n";
787 OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";
788 OS << "# Temporal Profile Trace Stream Size:\n"
789 << TemporalProfTraceStreamSize << "\n";
790 for (auto &Trace : TemporalProfTraces) {
791 OS << "# Weight:\n" << Trace.Weight << "\n";
792 for (auto &NameRef : Trace.FunctionNameRefs)
793 OS << Symtab.getFuncName(NameRef) << ",";
794 OS << "\n";
795 }
796 OS << "\n";
797}
basic Basic Alias true
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
std::string Name
static void setSummary(IndexedInstrProf::Summary *TheSummary, ProfileSummary &PS)
static const char * ValueProfKindStr[]
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Defines facilities for reading and writing on-disk hash tables.
#define P(N)
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file implements a set that has insertion order iteration characteristics.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
Definition: DenseMap.h:71
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
static std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
const InstrProfWriter::ProfilingData *const data_type_ref
InstrProfSummaryBuilder * SummaryBuilder
static hash_value_type ComputeHash(key_type_ref K)
support::endianness ValueProfDataEndianness
InstrProfSummaryBuilder * CSSummaryBuilder
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N)
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
const InstrProfWriter::ProfilingData *const data_type
void addRecord(const InstrProfRecord &)
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Definition: InstrProf.h:423
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:486
StringRef getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash)
Just like getFuncName, except that it will return a non-empty StringRef if the function is external t...
Definition: InstrProf.h:568
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
void setValueProfDataEndianness(support::endianness Endianness)
Error write(raw_fd_ostream &OS)
Write the profile to OS.
void addTemporalProfileTraces(SmallVectorImpl< TemporalProfTraceTy > &SrcTraces, uint64_t SrcStreamSize)
Add SrcTraces using reservoir sampling where SrcStreamSize is the total number of temporal profiling ...
void overlapRecord(NamedInstrProfRecord &&Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, const OverlapFuncFilters &FuncFilter)
Error writeText(raw_fd_ostream &OS)
Write the profile in text format to OS.
InstrProfWriter(bool Sparse=false, uint64_t TemporalProfTraceReservoirSize=0, uint64_t MaxTemporalProfTraceLength=0)
void addBinaryIds(ArrayRef< llvm::object::BuildID > BIs)
void addMemProfRecord(const GlobalValue::GUID Id, const memprof::IndexedMemProfRecord &Record)
Add a memprof record for a function identified by its Id.
static void writeRecordInText(StringRef Name, uint64_t Hash, const InstrProfRecord &Counters, InstrProfSymtab &Symtab, raw_fd_ostream &OS)
Write Record in text format to OS.
void addRecord(NamedInstrProfRecord &&I, uint64_t Weight, function_ref< void(Error)> Warn)
Add function counts for the given function.
void mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref< void(Error)> Warn)
Merge existing function counts from the given writer.
void writeTextTemporalProfTraceData(raw_fd_ostream &OS, InstrProfSymtab &Symtab)
Write temporal profile trace data to the header in text format to OS.
std::unique_ptr< MemoryBuffer > writeBuffer()
Write the profile, returning the raw data. For testing.
bool addMemProfFrame(const memprof::FrameId, const memprof::Frame &F, function_ref< void(Error)> Warn)
Add a memprof frame identified by the hash of the contents of the frame in FrameId.
void setOutputSparse(bool Sparse)
Error validateRecord(const InstrProfRecord &Func)
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it.
Generates an on disk hash table.
offset_type Emit(raw_ostream &Out)
Emit the table to Out, which must not be at offset 0.
void patch(PatchItem *P, int NItems)
void writeByte(uint8_t V)
ProfOStream(raw_string_ostream &STR)
support::endian::Writer LE
ProfOStream(raw_fd_ostream &FD)
void write(uint64_t V)
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:62
uint64_t getTotalCount() const
uint64_t getMaxCount() const
const SummaryEntryVector & getDetailedSummary()
uint32_t getNumCounts() const
uint64_t getMaxInternalCount() const
uint64_t getMaxFunctionCount() const
uint32_t getNumFunctions() const
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:941
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool empty() const
Definition: Trace.h:96
unsigned size() const
Definition: Trace.h:95
LLVM Value Representation.
Definition: Value.h:74
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:454
uint64_t seek(uint64_t off)
Flushes the stream and repositions the underlying file descriptor position to the offset specified fr...
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
uint64_t tell() const
tell - Return the current offset with the file.
Definition: raw_ostream.h:134
raw_ostream & write(unsigned char C)
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:660
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1135
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1025
const uint64_t Magic
Definition: InstrProf.h:986
const HashT HashType
Definition: InstrProf.h:1023
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition: STLExtras.h:863
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
Definition: MathExtras.h:382
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:2037
void shuffle(Iterator first, Iterator last, RNG &&g)
Definition: STLExtras.h:1546
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1734
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1652
@ Other
Any other memory.
instrprof_error
Definition: InstrProf.h:320
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
uint64_t Pos
uint64_t * D
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
void set(SummaryFieldKind K, uint64_t V)
Definition: InstrProf.h:1121
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
Definition: InstrProf.h:1127
Profiling information for a single function.
Definition: InstrProf.h:691
std::vector< uint64_t > Counts
Definition: InstrProf.h:692
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:767
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:663
void sortValueData()
Sort value profile data (per site) by count.
Definition: InstrProf.h:761
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:818
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:884
const std::string NameFilter
Definition: InstrProf.h:656
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1351
CountSumOrPercent Overlap
Definition: InstrProf.h:618
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1361
CountSumOrPercent Test
Definition: InstrProf.h:616
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
Definition: InstrProf.h:349
Function object to check whether the first component of a container supported by std::get (like std::...
Definition: STLExtras.h:1455
void merge(const IndexedMemProfRecord &Other)
Definition: MemProf.h:340
static MemProfSchema getSchema()
Definition: MemProf.h:102
Adapter to write values to a stream in a particular byte order.
Definition: EndianStream.h:68
void write(ArrayRef< value_type > Val)
Definition: EndianStream.h:72