LLVM 19.0.0git
InstrProf.h
Go to the documentation of this file.
1//===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Instrumentation-based profiling data is generated by instrumented
10// binaries through library functions in compiler-rt, and read by the clang
11// frontend to feed PGO.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_PROFILEDATA_INSTRPROF_H
16#define LLVM_PROFILEDATA_INSTRPROF_H
17
18#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/DenseMap.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/ADT/StringSet.h"
25#include "llvm/IR/GlobalValue.h"
31#include "llvm/Support/Error.h"
33#include "llvm/Support/MD5.h"
38#include <algorithm>
39#include <cassert>
40#include <cstddef>
41#include <cstdint>
42#include <cstring>
43#include <list>
44#include <memory>
45#include <string>
46#include <system_error>
47#include <utility>
48#include <vector>
49
50namespace llvm {
51
52class Function;
53class GlobalVariable;
54struct InstrProfRecord;
55class InstrProfSymtab;
56class Instruction;
57class MDNode;
58class Module;
59
61#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
63};
64
65/// Return the max count value. We reserver a few large values for special use.
67 return std::numeric_limits<uint64_t>::max() - 2;
68}
69
70/// Return the name of the profile section corresponding to \p IPSK.
71///
72/// The name of the section depends on the object format type \p OF. If
73/// \p AddSegmentInfo is true, a segment prefix and additional linker hints may
74/// be added to the section name (this is the default).
77 bool AddSegmentInfo = true);
78
79/// Return the name profile runtime entry point to do value profiling
80/// for a given site.
82 return INSTR_PROF_VALUE_PROF_FUNC_STR;
83}
84
85/// Return the name profile runtime entry point to do memop size value
86/// profiling.
88 return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
89}
90
91/// Return the name prefix of variables containing instrumented function names.
92inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
93
94/// Return the name prefix of variables containing virtual table profile data.
95inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_"; }
96
97/// Return the name prefix of variables containing per-function control data.
98inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }
99
100/// Return the name prefix of profile counter variables.
101inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; }
102
103/// Return the name prefix of profile bitmap variables.
104inline StringRef getInstrProfBitmapVarPrefix() { return "__profbm_"; }
105
106/// Return the name prefix of value profile variables.
107inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }
108
109/// Return the name of value profile node array variables:
110inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
111
112/// Return the name of the variable holding the strings (possibly compressed)
113/// of all function's PGO names.
114inline StringRef getInstrProfNamesVarName() { return "__llvm_prf_nm"; }
115
116inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; }
117
118/// Return the name of a covarage mapping variable (internal linkage)
119/// for each instrumented source module. Such variables are allocated
120/// in the __llvm_covmap section.
122 return "__llvm_coverage_mapping";
123}
124
125/// Return the name of the internal variable recording the array
126/// of PGO name vars referenced by the coverage mapping. The owning
127/// functions of those names are not emitted by FE (e.g, unused inline
128/// functions.)
130 return "__llvm_coverage_names";
131}
132
133/// Return the name of function that registers all the per-function control
134/// data at program startup time by calling __llvm_register_function. This
135/// function has internal linkage and is called by __llvm_profile_init
136/// runtime method. This function is not generated for these platforms:
137/// Darwin, Linux, and FreeBSD.
139 return "__llvm_profile_register_functions";
140}
141
142/// Return the name of the runtime interface that registers per-function control
143/// data for one instrumented function.
145 return "__llvm_profile_register_function";
146}
147
148/// Return the name of the runtime interface that registers the PGO name
149/// strings.
151 return "__llvm_profile_register_names_function";
152}
153
154/// Return the name of the runtime initialization method that is generated by
155/// the compiler. The function calls __llvm_profile_register_functions and
156/// __llvm_profile_override_default_filename functions if needed. This function
157/// has internal linkage and invoked at startup time via init_array.
158inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; }
159
160/// Return the name of the hook variable defined in profile runtime library.
161/// A reference to the variable causes the linker to link in the runtime
162/// initialization module (which defines the hook variable).
164 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
165}
166
167/// Return the name of the compiler generated function that references the
168/// runtime hook variable. The function is a weak global.
170 return "__llvm_profile_runtime_user";
171}
172
174 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR);
175}
176
177/// Return the marker used to separate PGO names during serialization.
178inline StringRef getInstrProfNameSeparator() { return "\01"; }
179
180/// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is
181/// for front-end (Clang, etc) instrumentation.
182/// Return the modified name for function \c F suitable to be
183/// used the key for profile lookup. Variable \c InLTO indicates if this
184/// is called in LTO optimization passes.
185std::string getPGOFuncName(const Function &F, bool InLTO = false,
186 uint64_t Version = INSTR_PROF_INDEX_VERSION);
187
188/// Return the modified name for a function suitable to be
189/// used the key for profile lookup. The function's original
190/// name is \c RawFuncName and has linkage of type \c Linkage.
191/// The function is defined in module \c FileName.
192std::string getPGOFuncName(StringRef RawFuncName,
194 StringRef FileName,
195 uint64_t Version = INSTR_PROF_INDEX_VERSION);
196
197/// \return the modified name for function \c F suitable to be
198/// used as the key for IRPGO profile lookup. \c InLTO indicates if this is
199/// called from LTO optimization passes.
200std::string getIRPGOFuncName(const Function &F, bool InLTO = false);
201
202/// \return the filename and the function name parsed from the output of
203/// \c getIRPGOFuncName()
204std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName);
205
206/// Return the name of the global variable used to store a function
207/// name in PGO instrumentation. \c FuncName is the IRPGO function name
208/// (returned by \c getIRPGOFuncName) for LLVM IR instrumentation and PGO
209/// function name (returned by \c getPGOFuncName) for front-end instrumentation.
210std::string getPGOFuncNameVarName(StringRef FuncName,
212
213/// Create and return the global variable for function name used in PGO
214/// instrumentation. \c FuncName is the IRPGO function name (returned by
215/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name
216/// (returned by \c getPGOFuncName) for front-end instrumentation.
217GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);
218
219/// Create and return the global variable for function name used in PGO
220/// instrumentation. \c FuncName is the IRPGO function name (returned by
221/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name
222/// (returned by \c getPGOFuncName) for front-end instrumentation.
223GlobalVariable *createPGOFuncNameVar(Module &M,
225 StringRef PGOFuncName);
226
227/// Return the initializer in string of the PGO name var \c NameVar.
228StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);
229
230/// Given a PGO function name, remove the filename prefix and return
231/// the original (static) function name.
232StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
233 StringRef FileName = "<unknown>");
234
235/// Given a vector of strings (names of global objects like functions or,
236/// virtual tables) \c NameStrs, the method generates a combined string \c
237/// Result that is ready to be serialized. The \c Result string is comprised of
238/// three fields: The first field is the length of the uncompressed strings, and
239/// the the second field is the length of the zlib-compressed string. Both
240/// fields are encoded in ULEB128. If \c doCompress is false, the
241/// third field is the uncompressed strings; otherwise it is the
242/// compressed string. When the string compression is off, the
243/// second field will have value zero.
244Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
245 bool doCompression, std::string &Result);
246
247/// Produce \c Result string with the same format described above. The input
248/// is vector of PGO function name variables that are referenced.
249/// The global variable element in 'NameVars' is a string containing the pgo
250/// name of a function. See `createPGOFuncNameVar` that creates these global
251/// variables.
252Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
253 std::string &Result, bool doCompression = true);
254
255Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
256 std::string &Result, bool doCompression);
257
258/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
259/// set in IR PGO compilation.
260bool isIRPGOFlagSet(const Module *M);
261
262/// Check if we can safely rename this Comdat function. Instances of the same
263/// comdat function may have different control flows thus can not share the
264/// same counter variable.
265bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);
266
268#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
270};
271
272/// Get the value profile data for value site \p SiteIdx from \p InstrProfR
273/// and annotate the instruction \p Inst with the value profile meta data.
274/// Annotate up to \p MaxMDCount (default 3) number of records per value site.
275void annotateValueSite(Module &M, Instruction &Inst,
276 const InstrProfRecord &InstrProfR,
277 InstrProfValueKind ValueKind, uint32_t SiteIndx,
278 uint32_t MaxMDCount = 3);
279
280/// Same as the above interface but using an ArrayRef, as well as \p Sum.
281/// This function will not annotate !prof metadata on the instruction if the
282/// referenced array is empty.
283void annotateValueSite(Module &M, Instruction &Inst,
284 ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
285 InstrProfValueKind ValueKind, uint32_t MaxMDCount);
286
287/// Extract the value profile data from \p Inst which is annotated with
288/// value profile meta data. Return false if there is no value data annotated,
289/// otherwise return true.
290bool getValueProfDataFromInst(const Instruction &Inst,
291 InstrProfValueKind ValueKind,
292 uint32_t MaxNumValueData,
293 InstrProfValueData ValueData[],
294 uint32_t &ActualNumValueData, uint64_t &TotalC,
295 bool GetNoICPValue = false);
296
297/// Extract the value profile data from \p Inst and returns them if \p Inst is
298/// annotated with value profile data. Returns nullptr otherwise. It's similar
299/// to `getValueProfDataFromInst` above except that an array is allocated only
300/// after a preliminary checking that the value profiles of kind `ValueKind`
301/// exist.
302std::unique_ptr<InstrProfValueData[]>
303getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind,
304 uint32_t MaxNumValueData, uint32_t &ActualNumValueData,
305 uint64_t &TotalC, bool GetNoICPValue = false);
306
307inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
308
309/// Return the PGOFuncName meta data associated with a function.
310MDNode *getPGOFuncNameMetadata(const Function &F);
311
312std::string getPGOName(const GlobalVariable &V, bool InLTO = false);
313
314/// Create the PGOFuncName meta data if PGOFuncName is different from
315/// function's raw name. This should only apply to internal linkage functions
316/// declared by users only.
317void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
318
319/// Check if we can use Comdat for profile variables. This will eliminate
320/// the duplicated profile variables for Comdat functions.
321bool needsComdatForCounter(const GlobalObject &GV, const Module &M);
322
323/// An enum describing the attributes of an instrumented profile.
324enum class InstrProfKind {
325 Unknown = 0x0,
326 // A frontend clang profile, incompatible with other attrs.
328 // An IR-level profile (default when -fprofile-generate is used).
329 IRInstrumentation = 0x2,
330 // A profile with entry basic block instrumentation.
332 // A context sensitive IR-level profile.
333 ContextSensitive = 0x8,
334 // Use single byte probes for coverage.
335 SingleByteCoverage = 0x10,
336 // Only instrument the function entry basic block.
337 FunctionEntryOnly = 0x20,
338 // A memory profile collected using -fprofile=memory.
339 MemProf = 0x40,
340 // A temporal profile.
341 TemporalProfile = 0x80,
343};
344
345const std::error_category &instrprof_category();
346
347enum class instrprof_error {
348 success = 0,
349 eof,
351 bad_magic,
355 too_large,
356 truncated,
357 malformed,
374};
375
376/// An ordered list of functions identified by their NameRef found in
377/// INSTR_PROF_DATA
379 std::vector<uint64_t> FunctionNameRefs;
381 TemporalProfTraceTy(std::initializer_list<uint64_t> Trace = {},
382 uint64_t Weight = 1)
384
385 /// Use a set of temporal profile traces to create a list of balanced
386 /// partitioning function nodes used by BalancedPartitioning to generate a
387 /// function order that reduces page faults during startup
388 static void createBPFunctionNodes(ArrayRef<TemporalProfTraceTy> Traces,
389 std::vector<BPFunctionNode> &Nodes,
390 bool RemoveOutlierUNs = true);
391};
392
393inline std::error_code make_error_code(instrprof_error E) {
394 return std::error_code(static_cast<int>(E), instrprof_category());
395}
396
397class InstrProfError : public ErrorInfo<InstrProfError> {
398public:
400 : Err(Err), Msg(ErrStr.str()) {
401 assert(Err != instrprof_error::success && "Not an error");
402 }
403
404 std::string message() const override;
405
406 void log(raw_ostream &OS) const override { OS << message(); }
407
408 std::error_code convertToErrorCode() const override {
409 return make_error_code(Err);
410 }
411
412 instrprof_error get() const { return Err; }
413 const std::string &getMessage() const { return Msg; }
414
415 /// Consume an Error and return the raw enum value contained within it, and
416 /// the optional error message. The Error must either be a success value, or
417 /// contain a single InstrProfError.
418 static std::pair<instrprof_error, std::string> take(Error E) {
419 auto Err = instrprof_error::success;
420 std::string Msg = "";
421 handleAllErrors(std::move(E), [&Err, &Msg](const InstrProfError &IPE) {
422 assert(Err == instrprof_error::success && "Multiple errors encountered");
423 Err = IPE.get();
424 Msg = IPE.getMessage();
425 });
426 return {Err, Msg};
427 }
428
429 static char ID;
430
431private:
432 instrprof_error Err;
433 std::string Msg;
434};
435
436namespace object {
437
438class SectionRef;
439
440} // end namespace object
441
442namespace IndexedInstrProf {
443
445
446} // end namespace IndexedInstrProf
447
448/// A symbol table used for function [IR]PGO name look-up with keys
449/// (such as pointers, md5hash values) to the function. A function's
450/// [IR]PGO name or name's md5hash are used in retrieving the profile
451/// data of the function. See \c getIRPGOFuncName() and \c getPGOFuncName
452/// methods for details how [IR]PGO name is formed.
454public:
455 using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
456
457private:
458 using AddrIntervalMap =
460 StringRef Data;
461 uint64_t Address = 0;
462 // Unique name strings. Used to ensure entries in MD5NameMap (a vector that's
463 // going to be sorted) has unique MD5 keys in the first place.
464 StringSet<> NameTab;
465 // Records the unique virtual table names. This is used by InstrProfWriter to
466 // write out an on-disk chained hash table of virtual table names.
467 // InstrProfWriter stores per function profile data (keyed by function names)
468 // so it doesn't use a StringSet for function names.
469 StringSet<> VTableNames;
470 // A map from MD5 keys to function name strings.
471 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
472 // A map from MD5 keys to function define. We only populate this map
473 // when build the Symtab from a Module.
474 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
475 // A map from MD5 to the global variable. This map is only populated when
476 // building the symtab from a module. Use separate container instances for
477 // `MD5FuncMap` and `MD5VTableMap`.
478 // TODO: Unify the container type and the lambda function 'mapName' inside
479 // add{Func,VTable}WithName.
481 // A map from function runtime address to function name MD5 hash.
482 // This map is only populated and used by raw instr profile reader.
483 AddrHashMap AddrToMD5Map;
484
485 AddrIntervalMap::Allocator VTableAddrMapAllocator;
486 // This map is only populated and used by raw instr profile reader.
487 AddrIntervalMap VTableAddrMap;
488 bool Sorted = false;
489
490 static StringRef getExternalSymbol() { return "** External Symbol **"; }
491
492 // Returns the canonial name of the given PGOName. In a canonical name, all
493 // suffixes that begins with "." except ".__uniq." are stripped.
494 // FIXME: Unify this with `FunctionSamples::getCanonicalFnName`.
495 static StringRef getCanonicalName(StringRef PGOName);
496
497 // Add the function into the symbol table, by creating the following
498 // map entries:
499 // name-set = {PGOFuncName} union {getCanonicalName(PGOFuncName)}
500 // - In MD5NameMap: <MD5Hash(name), name> for name in name-set
501 // - In MD5FuncMap: <MD5Hash(name), &F> for name in name-set
502 Error addFuncWithName(Function &F, StringRef PGOFuncName);
503
504 // Add the vtable into the symbol table, by creating the following
505 // map entries:
506 // name-set = {PGOName} union {getCanonicalName(PGOName)}
507 // - In MD5NameMap: <MD5Hash(name), name> for name in name-set
508 // - In MD5VTableMap: <MD5Hash(name), name> for name in name-set
509 Error addVTableWithName(GlobalVariable &V, StringRef PGOVTableName);
510
511 // If the symtab is created by a series of calls to \c addFuncName, \c
512 // finalizeSymtab needs to be called before looking up function names.
513 // This is required because the underlying map is a vector (for space
514 // efficiency) which needs to be sorted.
515 inline void finalizeSymtab();
516
517public:
518 InstrProfSymtab() : VTableAddrMap(VTableAddrMapAllocator) {}
519
520 // Not copyable or movable.
521 // Consider std::unique_ptr for move.
526
527 /// Create InstrProfSymtab from an object file section which
528 /// contains function PGO names. When section may contain raw
529 /// string data or string data in compressed form. This method
530 /// only initialize the symtab with reference to the data and
531 /// the section base address. The decompression will be delayed
532 /// until before it is used. See also \c create(StringRef) method.
534
535 /// \c NameStrings is a string composed of one of more sub-strings
536 /// encoded in the format described in \c collectPGOFuncNameStrings.
537 /// This method is a wrapper to \c readAndDecodeStrings method.
538 Error create(StringRef NameStrings);
539
540 /// Initialize symtab states with function names and vtable names. \c
541 /// FuncNameStrings is a string composed of one or more encoded function name
542 /// strings, and \c VTableNameStrings composes of one or more encoded vtable
543 /// names. This interface is solely used by raw profile reader.
544 Error create(StringRef FuncNameStrings, StringRef VTableNameStrings);
545
546 /// Initialize 'this' with the set of vtable names encoded in
547 /// \c CompressedVTableNames.
549
550 /// This interface is used by reader of CoverageMapping test
551 /// format.
552 inline Error create(StringRef D, uint64_t BaseAddr);
553
554 /// A wrapper interface to populate the PGO symtab with functions
555 /// decls from module \c M. This interface is used by transformation
556 /// passes such as indirect function call promotion. Variable \c InLTO
557 /// indicates if this is called from LTO optimization passes.
558 Error create(Module &M, bool InLTO = false);
559
560 /// Create InstrProfSymtab from a set of names iteratable from
561 /// \p IterRange. This interface is used by IndexedProfReader.
562 template <typename NameIterRange>
563 Error create(const NameIterRange &IterRange);
564
565 /// Create InstrProfSymtab from a set of function names and vtable
566 /// names iteratable from \p IterRange. This interface is used by
567 /// IndexedProfReader.
568 template <typename FuncNameIterRange, typename VTableNameIterRange>
569 Error create(const FuncNameIterRange &FuncIterRange,
570 const VTableNameIterRange &VTableIterRange);
571
572 // Map the MD5 of the symbol name to the name.
574 if (SymbolName.empty())
575 return make_error<InstrProfError>(instrprof_error::malformed,
576 "symbol name is empty");
577
578 // Insert into NameTab so that MD5NameMap (a vector that will be sorted)
579 // won't have duplicated entries in the first place.
580 auto Ins = NameTab.insert(SymbolName);
581 if (Ins.second) {
582 MD5NameMap.push_back(std::make_pair(
583 IndexedInstrProf::ComputeHash(SymbolName), Ins.first->getKey()));
584 Sorted = false;
585 }
586 return Error::success();
587 }
588
589 /// The method name is kept since there are many callers.
590 /// It just forwards to 'addSymbolName'.
591 Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); }
592
593 /// Adds VTableName as a known symbol, and inserts it to a map that
594 /// tracks all vtable names.
596 if (Error E = addSymbolName(VTableName))
597 return E;
598
599 // Record VTableName. InstrProfWriter uses this set. The comment around
600 // class member explains why.
601 VTableNames.insert(VTableName);
602 return Error::success();
603 }
604
605 const StringSet<> &getVTableNames() const { return VTableNames; }
606
607 /// Map a function address to its name's MD5 hash. This interface
608 /// is only used by the raw profiler reader.
610 AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
611 }
612
613 /// Map the address range (i.e., [start_address, end_address)) of a variable
614 /// to its names' MD5 hash. This interface is only used by the raw profile
615 /// reader.
616 void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) {
617 VTableAddrMap.insert(StartAddr, EndAddr, MD5Val);
618 }
619
620 /// Return a function's hash, or 0, if the function isn't in this SymTab.
622
623 /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
625
626 /// Return function's PGO name from the function name's symbol
627 /// address in the object file. If an error occurs, return
628 /// an empty string.
629 StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
630
631 /// Return name of functions or global variables from the name's md5 hash
632 /// value. If not found, return an empty string.
633 inline StringRef getFuncOrVarName(uint64_t ValMD5Hash);
634
635 /// Just like getFuncOrVarName, except that it will return literal string
636 /// 'External Symbol' if the function or global variable is external to
637 /// this symbol table.
639
640 /// True if Symbol is the value used to represent external symbols.
641 static bool isExternalSymbol(const StringRef &Symbol) {
642 return Symbol == InstrProfSymtab::getExternalSymbol();
643 }
644
645 /// Return function from the name's md5 hash. Return nullptr if not found.
646 inline Function *getFunction(uint64_t FuncMD5Hash);
647
648 /// Return the global variable corresponding to md5 hash. Return nullptr if
649 /// not found.
651
652 /// Return the name section data.
653 inline StringRef getNameData() const { return Data; }
654
655 /// Dump the symbols in this table.
656 void dumpNames(raw_ostream &OS) const;
657};
658
660 Data = D;
661 Address = BaseAddr;
662 return Error::success();
663}
664
665template <typename NameIterRange>
666Error InstrProfSymtab::create(const NameIterRange &IterRange) {
667 for (auto Name : IterRange)
668 if (Error E = addFuncName(Name))
669 return E;
670
671 finalizeSymtab();
672 return Error::success();
673}
674
675template <typename FuncNameIterRange, typename VTableNameIterRange>
676Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange,
677 const VTableNameIterRange &VTableIterRange) {
678 // Iterate elements by StringRef rather than by const reference.
679 // StringRef is small enough, so the loop is efficient whether
680 // element in the range is std::string or StringRef.
681 for (StringRef Name : FuncIterRange)
682 if (Error E = addFuncName(Name))
683 return E;
684
685 for (StringRef VTableName : VTableIterRange)
686 if (Error E = addVTableName(VTableName))
687 return E;
688
689 finalizeSymtab();
690 return Error::success();
691}
692
693void InstrProfSymtab::finalizeSymtab() {
694 if (Sorted)
695 return;
696 llvm::sort(MD5NameMap, less_first());
697 llvm::sort(MD5FuncMap, less_first());
698 llvm::sort(AddrToMD5Map, less_first());
699 AddrToMD5Map.erase(llvm::unique(AddrToMD5Map), AddrToMD5Map.end());
700 Sorted = true;
701}
702
705 if (ret.empty())
706 return InstrProfSymtab::getExternalSymbol();
707 return ret;
708}
709
711 finalizeSymtab();
712 auto Result = llvm::lower_bound(MD5NameMap, MD5Hash,
713 [](const std::pair<uint64_t, StringRef> &LHS,
714 uint64_t RHS) { return LHS.first < RHS; });
715 if (Result != MD5NameMap.end() && Result->first == MD5Hash)
716 return Result->second;
717 return StringRef();
718}
719
721 finalizeSymtab();
722 auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash,
723 [](const std::pair<uint64_t, Function *> &LHS,
724 uint64_t RHS) { return LHS.first < RHS; });
725 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
726 return Result->second;
727 return nullptr;
728}
729
731 return MD5VTableMap.lookup(MD5Hash);
732}
733
734// To store the sums of profile count values, or the percentage of
735// the sums of the total count values.
738 double CountSum = 0.0f;
739 std::array<double, IPVK_Last - IPVK_First + 1> ValueCounts = {};
740 CountSumOrPercent() = default;
741 void reset() {
742 NumEntries = 0;
743 CountSum = 0.0f;
744 ValueCounts.fill(0.0f);
745 }
746};
747
748// Function level or program level overlap information.
751 // Sum of the total count values for the base profile.
753 // Sum of the total count values for the test profile.
755 // Overlap lap score. Should be in range of [0.0f to 1.0f].
760 const std::string *BaseFilename = nullptr;
761 const std::string *TestFilename = nullptr;
764 bool Valid = false;
765
767
768 void dump(raw_fd_ostream &OS) const;
769
771 FuncName = Name;
772 FuncHash = Hash;
773 }
774
775 Error accumulateCounts(const std::string &BaseFilename,
776 const std::string &TestFilename, bool IsCS);
777 void addOneMismatch(const CountSumOrPercent &MismatchFunc);
778 void addOneUnique(const CountSumOrPercent &UniqueFunc);
779
780 static inline double score(uint64_t Val1, uint64_t Val2, double Sum1,
781 double Sum2) {
782 if (Sum1 < 1.0f || Sum2 < 1.0f)
783 return 0.0f;
784 return std::min(Val1 / Sum1, Val2 / Sum2);
785 }
786};
787
788// This is used to filter the functions whose overlap information
789// to be output.
792 const std::string NameFilter;
793};
794
796 /// Value profiling data pairs at a given value site.
797 std::vector<InstrProfValueData> ValueData;
798
800 template <class InputIterator>
801 InstrProfValueSiteRecord(InputIterator F, InputIterator L)
802 : ValueData(F, L) {}
803
804 /// Sort ValueData ascending by Value
807 [](const InstrProfValueData &L, const InstrProfValueData &R) {
808 return L.Value < R.Value;
809 });
810 }
811 /// Sort ValueData Descending by Count
812 inline void sortByCount();
813
814 /// Merge data from another InstrProfValueSiteRecord
815 /// Optionally scale merged counts by \p Weight.
816 void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
817 function_ref<void(instrprof_error)> Warn);
818 /// Scale up value profile data counts by N (Numerator) / D (Denominator).
820
821 /// Compute the overlap b/w this record and Input record.
822 void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind,
823 OverlapStats &Overlap, OverlapStats &FuncLevelOverlap);
824};
825
826/// Profiling information for a single function.
828 std::vector<uint64_t> Counts;
829 std::vector<uint8_t> BitmapBytes;
830
831 InstrProfRecord() = default;
832 InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {}
833 InstrProfRecord(std::vector<uint64_t> Counts,
834 std::vector<uint8_t> BitmapBytes)
839 ValueData(RHS.ValueData
840 ? std::make_unique<ValueProfData>(*RHS.ValueData)
841 : nullptr) {}
844 Counts = RHS.Counts;
845 BitmapBytes = RHS.BitmapBytes;
846 if (!RHS.ValueData) {
847 ValueData = nullptr;
848 return *this;
849 }
850 if (!ValueData)
851 ValueData = std::make_unique<ValueProfData>(*RHS.ValueData);
852 else
853 *ValueData = *RHS.ValueData;
854 return *this;
855 }
856
857 /// Return the number of value profile kinds with non-zero number
858 /// of profile sites.
859 inline uint32_t getNumValueKinds() const;
860 /// Return the number of instrumented sites for ValueKind.
861 inline uint32_t getNumValueSites(uint32_t ValueKind) const;
862
863 /// Return the total number of ValueData for ValueKind.
864 inline uint32_t getNumValueData(uint32_t ValueKind) const;
865
866 /// Return the array of profiled values at \p Site.
868 uint32_t Site) const;
869
870 /// Return the number of value data collected for ValueKind at profiling
871 /// site: Site.
873 uint32_t Site) const;
874
875 /// Return the array of profiled values at \p Site.
876 inline std::unique_ptr<InstrProfValueData[]>
877 getValueForSite(uint32_t ValueKind, uint32_t Site) const;
878
879 /// Get the target value/counts of kind \p ValueKind collected at site
880 /// \p Site and store the result in array \p Dest.
881 inline void getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind,
882 uint32_t Site) const;
883
884 /// Reserve space for NumValueSites sites.
885 inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites);
886
887 /// Add ValueData for ValueKind at value Site.
888 void addValueData(uint32_t ValueKind, uint32_t Site,
889 InstrProfValueData *VData, uint32_t N,
890 InstrProfSymtab *SymTab);
891
892 /// Merge the counts in \p Other into this one.
893 /// Optionally scale merged counts by \p Weight.
894 void merge(InstrProfRecord &Other, uint64_t Weight,
895 function_ref<void(instrprof_error)> Warn);
896
897 /// Scale up profile counts (including value profile data) by
898 /// a factor of (N / D).
900
901 /// Sort value profile data (per site) by count.
903 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
904 for (auto &SR : getValueSitesForKind(Kind))
905 SR.sortByCount();
906 }
907
908 /// Clear value data entries and edge counters.
909 void Clear() {
910 Counts.clear();
912 }
913
914 /// Clear value data entries
915 void clearValueData() { ValueData = nullptr; }
916
917 /// Compute the sums of all counts and store in Sum.
918 void accumulateCounts(CountSumOrPercent &Sum) const;
919
920 /// Compute the overlap b/w this IntrprofRecord and Other.
922 OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff);
923
924 /// Compute the overlap of value profile counts.
925 void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src,
926 OverlapStats &Overlap,
927 OverlapStats &FuncLevelOverlap);
928
933 };
937 };
939 uint64_t FirstCount = Counts[0];
940 if (FirstCount == (uint64_t)HotFunctionVal)
941 return PseudoHot;
942 if (FirstCount == (uint64_t)WarmFunctionVal)
943 return PseudoWarm;
944 return NotPseudo;
945 }
947 if (Kind == PseudoHot)
949 else if (Kind == PseudoWarm)
951 }
952
953private:
954 struct ValueProfData {
955 std::vector<InstrProfValueSiteRecord> IndirectCallSites;
956 std::vector<InstrProfValueSiteRecord> MemOPSizes;
957 std::vector<InstrProfValueSiteRecord> VTableTargets;
958 };
959 std::unique_ptr<ValueProfData> ValueData;
960
961 MutableArrayRef<InstrProfValueSiteRecord>
962 getValueSitesForKind(uint32_t ValueKind) {
963 // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever
964 // implemented in LLVM) to call the const overload of this function, then
965 // cast away the constness from the result.
966 auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
967 ValueKind);
968 return MutableArrayRef(
969 const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size());
970 }
971 ArrayRef<InstrProfValueSiteRecord>
972 getValueSitesForKind(uint32_t ValueKind) const {
973 if (!ValueData)
974 return std::nullopt;
975 switch (ValueKind) {
976 case IPVK_IndirectCallTarget:
977 return ValueData->IndirectCallSites;
978 case IPVK_MemOPSize:
979 return ValueData->MemOPSizes;
980 case IPVK_VTableTarget:
981 return ValueData->VTableTargets;
982 default:
983 llvm_unreachable("Unknown value kind!");
984 }
985 }
986
987 std::vector<InstrProfValueSiteRecord> &
988 getOrCreateValueSitesForKind(uint32_t ValueKind) {
989 if (!ValueData)
990 ValueData = std::make_unique<ValueProfData>();
991 switch (ValueKind) {
992 case IPVK_IndirectCallTarget:
993 return ValueData->IndirectCallSites;
994 case IPVK_MemOPSize:
995 return ValueData->MemOPSizes;
996 case IPVK_VTableTarget:
997 return ValueData->VTableTargets;
998 default:
999 llvm_unreachable("Unknown value kind!");
1000 }
1001 }
1002
1003 // Map indirect call target name hash to name string.
1004 uint64_t remapValue(uint64_t Value, uint32_t ValueKind,
1005 InstrProfSymtab *SymTab);
1006
1007 // Merge Value Profile data from Src record to this record for ValueKind.
1008 // Scale merged value counts by \p Weight.
1009 void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src,
1010 uint64_t Weight,
1011 function_ref<void(instrprof_error)> Warn);
1012
1013 // Scale up value profile data count by N (Numerator) / D (Denominator).
1014 void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D,
1015 function_ref<void(instrprof_error)> Warn);
1016};
1017
1021
1022 // We reserve this bit as the flag for context sensitive profile record.
1023 static const int CS_FLAG_IN_FUNC_HASH = 60;
1024
1027 std::vector<uint64_t> Counts)
1030 std::vector<uint64_t> Counts,
1031 std::vector<uint8_t> BitmapBytes)
1033 Hash(Hash) {}
1034
1035 static bool hasCSFlagInHash(uint64_t FuncHash) {
1036 return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1);
1037 }
1038 static void setCSFlagInHash(uint64_t &FuncHash) {
1039 FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH);
1040 }
1041};
1042
1044 uint32_t NumValueKinds = 0;
1045 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1046 NumValueKinds += !(getValueSitesForKind(Kind).empty());
1047 return NumValueKinds;
1048}
1049
1051 uint32_t N = 0;
1052 for (const auto &SR : getValueSitesForKind(ValueKind))
1053 N += SR.ValueData.size();
1054 return N;
1055}
1056
1058 return getValueSitesForKind(ValueKind).size();
1059}
1060
1062 uint32_t Site) const {
1063 return getValueSitesForKind(ValueKind)[Site].ValueData.size();
1064}
1065
1068 return getValueSitesForKind(ValueKind)[Site].ValueData;
1069}
1070
1071std::unique_ptr<InstrProfValueData[]>
1073 uint32_t N = getNumValueDataForSite(ValueKind, Site);
1074 if (N == 0)
1075 return std::unique_ptr<InstrProfValueData[]>(nullptr);
1076
1077 auto VD = std::make_unique<InstrProfValueData[]>(N);
1078 getValueForSite(VD.get(), ValueKind, Site);
1079
1080 return VD;
1081}
1082
1083void InstrProfRecord::getValueForSite(InstrProfValueData Dest[],
1084 uint32_t ValueKind, uint32_t Site) const {
1085 uint32_t I = 0;
1086 for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
1087 Dest[I].Value = V.Value;
1088 Dest[I].Count = V.Count;
1089 I++;
1090 }
1091}
1092
1093void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) {
1094 if (!NumValueSites)
1095 return;
1096 getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
1097}
1098
1099// Include definitions for value profile data
1100#define INSTR_PROF_VALUE_PROF_DATA
1102
1105 ValueData, [](const InstrProfValueData &L, const InstrProfValueData &R) {
1106 return L.Count > R.Count;
1107 });
1108 // Now truncate
1109 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
1110 if (ValueData.size() > max_s)
1111 ValueData.resize(max_s);
1112}
1113
1114namespace IndexedInstrProf {
1115
1116enum class HashT : uint32_t {
1117 MD5,
1118 Last = MD5
1119};
1120
1122 switch (Type) {
1123 case HashT::MD5:
1124 return MD5Hash(K);
1125 }
1126 llvm_unreachable("Unhandled hash type");
1127}
1128
1129const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81"
1130
1132 // Version 1 is the first version. In this version, the value of
1133 // a key/value pair can only include profile data of a single function.
1134 // Due to this restriction, the number of block counters for a given
1135 // function is not recorded but derived from the length of the value.
1137 // The version 2 format supports recording profile data of multiple
1138 // functions which share the same key in one value field. To support this,
1139 // the number block counters is recorded as an uint64_t field right after the
1140 // function structural hash.
1142 // Version 3 supports value profile data. The value profile data is expected
1143 // to follow the block counter profile data.
1145 // In this version, profile summary data \c IndexedInstrProf::Summary is
1146 // stored after the profile header.
1148 // In this version, the frontend PGO stable hash algorithm defaults to V2.
1150 // In this version, the frontend PGO stable hash algorithm got fixed and
1151 // may produce hashes different from Version5.
1153 // An additional counter is added around logical operators.
1155 // An additional (optional) memory profile type is added.
1157 // Binary ids are added.
1159 // An additional (optional) temporal profile traces section is added.
1161 // An additional field is used for bitmap bytes.
1163 // VTable profiling, decision record and bitmap are modified for mcdc.
1165 // The current version is 12.
1166 CurrentVersion = INSTR_PROF_INDEX_VERSION
1169
1171
1173
1174// This structure defines the file header of the LLVM profile
1175// data file in indexed-format. Please update llvm/docs/InstrProfileFormat.rst
1176// as appropriate when updating the indexed profile format.
1177struct Header {
1179 // The lower 32 bits specify the version of the indexed profile.
1180 // The most significant 32 bits are reserved to specify the variant types of
1181 // the profile.
1183 uint64_t Unused = 0; // Becomes unused since version 4
1185 // This field records the offset of this hash table's metadata (i.e., the
1186 // number of buckets and entries), which follows right after the payload of
1187 // the entire hash table.
1193 // New fields should only be added at the end to ensure that the size
1194 // computation is correct. The methods below need to be updated to ensure that
1195 // the new field is read correctly.
1196
1197 // Reads a header struct from the buffer. Header fields are in machine native
1198 // endianness.
1199 static Expected<Header> readFromBuffer(const unsigned char *Buffer);
1200
1201 // Returns the size of the header in bytes for all valid fields based on the
1202 // version. I.e a older version header will return a smaller size.
1203 size_t size() const;
1204
1205 // Return the indexed profile version, i.e., the least significant 32 bits
1206 // in Header.Version.
1208};
1209
1210// Profile summary data recorded in the profile data file in indexed
1211// format. It is introduced in version 4. The summary data follows
1212// right after the profile file header.
1213struct Summary {
1214 struct Entry {
1215 uint64_t Cutoff; ///< The required percentile of total execution count.
1216 uint64_t
1217 MinBlockCount; ///< The minimum execution count for this percentile.
1218 uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count.
1219 };
1220 // The field kind enumerator to assigned value mapping should remain
1221 // unchanged when a new kind is added or an old kind gets deleted in
1222 // the future.
1224 /// The total number of functions instrumented.
1226 /// Total number of instrumented blocks/edges.
1228 /// The maximal execution count among all functions.
1229 /// This field does not exist for profile data from IR based
1230 /// instrumentation.
1232 /// Max block count of the program.
1234 /// Max internal block count of the program (excluding entry blocks).
1236 /// The sum of all instrumented block counts.
1240
1241 // The number of summmary fields following the summary header.
1243 // The number of Cutoff Entries (Summary::Entry) following summary fields.
1245
1246 Summary() = delete;
1247 Summary(uint32_t Size) { memset(this, 0, Size); }
1248
1249 void operator delete(void *ptr) { ::operator delete(ptr); }
1250
1252 return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) +
1253 NumSumFields * sizeof(uint64_t);
1254 }
1255
1257 return reinterpret_cast<const uint64_t *>(this + 1);
1258 }
1259
1261 return reinterpret_cast<uint64_t *>(this + 1);
1262 }
1263
1264 const Entry *getCutoffEntryBase() const {
1265 return reinterpret_cast<const Entry *>(
1267 }
1268
1270 return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]);
1271 }
1272
1274 return getSummaryDataBase()[K];
1275 }
1276
1278 getSummaryDataBase()[K] = V;
1279 }
1280
1281 const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; }
1282
1284 Entry &ER = getCutoffEntryBase()[I];
1285 ER.Cutoff = E.Cutoff;
1286 ER.MinBlockCount = E.MinCount;
1287 ER.NumBlocks = E.NumCounts;
1288 }
1289};
1290
1291inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) {
1292 return std::unique_ptr<Summary>(new (::operator new(TotalSize))
1293 Summary(TotalSize));
1294}
1295
1296} // end namespace IndexedInstrProf
1297
1298namespace RawInstrProf {
1299
1300// Version 1: First version
1301// Version 2: Added value profile data section. Per-function control data
1302// struct has more fields to describe value profile information.
1303// Version 3: Compressed name section support. Function PGO name reference
1304// from control data struct is changed from raw pointer to Name's MD5 value.
1305// Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
1306// raw header.
1307// Version 5: Bit 60 of FuncHash is reserved for the flag for the context
1308// sensitive records.
1309// Version 6: Added binary id.
1310// Version 7: Reorder binary id and include version in signature.
1311// Version 8: Use relative counter pointer.
1312// Version 9: Added relative bitmap bytes pointer and count used by MC/DC.
1313// Version 10: Added vtable, a new type of value profile data.
1314const uint64_t Version = INSTR_PROF_RAW_VERSION;
1315
1316template <class IntPtrT> inline uint64_t getMagic();
1317template <> inline uint64_t getMagic<uint64_t>() {
1318 return INSTR_PROF_RAW_MAGIC_64;
1319}
1320
1321template <> inline uint64_t getMagic<uint32_t>() {
1322 return INSTR_PROF_RAW_MAGIC_32;
1323}
1324
1325// Per-function profile data header/control structure.
1326// The definition should match the structure defined in
1327// compiler-rt/lib/profile/InstrProfiling.h.
1328// It should also match the synthesized type in
1329// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
1330template <class IntPtrT> struct alignas(8) ProfileData {
1331#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
1333};
1334
1335template <class IntPtrT> struct alignas(8) VTableProfileData {
1336#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name;
1338};
1339
1340// File header structure of the LLVM profile data in raw format.
1341// The definition should match the header referenced in
1342// compiler-rt/lib/profile/InstrProfilingFile.c and
1343// InstrProfilingBuffer.c.
1344struct Header {
1345#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
1347};
1348
1349} // end namespace RawInstrProf
1350
1351// Create the variable for the profile file name.
1352void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
1353
1354// Whether to compress function names in profile records, and filenames in
1355// code coverage mappings. Used by the Instrumentation library and unit tests.
1356extern cl::opt<bool> DoInstrProfNameCompression;
1357
1358} // end namespace llvm
1359#endif // LLVM_PROFILEDATA_INSTRPROF_H
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file defines the DenseMap class.
uint64_t Addr
std::string Name
uint64_t Size
This file implements a coalescing interval map for small objects.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Machine Check Debug Module
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
StringSet - A set-like wrapper for the StringMap.
Value * RHS
Value * LHS
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
Base class for user error types.
Definition: Error.h:355
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:337
Tagged union holding either a T or a Error.
Definition: Error.h:481
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:50
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: InstrProf.h:406
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
Definition: InstrProf.h:418
const std::string & getMessage() const
Definition: InstrProf.h:413
static char ID
Definition: InstrProf.h:429
instrprof_error get() const
Definition: InstrProf.h:412
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: InstrProf.h:408
std::string message() const override
Return the error message as a string.
Definition: InstrProf.cpp:250
InstrProfError(instrprof_error Err, const Twine &ErrStr=Twine())
Definition: InstrProf.h:399
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
Definition: InstrProf.h:453
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
Definition: InstrProf.h:641
uint64_t getFunctionHashFromAddress(uint64_t Address)
Return a function's hash, or 0, if the function isn't in this SymTab.
Definition: InstrProf.cpp:637
Error addSymbolName(StringRef SymbolName)
Definition: InstrProf.h:573
std::vector< std::pair< uint64_t, uint64_t > > AddrHashMap
Definition: InstrProf.h:455
GlobalVariable * getGlobalVariable(uint64_t MD5Hash)
Return the global variable corresponding to md5 hash.
Definition: InstrProf.h:730
uint64_t getVTableHashFromAddress(uint64_t Address)
Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
Definition: InstrProf.cpp:631
StringRef getFuncOrVarName(uint64_t ValMD5Hash)
Return name of functions or global variables from the name's md5 hash value.
Definition: InstrProf.h:710
InstrProfSymtab & operator=(InstrProfSymtab &&)=delete
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Definition: InstrProf.h:609
Error addVTableName(StringRef VTableName)
Adds VTableName as a known symbol, and inserts it to a map that tracks all vtable names.
Definition: InstrProf.h:595
void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
Definition: InstrProf.cpp:651
Function * getFunction(uint64_t FuncMD5Hash)
Return function from the name's md5 hash. Return nullptr if not found.
Definition: InstrProf.h:720
StringRef getNameData() const
Return the name section data.
Definition: InstrProf.h:653
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
The method name is kept since there are many callers.
Definition: InstrProf.h:591
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val)
Map the address range (i.e., [start_address, end_address)) of a variable to its names' MD5 hash.
Definition: InstrProf.h:616
Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames)
Initialize 'this' with the set of vtable names encoded in CompressedVTableNames.
Definition: InstrProf.cpp:581
const StringSet & getVTableNames() const
Definition: InstrProf.h:605
StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize)
Return function's PGO name from the function name's symbol address in the object file.
InstrProfSymtab & operator=(const InstrProfSymtab &)=delete
InstrProfSymtab(InstrProfSymtab &&)=delete
InstrProfSymtab(const InstrProfSymtab &)=delete
StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash)
Just like getFuncOrVarName, except that it will return literal string 'External Symbol' if the functi...
Definition: InstrProf.h:703
void insert(KeyT a, KeyT b, ValT y)
insert - Add a mapping of [a;b] to y, coalesce with adjacent intervals.
Definition: IntervalMap.h:1129
typename Sizer::Allocator Allocator
Definition: IntervalMap.h:962
Definition: MD5.h:41
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
StringSet - A wrapper for StringMap that provides set-like functionality.
Definition: StringSet.h:23
std::pair< typename Base::iterator, bool > insert(StringRef key)
Definition: StringSet.h:38
ObjectFormatType
Definition: Triple.h:297
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
An efficient, type-erasing, non-owning reference to a callable.
This is a value type class that represents a single section in the list of sections in the object fil...
Definition: ObjectFile.h:81
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:471
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1291
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1172
const uint64_t Version
Definition: InstrProf.h:1168
const uint64_t Magic
Definition: InstrProf.h:1129
const HashT HashType
Definition: InstrProf.h:1170
uint64_t getMagic()
const uint64_t Version
Definition: InstrProf.h:1314
uint64_t getMagic< uint32_t >()
Definition: InstrProf.h:1321
uint64_t getMagic< uint64_t >()
Definition: InstrProf.h:1317
constexpr size_t NameSize
Definition: XCOFF.h:29
uint64_t MD5Hash(const FunctionId &Obj)
Definition: FunctionId.h:167
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void stable_sort(R &&Range)
Definition: STLExtras.h:1995
bool getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, InstrProfValueData ValueData[], uint32_t &ActualNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst which is annotated with value profile meta data.
Definition: InstrProf.cpp:1386
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:92
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
Definition: InstrProf.cpp:374
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:163
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
Definition: InstrProf.cpp:1404
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
Definition: InstrProf.cpp:363
std::error_code make_error_code(BitcodeError E)
StringRef getPGOFuncNameMetadataName()
Definition: InstrProf.h:307
StringRef getCoverageMappingVarName()
Return the name of a covarage mapping variable (internal linkage) for each instrumented source module...
Definition: InstrProf.h:121
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
Definition: Error.h:977
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
Definition: InstrProf.h:104
cl::opt< bool > DoInstrProfNameCompression
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="<unknown>")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
Definition: InstrProf.cpp:405
StringRef getInstrProfVTableNamesVarName()
Definition: InstrProf.h:116
std::pair< StringRef, StringRef > getParsedIRPGOName(StringRef IRPGOName)
Definition: InstrProf.cpp:398
auto unique(Range &&R, Predicate P)
Definition: STLExtras.h:2013
MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
Definition: InstrProf.cpp:1400
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:98
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:129
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:231
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
Definition: InstrProf.h:66
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1416
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
Definition: InstrProf.cpp:390
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:158
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:107
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:173
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
Definition: InstrProf.cpp:462
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
Definition: InstrProf.cpp:1272
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:169
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:138
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
Definition: InstrProf.cpp:703
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1647
InstrProfSectKind
Definition: InstrProf.h:60
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:101
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:696
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
Definition: InstrProf.h:178
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:87
@ Other
Any other memory.
instrprof_error
Definition: InstrProf.h:347
InstrProfValueKind
Definition: InstrProf.h:267
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:150
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1954
const std::error_category & instrprof_category()
Definition: InstrProf.cpp:192
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
Definition: InstrProf.cpp:713
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1464
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1487
Error collectGlobalObjectNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (names of global objects like functions or, virtual tables) NameStrs,...
Definition: InstrProf.cpp:658
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1849
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:81
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:144
std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
Definition: InstrProf.cpp:417
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:114
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1442
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:110
StringRef getInstrProfVTableVarPrefix()
Return the name prefix of variables containing virtual table profile data.
Definition: InstrProf.h:95
InstrProfKind
An enum describing the attributes of an instrumented profile.
Definition: InstrProf.h:324
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
#define N
std::array< double, IPVK_Last - IPVK_First+1 > ValueCounts
Definition: InstrProf.h:739
uint64_t getIndexedProfileVersion() const
Definition: InstrProf.cpp:1664
static Expected< Header > readFromBuffer(const unsigned char *Buffer)
Definition: InstrProf.cpp:1624
uint64_t Cutoff
The required percentile of total execution count.
Definition: InstrProf.h:1215
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
Definition: InstrProf.h:1218
uint64_t MinBlockCount
The minimum execution count for this percentile.
Definition: InstrProf.h:1217
const Entry * getCutoffEntryBase() const
Definition: InstrProf.h:1264
uint64_t get(SummaryFieldKind K) const
Definition: InstrProf.h:1273
void set(SummaryFieldKind K, uint64_t V)
Definition: InstrProf.h:1277
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
Definition: InstrProf.h:1283
@ TotalNumFunctions
The total number of functions instrumented.
Definition: InstrProf.h:1225
@ TotalNumBlocks
Total number of instrumented blocks/edges.
Definition: InstrProf.h:1227
@ MaxFunctionCount
The maximal execution count among all functions.
Definition: InstrProf.h:1231
@ TotalBlockCount
The sum of all instrumented block counts.
Definition: InstrProf.h:1237
@ MaxBlockCount
Max block count of the program.
Definition: InstrProf.h:1233
@ MaxInternalBlockCount
Max internal block count of the program (excluding entry blocks).
Definition: InstrProf.h:1235
const uint64_t * getSummaryDataBase() const
Definition: InstrProf.h:1256
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
Definition: InstrProf.h:1251
const Entry & getEntry(uint32_t I) const
Definition: InstrProf.h:1281
Profiling information for a single function.
Definition: InstrProf.h:827
void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
Definition: InstrProf.cpp:772
std::vector< uint64_t > Counts
Definition: InstrProf.h:828
std::unique_ptr< InstrProfValueData[]> getValueForSite(uint32_t ValueKind, uint32_t Site) const
Return the array of profiled values at Site.
Definition: InstrProf.h:1072
ArrayRef< InstrProfValueData > getValueArrayForSite(uint32_t ValueKind, uint32_t Site) const
Return the array of profiled values at Site.
Definition: InstrProf.h:1067
CountPseudoKind getCountPseudoKind() const
Definition: InstrProf.h:938
InstrProfRecord(std::vector< uint64_t > Counts)
Definition: InstrProf.h:832
void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
Definition: InstrProf.cpp:723
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
Definition: InstrProf.h:1057
uint32_t getNumValueKinds() const
Return the number of value profile kinds with non-zero number of profile sites.
Definition: InstrProf.h:1043
void setPseudoCount(CountPseudoKind Kind)
Definition: InstrProf.h:946
InstrProfRecord(InstrProfRecord &&)=default
uint32_t getNumValueData(uint32_t ValueKind) const
Return the total number of ValueData for ValueKind.
Definition: InstrProf.h:1050
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:901
void addValueData(uint32_t ValueKind, uint32_t Site, InstrProfValueData *VData, uint32_t N, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
Definition: InstrProf.cpp:996
InstrProfRecord & operator=(const InstrProfRecord &RHS)
Definition: InstrProf.h:843
void clearValueData()
Clear value data entries.
Definition: InstrProf.h:915
InstrProfRecord(const InstrProfRecord &RHS)
Definition: InstrProf.h:837
uint32_t getNumValueDataForSite(uint32_t ValueKind, uint32_t Site) const
Return the number of value data collected for ValueKind at profiling site: Site.
Definition: InstrProf.h:1061
InstrProfRecord(std::vector< uint64_t > Counts, std::vector< uint8_t > BitmapBytes)
Definition: InstrProf.h:833
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
Definition: InstrProf.h:1093
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:790
void sortValueData()
Sort value profile data (per site) by count.
Definition: InstrProf.h:902
std::vector< uint8_t > BitmapBytes
Definition: InstrProf.h:829
InstrProfRecord & operator=(InstrProfRecord &&)=default
void Clear()
Clear value data entries and edge counters.
Definition: InstrProf.h:909
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:964
InstrProfValueSiteRecord(InputIterator F, InputIterator L)
Definition: InstrProf.h:801
void sortByCount()
Sort ValueData Descending by Count.
Definition: InstrProf.h:1103
void sortByTargetValues()
Sort ValueData ascending by Value.
Definition: InstrProf.h:805
std::vector< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
Definition: InstrProf.h:797
void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
Definition: InstrProf.cpp:841
void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
Definition: InstrProf.cpp:741
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
Definition: InstrProf.cpp:870
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:1035
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts)
Definition: InstrProf.h:1026
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts, std::vector< uint8_t > BitmapBytes)
Definition: InstrProf.h:1029
static void setCSFlagInHash(uint64_t &FuncHash)
Definition: InstrProf.h:1038
static const int CS_FLAG_IN_FUNC_HASH
Definition: InstrProf.h:1023
const std::string NameFilter
Definition: InstrProf.h:792
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1532
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Definition: InstrProf.h:780
Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
Definition: InstrProf.cpp:1504
void dump(raw_fd_ostream &OS) const
Definition: InstrProf.cpp:1551
CountSumOrPercent Overlap
Definition: InstrProf.h:756
CountSumOrPercent Base
Definition: InstrProf.h:752
uint64_t FuncHash
Definition: InstrProf.h:763
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1542
const std::string * BaseFilename
Definition: InstrProf.h:760
const std::string * TestFilename
Definition: InstrProf.h:761
void setFuncInfo(StringRef Name, uint64_t Hash)
Definition: InstrProf.h:770
CountSumOrPercent Unique
Definition: InstrProf.h:758
CountSumOrPercent Mismatch
Definition: InstrProf.h:757
StringRef FuncName
Definition: InstrProf.h:762
OverlapStatsLevel Level
Definition: InstrProf.h:759
OverlapStats(OverlapStatsLevel L=ProgramLevel)
Definition: InstrProf.h:766
CountSumOrPercent Test
Definition: InstrProf.h:754
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
Definition: InstrProf.h:378
static void createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces, std::vector< BPFunctionNode > &Nodes, bool RemoveOutlierUNs=true)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...
Definition: InstrProf.cpp:1010
std::vector< uint64_t > FunctionNameRefs
Definition: InstrProf.h:379
TemporalProfTraceTy(std::initializer_list< uint64_t > Trace={}, uint64_t Weight=1)
Definition: InstrProf.h:381
Function object to check whether the first component of a container supported by std::get (like std::...
Definition: STLExtras.h:1450