LLVM  14.0.0git
InstrProfData.inc
Go to the documentation of this file.
1 /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8 /*
9  * This is the main file that defines all the data structure, signature,
10  * constant literals that are shared across profiling runtime library,
11  * compiler (instrumentation), and host tools (reader/writer). The entities
12  * defined in this file affect the profile runtime ABI, the raw profile format,
13  * or both.
14  *
15  * The file has two identical copies. The primary copy lives in LLVM and
16  * the other one sits in compiler-rt/lib/profile directory. To make changes
17  * in this file, first modify the primary copy and copy it over to compiler-rt.
18  * Testing of any change in this file can start only after the two copies are
19  * synced up.
20  *
21  * The first part of the file includes macros that defines types, names, and
22  * initializers for the member fields of the core data structures. The field
23  * declarations for one structure is enabled by defining the field activation
24  * macro associated with that structure. Only one field activation record
25  * can be defined at one time and the rest definitions will be filtered out by
26  * the preprocessor.
27  *
28  * Examples of how the template is used to instantiate structure definition:
29  * 1. To declare a structure:
30  *
31  * struct ProfData {
32  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33  * Type Name;
34  * #include "llvm/ProfileData/InstrProfData.inc"
35  * };
36  *
37  * 2. To construct LLVM type arrays for the struct type:
38  *
39  * Type *DataTypes[] = {
40  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41  * LLVMType,
42  * #include "llvm/ProfileData/InstrProfData.inc"
43  * };
44  *
45  * 4. To construct constant array for the initializers:
46  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47  * Initializer,
48  * Constant *ConstantVals[] = {
49  * #include "llvm/ProfileData/InstrProfData.inc"
50  * };
51  *
52  *
53  * The second part of the file includes definitions all other entities that
54  * are related to runtime ABI and format. When no field activation macro is
55  * defined, this file can be included to introduce the definitions.
56  *
57 \*===----------------------------------------------------------------------===*/
58 
59 /* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60  * the compiler runtime. */
61 #ifndef INSTR_PROF_VISIBILITY
62 #define INSTR_PROF_VISIBILITY
63 #endif
64 
65 /* INSTR_PROF_DATA start. */
66 /* Definition of member fields of the per-function control structure. */
67 #ifndef INSTR_PROF_DATA
68 #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
69 #else
70 #define INSTR_PROF_DATA_DEFINED
71 #endif
72 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
75 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
76  ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
77  Inc->getHash()->getZExtValue()))
78 INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
79 /* This is used to map function pointers for the indirect call targets to
80  * function name hashes during the conversion from raw to merged profile
81  * data.
82  */
83 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
84  FunctionAddr)
85 INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
86  ValuesPtrExpr)
87 INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
88  ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
89 INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
90  ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
91 #undef INSTR_PROF_DATA
92 /* INSTR_PROF_DATA end. */
93 
94 
95 /* This is an internal data structure used by value profiler. It
96  * is defined here to allow serialization code sharing by LLVM
97  * to be used in unit test.
98  *
99  * typedef struct ValueProfNode {
100  * // InstrProfValueData VData;
101  * uint64_t Value;
102  * uint64_t Count;
103  * struct ValueProfNode *Next;
104  * } ValueProfNode;
105  */
106 /* INSTR_PROF_VALUE_NODE start. */
107 #ifndef INSTR_PROF_VALUE_NODE
108 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
109 #else
110 #define INSTR_PROF_DATA_DEFINED
111 #endif
113  ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
114 INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
115  ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
116 INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \
117  ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
118 #undef INSTR_PROF_VALUE_NODE
119 /* INSTR_PROF_VALUE_NODE end. */
120 
121 /* INSTR_PROF_RAW_HEADER start */
122 /* Definition of member fields of the raw profile header data structure. */
123 #ifndef INSTR_PROF_RAW_HEADER
124 #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
125 #else
126 #define INSTR_PROF_DATA_DEFINED
127 #endif
128 INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
129 INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
130 INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
131 INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
132 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
133 INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
134 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
135 INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
136 INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
137  (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
138 INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
139 INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
140 #undef INSTR_PROF_RAW_HEADER
141 /* INSTR_PROF_RAW_HEADER end */
142 
143 /* VALUE_PROF_FUNC_PARAM start */
144 /* Definition of parameter types of the runtime API used to do value profiling
145  * for a given value site.
146  */
147 #ifndef VALUE_PROF_FUNC_PARAM
148 #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
149 #define INSTR_PROF_COMMA
150 #else
151 #define INSTR_PROF_DATA_DEFINED
152 #define INSTR_PROF_COMMA ,
153 #endif
154 VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
155  INSTR_PROF_COMMA
156 VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
157 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
158 #undef VALUE_PROF_FUNC_PARAM
159 #undef INSTR_PROF_COMMA
160 /* VALUE_PROF_FUNC_PARAM end */
161 
162 /* VALUE_PROF_KIND start */
163 #ifndef VALUE_PROF_KIND
164 #define VALUE_PROF_KIND(Enumerator, Value, Descr)
165 #else
166 #define INSTR_PROF_DATA_DEFINED
167 #endif
168 /* For indirect function call value profiling, the addresses of the target
169  * functions are profiled by the instrumented code. The target addresses are
170  * written in the raw profile data and converted to target function name's MD5
171  * hash by the profile reader during deserialization. Typically, this happens
172  * when the raw profile data is read during profile merging.
173  *
174  * For this remapping the ProfData is used. ProfData contains both the function
175  * name hash and the function address.
176  */
177 VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
178 /* For memory intrinsic functions size profiling. */
179 VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
180 /* These two kinds must be the last to be
181  * declared. This is to make sure the string
182  * array created with the template can be
183  * indexed with the kind value.
184  */
185 VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
186 VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
187 
188 #undef VALUE_PROF_KIND
189 /* VALUE_PROF_KIND end */
190 
191 #undef COVMAP_V2_OR_V3
192 #ifdef COVMAP_V2
193 #define COVMAP_V2_OR_V3
194 #endif
195 #ifdef COVMAP_V3
196 #define COVMAP_V2_OR_V3
197 #endif
198 
199 /* COVMAP_FUNC_RECORD start */
200 /* Definition of member fields of the function record structure in coverage
201  * map.
202  */
203 #ifndef COVMAP_FUNC_RECORD
204 #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
205 #else
206 #define INSTR_PROF_DATA_DEFINED
207 #endif
208 #ifdef COVMAP_V1
209 COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
210  NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
214  NameValue.size()))
215 #endif
216 #ifdef COVMAP_V2_OR_V3
217 COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
219  llvm::Type::getInt64Ty(Ctx), NameHash))
220 #endif
221 COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
223  llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
224 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
225  llvm::ConstantInt::get( \
226  llvm::Type::getInt64Ty(Ctx), FuncHash))
227 #ifdef COVMAP_V3
228 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
230  llvm::Type::getInt64Ty(Ctx), FilenamesRef))
231 COVMAP_FUNC_RECORD(const char, \
233  CoverageMapping.size()), \
234  CoverageMapping,
236  CoverageMapping, CoverageMapping.size(), \
237  llvm::Type::getInt8Ty(Ctx)))
238 #endif
239 #undef COVMAP_FUNC_RECORD
240 /* COVMAP_FUNC_RECORD end. */
241 
242 /* COVMAP_HEADER start */
243 /* Definition of member fields of coverage map header.
244  */
245 #ifndef COVMAP_HEADER
246 #define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
247 #else
248 #define INSTR_PROF_DATA_DEFINED
249 #endif
250 COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
251  llvm::ConstantInt::get(Int32Ty, NRecords))
252 COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
253  llvm::ConstantInt::get(Int32Ty, FilenamesSize))
254 COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
255  llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
258 #undef COVMAP_HEADER
259 /* COVMAP_HEADER end. */
260 
261 
262 #ifdef INSTR_PROF_SECT_ENTRY
263 #define INSTR_PROF_DATA_DEFINED
264 INSTR_PROF_SECT_ENTRY(IPSK_data, \
265  INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
266  INSTR_PROF_DATA_COFF, "__DATA,")
267 INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
268  INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
269  INSTR_PROF_CNTS_COFF, "__DATA,")
270 INSTR_PROF_SECT_ENTRY(IPSK_name, \
271  INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
272  INSTR_PROF_NAME_COFF, "__DATA,")
273 INSTR_PROF_SECT_ENTRY(IPSK_vals, \
274  INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
275  INSTR_PROF_VALS_COFF, "__DATA,")
276 INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
277  INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
278  INSTR_PROF_VNODES_COFF, "__DATA,")
279 INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
280  INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
281  INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
282 INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
283  INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
284  INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
285 INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
286  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
287  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
288 
289 #undef INSTR_PROF_SECT_ENTRY
290 #endif
291 
292 
293 #ifdef INSTR_PROF_VALUE_PROF_DATA
294 #define INSTR_PROF_DATA_DEFINED
295 
296 #define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
297 /*!
298  * This is the header of the data structure that defines the on-disk
299  * layout of the value profile data of a particular kind for one function.
300  */
301 typedef struct ValueProfRecord {
302  /* The kind of the value profile record. */
303  uint32_t Kind;
304  /*
305  * The number of value profile sites. It is guaranteed to be non-zero;
306  * otherwise the record for this kind won't be emitted.
307  */
308  uint32_t NumValueSites;
309  /*
310  * The first element of the array that stores the number of profiled
311  * values for each value site. The size of the array is NumValueSites.
312  * Since NumValueSites is greater than zero, there is at least one
313  * element in the array.
314  */
315  uint8_t SiteCountArray[1];
316 
317  /*
318  * The fake declaration is for documentation purpose only.
319  * Align the start of next field to be on 8 byte boundaries.
320  uint8_t Padding[X];
321  */
322 
323  /* The array of value profile data. The size of the array is the sum
324  * of all elements in SiteCountArray[].
325  InstrProfValueData ValueData[];
326  */
327 
328 #ifdef __cplusplus
329  /*!
330  * Return the number of value sites.
331  */
332  uint32_t getNumValueSites() const { return NumValueSites; }
333  /*!
334  * Read data from this record and save it to Record.
335  */
336  void deserializeTo(InstrProfRecord &Record,
337  InstrProfSymtab *SymTab);
338  /*
339  * In-place byte swap:
340  * Do byte swap for this instance. \c Old is the original order before
341  * the swap, and \c New is the New byte order.
342  */
343  void swapBytes(support::endianness Old, support::endianness New);
344 #endif
345 } ValueProfRecord;
346 
347 /*!
348  * Per-function header/control data structure for value profiling
349  * data in indexed format.
350  */
351 typedef struct ValueProfData {
352  /*
353  * Total size in bytes including this field. It must be a multiple
354  * of sizeof(uint64_t).
355  */
356  uint32_t TotalSize;
357  /*
358  *The number of value profile kinds that has value profile data.
359  * In this implementation, a value profile kind is considered to
360  * have profile data if the number of value profile sites for the
361  * kind is not zero. More aggressively, the implementation can
362  * choose to check the actual data value: if none of the value sites
363  * has any profiled values, the kind can be skipped.
364  */
365  uint32_t NumValueKinds;
366 
367  /*
368  * Following are a sequence of variable length records. The prefix/header
369  * of each record is defined by ValueProfRecord type. The number of
370  * records is NumValueKinds.
371  * ValueProfRecord Record_1;
372  * ValueProfRecord Record_N;
373  */
374 
375 #if __cplusplus
376  /*!
377  * Return the total size in bytes of the on-disk value profile data
378  * given the data stored in Record.
379  */
380  static uint32_t getSize(const InstrProfRecord &Record);
381  /*!
382  * Return a pointer to \c ValueProfData instance ready to be streamed.
383  */
384  static std::unique_ptr<ValueProfData>
385  serializeFrom(const InstrProfRecord &Record);
386  /*!
387  * Check the integrity of the record.
388  */
389  Error checkIntegrity();
390  /*!
391  * Return a pointer to \c ValueProfileData instance ready to be read.
392  * All data in the instance are properly byte swapped. The input
393  * data is assumed to be in little endian order.
394  */
395  static Expected<std::unique_ptr<ValueProfData>>
396  getValueProfData(const unsigned char *SrcBuffer,
397  const unsigned char *const SrcBufferEnd,
398  support::endianness SrcDataEndianness);
399  /*!
400  * Swap byte order from \c Endianness order to host byte order.
401  */
402  void swapBytesToHost(support::endianness Endianness);
403  /*!
404  * Swap byte order from host byte order to \c Endianness order.
405  */
406  void swapBytesFromHost(support::endianness Endianness);
407  /*!
408  * Return the total size of \c ValueProfileData.
409  */
410  uint32_t getSize() const { return TotalSize; }
411  /*!
412  * Read data from this data and save it to \c Record.
413  */
414  void deserializeTo(InstrProfRecord &Record,
415  InstrProfSymtab *SymTab);
416  void operator delete(void *ptr) { ::operator delete(ptr); }
417 #endif
418 } ValueProfData;
419 
420 /*
421  * The closure is designed to abstact away two types of value profile data:
422  * - InstrProfRecord which is the primary data structure used to
423  * represent profile data in host tools (reader, writer, and profile-use)
424  * - value profile runtime data structure suitable to be used by C
425  * runtime library.
426  *
427  * Both sources of data need to serialize to disk/memory-buffer in common
428  * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
429  * writer to share the same format and code with indexed profile writer.
430  *
431  * For documentation of the member methods below, refer to corresponding methods
432  * in class InstrProfRecord.
433  */
434 typedef struct ValueProfRecordClosure {
435  const void *Record;
436  uint32_t (*GetNumValueKinds)(const void *Record);
437  uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
438  uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
439  uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
440 
441  /*
442  * After extracting the value profile data from the value profile record,
443  * this method is used to map the in-memory value to on-disk value. If
444  * the method is null, value will be written out untranslated.
445  */
446  uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
447  void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
448  uint32_t S);
449  ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
450 } ValueProfRecordClosure;
451 
452 INSTR_PROF_VISIBILITY ValueProfRecord *
453 getFirstValueProfRecord(ValueProfData *VPD);
454 INSTR_PROF_VISIBILITY ValueProfRecord *
455 getValueProfRecordNext(ValueProfRecord *VPR);
456 INSTR_PROF_VISIBILITY InstrProfValueData *
457 getValueProfRecordValueData(ValueProfRecord *VPR);
458 INSTR_PROF_VISIBILITY uint32_t
459 getValueProfRecordHeaderSize(uint32_t NumValueSites);
460 
461 #undef INSTR_PROF_VALUE_PROF_DATA
462 #endif /* INSTR_PROF_VALUE_PROF_DATA */
463 
464 
465 #ifdef INSTR_PROF_COMMON_API_IMPL
466 #define INSTR_PROF_DATA_DEFINED
467 #ifdef __cplusplus
468 #define INSTR_PROF_INLINE inline
469 #define INSTR_PROF_NULLPTR nullptr
470 #else
471 #define INSTR_PROF_INLINE
472 #define INSTR_PROF_NULLPTR NULL
473 #endif
474 
475 #ifndef offsetof
476 #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
477 #endif
478 
479 /*!
480  * Return the \c ValueProfRecord header size including the
481  * padding bytes.
482  */
483 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
484 uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
485  uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
486  sizeof(uint8_t) * NumValueSites;
487  /* Round the size to multiple of 8 bytes. */
488  Size = (Size + 7) & ~7;
489  return Size;
490 }
491 
492 /*!
493  * Return the total size of the value profile record including the
494  * header and the value data.
495  */
496 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
497 uint32_t getValueProfRecordSize(uint32_t NumValueSites,
498  uint32_t NumValueData) {
499  return getValueProfRecordHeaderSize(NumValueSites) +
500  sizeof(InstrProfValueData) * NumValueData;
501 }
502 
503 /*!
504  * Return the pointer to the start of value data array.
505  */
506 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
507 InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
508  return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
509  This->NumValueSites));
510 }
511 
512 /*!
513  * Return the total number of value data for \c This record.
514  */
515 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
516 uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
517  uint32_t NumValueData = 0;
518  uint32_t I;
519  for (I = 0; I < This->NumValueSites; I++)
520  NumValueData += This->SiteCountArray[I];
521  return NumValueData;
522 }
523 
524 /*!
525  * Use this method to advance to the next \c This \c ValueProfRecord.
526  */
527 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
528 ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
529  uint32_t NumValueData = getValueProfRecordNumValueData(This);
530  return (ValueProfRecord *)((char *)This +
531  getValueProfRecordSize(This->NumValueSites,
532  NumValueData));
533 }
534 
535 /*!
536  * Return the first \c ValueProfRecord instance.
537  */
538 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
539 ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
540  return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
541 }
542 
543 /* Closure based interfaces. */
544 
545 /*!
546  * Return the total size in bytes of the on-disk value profile data
547  * given the data stored in Record.
548  */
549 INSTR_PROF_VISIBILITY uint32_t
550 getValueProfDataSize(ValueProfRecordClosure *Closure) {
551  uint32_t Kind;
552  uint32_t TotalSize = sizeof(ValueProfData);
553  const void *Record = Closure->Record;
554 
555  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
556  uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
557  if (!NumValueSites)
558  continue;
559  TotalSize += getValueProfRecordSize(NumValueSites,
560  Closure->GetNumValueData(Record, Kind));
561  }
562  return TotalSize;
563 }
564 
565 /*!
566  * Extract value profile data of a function for the profile kind \c ValueKind
567  * from the \c Closure and serialize the data into \c This record instance.
568  */
569 INSTR_PROF_VISIBILITY void
570 serializeValueProfRecordFrom(ValueProfRecord *This,
571  ValueProfRecordClosure *Closure,
572  uint32_t ValueKind, uint32_t NumValueSites) {
573  uint32_t S;
574  const void *Record = Closure->Record;
575  This->Kind = ValueKind;
576  This->NumValueSites = NumValueSites;
577  InstrProfValueData *DstVD = getValueProfRecordValueData(This);
578 
579  for (S = 0; S < NumValueSites; S++) {
580  uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
581  This->SiteCountArray[S] = ND;
582  Closure->GetValueForSite(Record, DstVD, ValueKind, S);
583  DstVD += ND;
584  }
585 }
586 
587 /*!
588  * Extract value profile data of a function from the \c Closure
589  * and serialize the data into \c DstData if it is not NULL or heap
590  * memory allocated by the \c Closure's allocator method. If \c
591  * DstData is not null, the caller is expected to set the TotalSize
592  * in DstData.
593  */
594 INSTR_PROF_VISIBILITY ValueProfData *
595 serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
596  ValueProfData *DstData) {
597  uint32_t Kind;
598  uint32_t TotalSize =
599  DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
600 
601  ValueProfData *VPD =
602  DstData ? DstData : Closure->AllocValueProfData(TotalSize);
603 
604  VPD->TotalSize = TotalSize;
605  VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
606  ValueProfRecord *VR = getFirstValueProfRecord(VPD);
607  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
608  uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
609  if (!NumValueSites)
610  continue;
611  serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
612  VR = getValueProfRecordNext(VR);
613  }
614  return VPD;
615 }
616 
617 #undef INSTR_PROF_COMMON_API_IMPL
618 #endif /* INSTR_PROF_COMMON_API_IMPL */
619 
620 /*============================================================================*/
621 
622 #ifndef INSTR_PROF_DATA_DEFINED
623 
624 #ifndef INSTR_PROF_DATA_INC
625 #define INSTR_PROF_DATA_INC
626 
627 /* Helper macros. */
628 #define INSTR_PROF_SIMPLE_QUOTE(x) #x
629 #define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
630 #define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
631 #define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
632 
633 /* Magic number to detect file format and endianness.
634  * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
635  * so that utilities, like strings, don't grab it as a string. 129 is also
636  * invalid UTF-8, and high enough to be interesting.
637  * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
638  * for 32-bit platforms.
639  */
640 #define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
641  (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
642  (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
643 #define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
644  (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
645  (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
646 
647 /* Raw profile format version (start from 1). */
648 #define INSTR_PROF_RAW_VERSION 7
649 /* Indexed profile format version (start from 1). */
650 #define INSTR_PROF_INDEX_VERSION 7
651 /* Coverage mapping format version (start from 0). */
652 #define INSTR_PROF_COVMAP_VERSION 5
653 
654 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
655  * version for other variants of profile. We set the lowest bit of the upper 8
656  * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
657  * generated profile, and 0 if this is a Clang FE generated profile.
658  * 1 in bit 57 indicates there are context-sensitive records in the profile.
659  */
660 #define VARIANT_MASKS_ALL 0xff00000000000000ULL
661 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
662 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
663 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
664 #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
665 #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
666 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
667 #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
668 
669 /* The variable that holds the name of the profile data
670  * specified via command line. */
671 #define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
672 
673 /* section name strings common to all targets other
674  than WIN32 */
675 #define INSTR_PROF_DATA_COMMON __llvm_prf_data
676 #define INSTR_PROF_NAME_COMMON __llvm_prf_names
677 #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
678 #define INSTR_PROF_VALS_COMMON __llvm_prf_vals
679 #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
680 #define INSTR_PROF_COVMAP_COMMON __llvm_covmap
681 #define INSTR_PROF_COVFUN_COMMON __llvm_covfun
682 #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
683 /* Windows section names. Because these section names contain dollar characters,
684  * they must be quoted.
685  */
686 #define INSTR_PROF_DATA_COFF ".lprfd$M"
687 #define INSTR_PROF_NAME_COFF ".lprfn$M"
688 #define INSTR_PROF_CNTS_COFF ".lprfc$M"
689 #define INSTR_PROF_VALS_COFF ".lprfv$M"
690 #define INSTR_PROF_VNODES_COFF ".lprfnd$M"
691 #define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
692 #define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
693 #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
694 
695 #ifdef _WIN32
696 /* Runtime section names and name strings. */
697 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
698 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
699 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
700 /* Array of pointers. Each pointer points to a list
701  * of value nodes associated with one value site.
702  */
703 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
704 /* Value profile nodes section. */
705 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
706 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
707 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
708 #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
709 #else
710 /* Runtime section names and name strings. */
711 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
712 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
713 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
714 /* Array of pointers. Each pointer points to a list
715  * of value nodes associated with one value site.
716  */
717 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
718 /* Value profile nodes section. */
719 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
720 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
721 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
722 /* Order file instrumentation. */
723 #define INSTR_PROF_ORDERFILE_SECT_NAME \
724  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
725 #endif
726 
727 #define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
728 #define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
729  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
730 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
731 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
732  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
733 
734 /* Macros to define start/stop section symbol for a given
735  * section on Linux. For instance
736  * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
737  * expand to __start___llvm_prof_data
738  */
739 #define INSTR_PROF_SECT_START(Sect) \
740  INSTR_PROF_CONCAT(__start_,Sect)
741 #define INSTR_PROF_SECT_STOP(Sect) \
742  INSTR_PROF_CONCAT(__stop_,Sect)
743 
744 /* Value Profiling API linkage name. */
745 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
746 #define INSTR_PROF_VALUE_PROF_FUNC_STR \
747  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
748 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
749 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
750  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
751 
752 /* InstrProfile per-function control data alignment. */
753 #define INSTR_PROF_DATA_ALIGNMENT 8
754 
755 /* The data structure that represents a tracked value by the
756  * value profiler.
757  */
758 typedef struct InstrProfValueData {
759  /* Profiled value. */
760  uint64_t Value;
761  /* Number of times the value appears in the training run. */
762  uint64_t Count;
763 } InstrProfValueData;
764 
765 #endif /* INSTR_PROF_DATA_INC */
766 
767 #ifndef INSTR_ORDER_FILE_INC
768 /* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
769 #define INSTR_ORDER_FILE_BUFFER_SIZE 131072
770 #define INSTR_ORDER_FILE_BUFFER_BITS 17
771 #define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
772 #endif /* INSTR_ORDER_FILE_INC */
773 #else
774 #undef INSTR_PROF_DATA_DEFINED
775 #endif
776 
777 #undef COVMAP_V2_OR_V3
778 
779 #ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
780 
781 #ifdef __cplusplus
782 #define INSTR_PROF_INLINE inline
783 #else
784 #define INSTR_PROF_INLINE
785 #endif
786 
787 /* The value range buckets (22 buckets) for the memop size value profiling looks
788  * like:
789  *
790  * [0, 0]
791  * [1, 1]
792  * [2, 2]
793  * [3, 3]
794  * [4, 4]
795  * [5, 5]
796  * [6, 6]
797  * [7, 7]
798  * [8, 8]
799  * [9, 15]
800  * [16, 16]
801  * [17, 31]
802  * [32, 32]
803  * [33, 63]
804  * [64, 64]
805  * [65, 127]
806  * [128, 128]
807  * [129, 255]
808  * [256, 256]
809  * [257, 511]
810  * [512, 512]
811  * [513, UINT64_MAX]
812  *
813  * Each range has a 'representative value' which is the lower end value of the
814  * range and used to store in the runtime profile data records and the VP
815  * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
816  */
817 #define INSTR_PROF_NUM_BUCKETS 22
818 
819 /*
820  * Clz and Popcount. This code was copied from
821  * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
822  * llvm/include/llvm/Support/MathExtras.h.
823  */
824 #if defined(_MSC_VER) && !defined(__clang__)
825 
826 #include <intrin.h>
827 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
828 int InstProfClzll(unsigned long long X) {
829  unsigned long LeadZeroIdx = 0;
830 #if !defined(_M_ARM64) && !defined(_M_X64)
831  // Scan the high 32 bits.
832  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
833  return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
834  // from the MSB.
835  // Scan the low 32 bits.
836  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
837  return (int)(63 - LeadZeroIdx);
838 #else
839  if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
840 #endif
841  return 64;
842 }
843 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
844 int InstProfPopcountll(unsigned long long X) {
845  // This code originates from https://reviews.llvm.org/rG30626254510f.
846  unsigned long long v = X;
847  v = v - ((v >> 1) & 0x5555555555555555ULL);
848  v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
849  v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
850  return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
851 }
852 
853 #else
854 
855 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
856 int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
857 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
858 int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
859 
860 #endif /* defined(_MSC_VER) && !defined(__clang__) */
861 
862 /* Map an (observed) memop size value to the representative value of its range.
863  * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
864 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
865 InstrProfGetRangeRepValue(uint64_t Value) {
866  if (Value <= 8)
867  // The first ranges are individually tracked. Use the value as is.
868  return Value;
869  else if (Value >= 513)
870  // The last range is mapped to its lowest value.
871  return 513;
872  else if (InstProfPopcountll(Value) == 1)
873  // If it's a power of two, use it as is.
874  return Value;
875  else
876  // Otherwise, take to the previous power of two + 1.
877  return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
878 }
879 
880 /* Return true if the range that an (observed) memop size value belongs to has
881  * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
882  * false, 64 -> true, 100 -> false, 513 -> false. */
883 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
884 InstrProfIsSingleValRange(uint64_t Value) {
885  if (Value <= 8)
886  // The first ranges are individually tracked.
887  return 1;
888  else if (InstProfPopcountll(Value) == 1)
889  // If it's a power of two, there's only one value.
890  return 1;
891  else
892  // Otherwise, there's more than one value in the range.
893  return 0;
894 }
895 
896 #endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
VALUE_PROF_FUNC_PARAM
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType)
Int32Ty
IntegerType * Int32Ty
Definition: NVVMIntrRange.cpp:67
functions
amdgpu propagate attributes Late propagate attributes from kernels to functions
Definition: AMDGPUPropagateAttributes.cpp:199
INSTR_PROF_DATA
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init)
Definition: InstrProf.h:1125
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
llvm::msgpack::Endianness
constexpr support::endianness Endianness
The endianness of all multi-byte encoded values in MessagePack.
Definition: MsgPack.h:24
llvm::lltok::Error
@ Error
Definition: LLToken.h:21
endianness
llvm::support::endianness endianness
Definition: BinaryStreamReader.cpp:16
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:255
llvm::getPGOFuncNameVarInitializer
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:442
offsetof
#define offsetof(TYPE, MEMBER)
Definition: AMDHSAKernelDescriptor.h:23
llvm::IndexedInstrProf::ComputeHash
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:995
llvm::ConstantExpr::getBitCast
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2221
llvm::COFF::NameSize
@ NameSize
Definition: COFF.h:58
intrinsic
QP Compare Ordered outs ins xscmpudp No intrinsic
Definition: README_P9.txt:303
INSTR_PROF_RAW_HEADER
#define INSTR_PROF_RAW_HEADER(Type, Name, Init)
Definition: InstrProf.h:1134
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:201
llvm::Data
@ Data
Definition: SIMachineScheduler.h:56
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:203
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:180
COVMAP_FUNC_RECORD
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init)
Definition: CoverageMapping.h:919
llvm::AMDGPU::HSAMD::ValueKind
ValueKind
Value kinds.
Definition: AMDGPUMetadata.h:73
llvm::coverage::CurrentVersion
@ CurrentVersion
Definition: CoverageMapping.h:1010
VALUE_PROF_KIND
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
Definition: InstrProf.h:239
COVMAP_HEADER
#define COVMAP_HEADER(Type, LLVMType, Name, Init)
Definition: CoverageMapping.h:972
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:899
llvm::ConstantDataArray::getRaw
static Constant * getRaw(StringRef Data, uint64_t NumElements, Type *ElementTy)
getRaw() constructor - Return a constant with array type with an element count and element type match...
Definition: Constants.h:710
llvm::IndexedInstrProf::Version
const uint64_t Version
Definition: InstrProf.h:991
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
INSTR_PROF_VALUE_NODE
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init)
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
__DATA
the multiplication has a latency of four as opposed to two cycles for the movl lea variant It appears gcc place string data with linkonce linkage in section coalesced instead of section __DATA
Definition: README.txt:260
I
#define I(x, y, z)
Definition: MD5.cpp:59
Magic
const char Magic[]
Definition: Archive.cpp:41
size
i< reg-> size
Definition: README.txt:166
llvm::ArrayType::get
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:604
uint32_t
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:204
uint16_t
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
This
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical This
Definition: README.txt:418
llvm::msgpack::Type
Type
MessagePack types as defined in the standard, with the exception of Integer being divided into a sign...
Definition: MsgPackReader.h:49
INSTR_PROF_SECT_ENTRY
#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix)
Definition: InstrProf.h:58