LLVM  13.0.0git
InstrProfData.inc
Go to the documentation of this file.
1 /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8 /*
9  * This is the main file that defines all the data structure, signature,
10  * constant literals that are shared across profiling runtime library,
11  * compiler (instrumentation), and host tools (reader/writer). The entities
12  * defined in this file affect the profile runtime ABI, the raw profile format,
13  * or both.
14  *
15  * The file has two identical copies. The primary copy lives in LLVM and
16  * the other one sits in compiler-rt/lib/profile directory. To make changes
17  * in this file, first modify the primary copy and copy it over to compiler-rt.
18  * Testing of any change in this file can start only after the two copies are
19  * synced up.
20  *
21  * The first part of the file includes macros that defines types, names, and
22  * initializers for the member fields of the core data structures. The field
23  * declarations for one structure is enabled by defining the field activation
24  * macro associated with that structure. Only one field activation record
25  * can be defined at one time and the rest definitions will be filtered out by
26  * the preprocessor.
27  *
28  * Examples of how the template is used to instantiate structure definition:
29  * 1. To declare a structure:
30  *
31  * struct ProfData {
32  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33  * Type Name;
34  * #include "llvm/ProfileData/InstrProfData.inc"
35  * };
36  *
37  * 2. To construct LLVM type arrays for the struct type:
38  *
39  * Type *DataTypes[] = {
40  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41  * LLVMType,
42  * #include "llvm/ProfileData/InstrProfData.inc"
43  * };
44  *
45  * 4. To construct constant array for the initializers:
46  * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47  * Initializer,
48  * Constant *ConstantVals[] = {
49  * #include "llvm/ProfileData/InstrProfData.inc"
50  * };
51  *
52  *
53  * The second part of the file includes definitions all other entities that
54  * are related to runtime ABI and format. When no field activation macro is
55  * defined, this file can be included to introduce the definitions.
56  *
57 \*===----------------------------------------------------------------------===*/
58 
59 /* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60  * the compiler runtime. */
61 #ifndef INSTR_PROF_VISIBILITY
62 #define INSTR_PROF_VISIBILITY
63 #endif
64 
65 /* INSTR_PROF_DATA start. */
66 /* Definition of member fields of the per-function control structure. */
67 #ifndef INSTR_PROF_DATA
68 #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
69 #else
70 #define INSTR_PROF_DATA_DEFINED
71 #endif
72 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
75 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
76  ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
77  Inc->getHash()->getZExtValue()))
78 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
79  ConstantExpr::getBitCast(CounterPtr, \
80  llvm::Type::getInt64PtrTy(Ctx)))
81 /* This is used to map function pointers for the indirect call targets to
82  * function name hashes during the conversion from raw to merged profile
83  * data.
84  */
85 INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
86  FunctionAddr)
87 INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
88  ValuesPtrExpr)
89 INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
90  ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
91 INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
92  ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
93 #undef INSTR_PROF_DATA
94 /* INSTR_PROF_DATA end. */
95 
96 
97 /* This is an internal data structure used by value profiler. It
98  * is defined here to allow serialization code sharing by LLVM
99  * to be used in unit test.
100  *
101  * typedef struct ValueProfNode {
102  * // InstrProfValueData VData;
103  * uint64_t Value;
104  * uint64_t Count;
105  * struct ValueProfNode *Next;
106  * } ValueProfNode;
107  */
108 /* INSTR_PROF_VALUE_NODE start. */
109 #ifndef INSTR_PROF_VALUE_NODE
110 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
111 #else
112 #define INSTR_PROF_DATA_DEFINED
113 #endif
115  ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
116 INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
117  ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
118 INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \
119  ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
120 #undef INSTR_PROF_VALUE_NODE
121 /* INSTR_PROF_VALUE_NODE end. */
122 
123 /* INSTR_PROF_RAW_HEADER start */
124 /* Definition of member fields of the raw profile header data structure. */
125 #ifndef INSTR_PROF_RAW_HEADER
126 #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
127 #else
128 #define INSTR_PROF_DATA_DEFINED
129 #endif
130 INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
131 INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
132 INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
133 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
134 INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
135 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
136 INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
137 INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
138 INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
139 INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
140 #undef INSTR_PROF_RAW_HEADER
141 /* INSTR_PROF_RAW_HEADER end */
142 
143 /* VALUE_PROF_FUNC_PARAM start */
144 /* Definition of parameter types of the runtime API used to do value profiling
145  * for a given value site.
146  */
147 #ifndef VALUE_PROF_FUNC_PARAM
148 #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
149 #define INSTR_PROF_COMMA
150 #else
151 #define INSTR_PROF_DATA_DEFINED
152 #define INSTR_PROF_COMMA ,
153 #endif
154 VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
155  INSTR_PROF_COMMA
156 VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
157 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
158 #undef VALUE_PROF_FUNC_PARAM
159 #undef INSTR_PROF_COMMA
160 /* VALUE_PROF_FUNC_PARAM end */
161 
162 /* VALUE_PROF_KIND start */
163 #ifndef VALUE_PROF_KIND
164 #define VALUE_PROF_KIND(Enumerator, Value, Descr)
165 #else
166 #define INSTR_PROF_DATA_DEFINED
167 #endif
168 /* For indirect function call value profiling, the addresses of the target
169  * functions are profiled by the instrumented code. The target addresses are
170  * written in the raw profile data and converted to target function name's MD5
171  * hash by the profile reader during deserialization. Typically, this happens
172  * when the raw profile data is read during profile merging.
173  *
174  * For this remapping the ProfData is used. ProfData contains both the function
175  * name hash and the function address.
176  */
177 VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
178 /* For memory intrinsic functions size profiling. */
179 VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
180 /* These two kinds must be the last to be
181  * declared. This is to make sure the string
182  * array created with the template can be
183  * indexed with the kind value.
184  */
185 VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
186 VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
187 
188 #undef VALUE_PROF_KIND
189 /* VALUE_PROF_KIND end */
190 
191 #undef COVMAP_V2_OR_V3
192 #ifdef COVMAP_V2
193 #define COVMAP_V2_OR_V3
194 #endif
195 #ifdef COVMAP_V3
196 #define COVMAP_V2_OR_V3
197 #endif
198 
199 /* COVMAP_FUNC_RECORD start */
200 /* Definition of member fields of the function record structure in coverage
201  * map.
202  */
203 #ifndef COVMAP_FUNC_RECORD
204 #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
205 #else
206 #define INSTR_PROF_DATA_DEFINED
207 #endif
208 #ifdef COVMAP_V1
209 COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
210  NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
214  NameValue.size()))
215 #endif
216 #ifdef COVMAP_V2_OR_V3
217 COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
219  llvm::Type::getInt64Ty(Ctx), NameHash))
220 #endif
221 COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
223  llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
224 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
225  llvm::ConstantInt::get( \
226  llvm::Type::getInt64Ty(Ctx), FuncHash))
227 #ifdef COVMAP_V3
228 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
230  llvm::Type::getInt64Ty(Ctx), FilenamesRef))
231 COVMAP_FUNC_RECORD(const char, \
233  CoverageMapping.size()), \
234  CoverageMapping,
236  CoverageMapping, CoverageMapping.size(), \
237  llvm::Type::getInt8Ty(Ctx)))
238 #endif
239 #undef COVMAP_FUNC_RECORD
240 /* COVMAP_FUNC_RECORD end. */
241 
242 /* COVMAP_HEADER start */
243 /* Definition of member fields of coverage map header.
244  */
245 #ifndef COVMAP_HEADER
246 #define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
247 #else
248 #define INSTR_PROF_DATA_DEFINED
249 #endif
250 COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
251  llvm::ConstantInt::get(Int32Ty, NRecords))
252 COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
253  llvm::ConstantInt::get(Int32Ty, FilenamesSize))
254 COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
255  llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
258 #undef COVMAP_HEADER
259 /* COVMAP_HEADER end. */
260 
261 
262 #ifdef INSTR_PROF_SECT_ENTRY
263 #define INSTR_PROF_DATA_DEFINED
264 INSTR_PROF_SECT_ENTRY(IPSK_data, \
265  INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
266  INSTR_PROF_DATA_COFF, "__DATA,")
267 INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
268  INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
269  INSTR_PROF_CNTS_COFF, "__DATA,")
270 INSTR_PROF_SECT_ENTRY(IPSK_name, \
271  INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
272  INSTR_PROF_NAME_COFF, "__DATA,")
273 INSTR_PROF_SECT_ENTRY(IPSK_vals, \
274  INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
275  INSTR_PROF_VALS_COFF, "__DATA,")
276 INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
277  INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
278  INSTR_PROF_VNODES_COFF, "__DATA,")
279 INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
280  INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
281  INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
282 INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
283  INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
284  INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
285 INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
286  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
287  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
288 
289 #undef INSTR_PROF_SECT_ENTRY
290 #endif
291 
292 
293 #ifdef INSTR_PROF_VALUE_PROF_DATA
294 #define INSTR_PROF_DATA_DEFINED
295 
296 #define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
297 /*!
298  * This is the header of the data structure that defines the on-disk
299  * layout of the value profile data of a particular kind for one function.
300  */
301 typedef struct ValueProfRecord {
302  /* The kind of the value profile record. */
303  uint32_t Kind;
304  /*
305  * The number of value profile sites. It is guaranteed to be non-zero;
306  * otherwise the record for this kind won't be emitted.
307  */
308  uint32_t NumValueSites;
309  /*
310  * The first element of the array that stores the number of profiled
311  * values for each value site. The size of the array is NumValueSites.
312  * Since NumValueSites is greater than zero, there is at least one
313  * element in the array.
314  */
315  uint8_t SiteCountArray[1];
316 
317  /*
318  * The fake declaration is for documentation purpose only.
319  * Align the start of next field to be on 8 byte boundaries.
320  uint8_t Padding[X];
321  */
322 
323  /* The array of value profile data. The size of the array is the sum
324  * of all elements in SiteCountArray[].
325  InstrProfValueData ValueData[];
326  */
327 
328 #ifdef __cplusplus
329  /*!
330  * Return the number of value sites.
331  */
332  uint32_t getNumValueSites() const { return NumValueSites; }
333  /*!
334  * Read data from this record and save it to Record.
335  */
336  void deserializeTo(InstrProfRecord &Record,
337  InstrProfSymtab *SymTab);
338  /*
339  * In-place byte swap:
340  * Do byte swap for this instance. \c Old is the original order before
341  * the swap, and \c New is the New byte order.
342  */
343  void swapBytes(support::endianness Old, support::endianness New);
344 #endif
345 } ValueProfRecord;
346 
347 /*!
348  * Per-function header/control data structure for value profiling
349  * data in indexed format.
350  */
351 typedef struct ValueProfData {
352  /*
353  * Total size in bytes including this field. It must be a multiple
354  * of sizeof(uint64_t).
355  */
356  uint32_t TotalSize;
357  /*
358  *The number of value profile kinds that has value profile data.
359  * In this implementation, a value profile kind is considered to
360  * have profile data if the number of value profile sites for the
361  * kind is not zero. More aggressively, the implementation can
362  * choose to check the actual data value: if none of the value sites
363  * has any profiled values, the kind can be skipped.
364  */
365  uint32_t NumValueKinds;
366 
367  /*
368  * Following are a sequence of variable length records. The prefix/header
369  * of each record is defined by ValueProfRecord type. The number of
370  * records is NumValueKinds.
371  * ValueProfRecord Record_1;
372  * ValueProfRecord Record_N;
373  */
374 
375 #if __cplusplus
376  /*!
377  * Return the total size in bytes of the on-disk value profile data
378  * given the data stored in Record.
379  */
380  static uint32_t getSize(const InstrProfRecord &Record);
381  /*!
382  * Return a pointer to \c ValueProfData instance ready to be streamed.
383  */
384  static std::unique_ptr<ValueProfData>
385  serializeFrom(const InstrProfRecord &Record);
386  /*!
387  * Check the integrity of the record.
388  */
389  Error checkIntegrity();
390  /*!
391  * Return a pointer to \c ValueProfileData instance ready to be read.
392  * All data in the instance are properly byte swapped. The input
393  * data is assumed to be in little endian order.
394  */
395  static Expected<std::unique_ptr<ValueProfData>>
396  getValueProfData(const unsigned char *SrcBuffer,
397  const unsigned char *const SrcBufferEnd,
398  support::endianness SrcDataEndianness);
399  /*!
400  * Swap byte order from \c Endianness order to host byte order.
401  */
402  void swapBytesToHost(support::endianness Endianness);
403  /*!
404  * Swap byte order from host byte order to \c Endianness order.
405  */
406  void swapBytesFromHost(support::endianness Endianness);
407  /*!
408  * Return the total size of \c ValueProfileData.
409  */
410  uint32_t getSize() const { return TotalSize; }
411  /*!
412  * Read data from this data and save it to \c Record.
413  */
414  void deserializeTo(InstrProfRecord &Record,
415  InstrProfSymtab *SymTab);
416  void operator delete(void *ptr) { ::operator delete(ptr); }
417 #endif
418 } ValueProfData;
419 
420 /*
421  * The closure is designed to abstact away two types of value profile data:
422  * - InstrProfRecord which is the primary data structure used to
423  * represent profile data in host tools (reader, writer, and profile-use)
424  * - value profile runtime data structure suitable to be used by C
425  * runtime library.
426  *
427  * Both sources of data need to serialize to disk/memory-buffer in common
428  * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
429  * writer to share the same format and code with indexed profile writer.
430  *
431  * For documentation of the member methods below, refer to corresponding methods
432  * in class InstrProfRecord.
433  */
434 typedef struct ValueProfRecordClosure {
435  const void *Record;
436  uint32_t (*GetNumValueKinds)(const void *Record);
437  uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
438  uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
439  uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
440 
441  /*
442  * After extracting the value profile data from the value profile record,
443  * this method is used to map the in-memory value to on-disk value. If
444  * the method is null, value will be written out untranslated.
445  */
446  uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
447  void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
448  uint32_t S);
449  ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
450 } ValueProfRecordClosure;
451 
452 INSTR_PROF_VISIBILITY ValueProfRecord *
453 getFirstValueProfRecord(ValueProfData *VPD);
454 INSTR_PROF_VISIBILITY ValueProfRecord *
455 getValueProfRecordNext(ValueProfRecord *VPR);
456 INSTR_PROF_VISIBILITY InstrProfValueData *
457 getValueProfRecordValueData(ValueProfRecord *VPR);
458 INSTR_PROF_VISIBILITY uint32_t
459 getValueProfRecordHeaderSize(uint32_t NumValueSites);
460 
461 #undef INSTR_PROF_VALUE_PROF_DATA
462 #endif /* INSTR_PROF_VALUE_PROF_DATA */
463 
464 
465 #ifdef INSTR_PROF_COMMON_API_IMPL
466 #define INSTR_PROF_DATA_DEFINED
467 #ifdef __cplusplus
468 #define INSTR_PROF_INLINE inline
469 #define INSTR_PROF_NULLPTR nullptr
470 #else
471 #define INSTR_PROF_INLINE
472 #define INSTR_PROF_NULLPTR NULL
473 #endif
474 
475 #ifndef offsetof
476 #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
477 #endif
478 
479 /*!
480  * Return the \c ValueProfRecord header size including the
481  * padding bytes.
482  */
483 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
484 uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
485  uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
486  sizeof(uint8_t) * NumValueSites;
487  /* Round the size to multiple of 8 bytes. */
488  Size = (Size + 7) & ~7;
489  return Size;
490 }
491 
492 /*!
493  * Return the total size of the value profile record including the
494  * header and the value data.
495  */
496 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
497 uint32_t getValueProfRecordSize(uint32_t NumValueSites,
498  uint32_t NumValueData) {
499  return getValueProfRecordHeaderSize(NumValueSites) +
500  sizeof(InstrProfValueData) * NumValueData;
501 }
502 
503 /*!
504  * Return the pointer to the start of value data array.
505  */
506 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
507 InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
508  return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
509  This->NumValueSites));
510 }
511 
512 /*!
513  * Return the total number of value data for \c This record.
514  */
515 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
516 uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
517  uint32_t NumValueData = 0;
518  uint32_t I;
519  for (I = 0; I < This->NumValueSites; I++)
520  NumValueData += This->SiteCountArray[I];
521  return NumValueData;
522 }
523 
524 /*!
525  * Use this method to advance to the next \c This \c ValueProfRecord.
526  */
527 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
528 ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
529  uint32_t NumValueData = getValueProfRecordNumValueData(This);
530  return (ValueProfRecord *)((char *)This +
531  getValueProfRecordSize(This->NumValueSites,
532  NumValueData));
533 }
534 
535 /*!
536  * Return the first \c ValueProfRecord instance.
537  */
538 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
539 ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
540  return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
541 }
542 
543 /* Closure based interfaces. */
544 
545 /*!
546  * Return the total size in bytes of the on-disk value profile data
547  * given the data stored in Record.
548  */
549 INSTR_PROF_VISIBILITY uint32_t
550 getValueProfDataSize(ValueProfRecordClosure *Closure) {
551  uint32_t Kind;
552  uint32_t TotalSize = sizeof(ValueProfData);
553  const void *Record = Closure->Record;
554 
555  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
556  uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
557  if (!NumValueSites)
558  continue;
559  TotalSize += getValueProfRecordSize(NumValueSites,
560  Closure->GetNumValueData(Record, Kind));
561  }
562  return TotalSize;
563 }
564 
565 /*!
566  * Extract value profile data of a function for the profile kind \c ValueKind
567  * from the \c Closure and serialize the data into \c This record instance.
568  */
569 INSTR_PROF_VISIBILITY void
570 serializeValueProfRecordFrom(ValueProfRecord *This,
571  ValueProfRecordClosure *Closure,
572  uint32_t ValueKind, uint32_t NumValueSites) {
573  uint32_t S;
574  const void *Record = Closure->Record;
575  This->Kind = ValueKind;
576  This->NumValueSites = NumValueSites;
577  InstrProfValueData *DstVD = getValueProfRecordValueData(This);
578 
579  for (S = 0; S < NumValueSites; S++) {
580  uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
581  This->SiteCountArray[S] = ND;
582  Closure->GetValueForSite(Record, DstVD, ValueKind, S);
583  DstVD += ND;
584  }
585 }
586 
587 /*!
588  * Extract value profile data of a function from the \c Closure
589  * and serialize the data into \c DstData if it is not NULL or heap
590  * memory allocated by the \c Closure's allocator method. If \c
591  * DstData is not null, the caller is expected to set the TotalSize
592  * in DstData.
593  */
594 INSTR_PROF_VISIBILITY ValueProfData *
595 serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
596  ValueProfData *DstData) {
597  uint32_t Kind;
598  uint32_t TotalSize =
599  DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
600 
601  ValueProfData *VPD =
602  DstData ? DstData : Closure->AllocValueProfData(TotalSize);
603 
604  VPD->TotalSize = TotalSize;
605  VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
606  ValueProfRecord *VR = getFirstValueProfRecord(VPD);
607  for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
608  uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
609  if (!NumValueSites)
610  continue;
611  serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
612  VR = getValueProfRecordNext(VR);
613  }
614  return VPD;
615 }
616 
617 #undef INSTR_PROF_COMMON_API_IMPL
618 #endif /* INSTR_PROF_COMMON_API_IMPL */
619 
620 /*============================================================================*/
621 
622 #ifndef INSTR_PROF_DATA_DEFINED
623 
624 #ifndef INSTR_PROF_DATA_INC
625 #define INSTR_PROF_DATA_INC
626 
627 /* Helper macros. */
628 #define INSTR_PROF_SIMPLE_QUOTE(x) #x
629 #define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
630 #define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
631 #define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
632 
633 /* Magic number to detect file format and endianness.
634  * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
635  * so that utilities, like strings, don't grab it as a string. 129 is also
636  * invalid UTF-8, and high enough to be interesting.
637  * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
638  * for 32-bit platforms.
639  */
640 #define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
641  (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
642  (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
643 #define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
644  (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
645  (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
646 
647 /* Raw profile format version (start from 1). */
648 #define INSTR_PROF_RAW_VERSION 5
649 /* Indexed profile format version (start from 1). */
650 #define INSTR_PROF_INDEX_VERSION 7
651 /* Coverage mapping format version (start from 0). */
652 #define INSTR_PROF_COVMAP_VERSION 5
653 
654 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
655  * version for other variants of profile. We set the lowest bit of the upper 8
656  * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
657  * generated profile, and 0 if this is a Clang FE generated profile.
658  * 1 in bit 57 indicates there are context-sensitive records in the profile.
659  */
660 #define VARIANT_MASKS_ALL 0xff00000000000000ULL
661 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
662 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
663 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
664 #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
665 #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
666 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
667 
668 /* The variable that holds the name of the profile data
669  * specified via command line. */
670 #define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
671 
672 /* section name strings common to all targets other
673  than WIN32 */
674 #define INSTR_PROF_DATA_COMMON __llvm_prf_data
675 #define INSTR_PROF_NAME_COMMON __llvm_prf_names
676 #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
677 #define INSTR_PROF_VALS_COMMON __llvm_prf_vals
678 #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
679 #define INSTR_PROF_COVMAP_COMMON __llvm_covmap
680 #define INSTR_PROF_COVFUN_COMMON __llvm_covfun
681 #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
682 /* Windows section names. Because these section names contain dollar characters,
683  * they must be quoted.
684  */
685 #define INSTR_PROF_DATA_COFF ".lprfd$M"
686 #define INSTR_PROF_NAME_COFF ".lprfn$M"
687 #define INSTR_PROF_CNTS_COFF ".lprfc$M"
688 #define INSTR_PROF_VALS_COFF ".lprfv$M"
689 #define INSTR_PROF_VNODES_COFF ".lprfnd$M"
690 #define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
691 #define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
692 #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
693 
694 #ifdef _WIN32
695 /* Runtime section names and name strings. */
696 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
697 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
698 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
699 /* Array of pointers. Each pointer points to a list
700  * of value nodes associated with one value site.
701  */
702 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
703 /* Value profile nodes section. */
704 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
705 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
706 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
707 #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
708 #else
709 /* Runtime section names and name strings. */
710 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
711 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
712 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
713 /* Array of pointers. Each pointer points to a list
714  * of value nodes associated with one value site.
715  */
716 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
717 /* Value profile nodes section. */
718 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
719 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
720 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
721 /* Order file instrumentation. */
722 #define INSTR_PROF_ORDERFILE_SECT_NAME \
723  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
724 #endif
725 
726 #define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
727 #define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
728  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
729 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
730 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
731  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
732 
733 /* Macros to define start/stop section symbol for a given
734  * section on Linux. For instance
735  * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
736  * expand to __start___llvm_prof_data
737  */
738 #define INSTR_PROF_SECT_START(Sect) \
739  INSTR_PROF_CONCAT(__start_,Sect)
740 #define INSTR_PROF_SECT_STOP(Sect) \
741  INSTR_PROF_CONCAT(__stop_,Sect)
742 
743 /* Value Profiling API linkage name. */
744 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
745 #define INSTR_PROF_VALUE_PROF_FUNC_STR \
746  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
747 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
748 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
749  INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
750 
751 /* InstrProfile per-function control data alignment. */
752 #define INSTR_PROF_DATA_ALIGNMENT 8
753 
754 /* The data structure that represents a tracked value by the
755  * value profiler.
756  */
757 typedef struct InstrProfValueData {
758  /* Profiled value. */
759  uint64_t Value;
760  /* Number of times the value appears in the training run. */
761  uint64_t Count;
762 } InstrProfValueData;
763 
764 #endif /* INSTR_PROF_DATA_INC */
765 
766 #ifndef INSTR_ORDER_FILE_INC
767 /* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
768 #define INSTR_ORDER_FILE_BUFFER_SIZE 131072
769 #define INSTR_ORDER_FILE_BUFFER_BITS 17
770 #define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
771 #endif /* INSTR_ORDER_FILE_INC */
772 #else
773 #undef INSTR_PROF_DATA_DEFINED
774 #endif
775 
776 #undef COVMAP_V2_OR_V3
777 
778 #ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
779 
780 #ifdef __cplusplus
781 #define INSTR_PROF_INLINE inline
782 #else
783 #define INSTR_PROF_INLINE
784 #endif
785 
786 /* The value range buckets (22 buckets) for the memop size value profiling looks
787  * like:
788  *
789  * [0, 0]
790  * [1, 1]
791  * [2, 2]
792  * [3, 3]
793  * [4, 4]
794  * [5, 5]
795  * [6, 6]
796  * [7, 7]
797  * [8, 8]
798  * [9, 15]
799  * [16, 16]
800  * [17, 31]
801  * [32, 32]
802  * [33, 63]
803  * [64, 64]
804  * [65, 127]
805  * [128, 128]
806  * [129, 255]
807  * [256, 256]
808  * [257, 511]
809  * [512, 512]
810  * [513, UINT64_MAX]
811  *
812  * Each range has a 'representative value' which is the lower end value of the
813  * range and used to store in the runtime profile data records and the VP
814  * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
815  */
816 #define INSTR_PROF_NUM_BUCKETS 22
817 
818 /*
819  * Clz and Popcount. This code was copied from
820  * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
821  * llvm/include/llvm/Support/MathExtras.h.
822  */
823 #if defined(_MSC_VER) && !defined(__clang__)
824 
825 #include <intrin.h>
826 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
827 int InstProfClzll(unsigned long long X) {
828  unsigned long LeadZeroIdx = 0;
829 #if !defined(_M_ARM64) && !defined(_M_X64)
830  // Scan the high 32 bits.
831  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
832  return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
833  // from the MSB.
834  // Scan the low 32 bits.
835  if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
836  return (int)(63 - LeadZeroIdx);
837 #else
838  if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
839 #endif
840  return 64;
841 }
842 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
843 int InstProfPopcountll(unsigned long long X) {
844  // This code originates from https://reviews.llvm.org/rG30626254510f.
845  unsigned long long v = X;
846  v = v - ((v >> 1) & 0x5555555555555555ULL);
847  v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
848  v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
849  return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
850 }
851 
852 #else
853 
854 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
855 int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
856 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
857 int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
858 
859 #endif /* defined(_MSC_VER) && !defined(__clang__) */
860 
861 /* Map an (observed) memop size value to the representative value of its range.
862  * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
863 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
864 InstrProfGetRangeRepValue(uint64_t Value) {
865  if (Value <= 8)
866  // The first ranges are individually tracked. Use the value as is.
867  return Value;
868  else if (Value >= 513)
869  // The last range is mapped to its lowest value.
870  return 513;
871  else if (InstProfPopcountll(Value) == 1)
872  // If it's a power of two, use it as is.
873  return Value;
874  else
875  // Otherwise, take to the previous power of two + 1.
876  return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
877 }
878 
879 /* Return true if the range that an (observed) memop size value belongs to has
880  * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
881  * false, 64 -> true, 100 -> false, 513 -> false. */
882 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
883 InstrProfIsSingleValRange(uint64_t Value) {
884  if (Value <= 8)
885  // The first ranges are individually tracked.
886  return 1;
887  else if (InstProfPopcountll(Value) == 1)
888  // If it's a power of two, there's only one value.
889  return 1;
890  else
891  // Otherwise, there's more than one value in the range.
892  return 0;
893 }
894 
895 #endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
VALUE_PROF_FUNC_PARAM
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType)
Int32Ty
IntegerType * Int32Ty
Definition: NVVMIntrRange.cpp:67
functions
amdgpu propagate attributes Late propagate attributes from kernels to functions
Definition: AMDGPUPropagateAttributes.cpp:199
INSTR_PROF_DATA
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init)
Definition: InstrProf.h:1123
llvm
Definition: AllocatorList.h:23
llvm::msgpack::Endianness
constexpr support::endianness Endianness
The endianness of all multi-byte encoded values in MessagePack.
Definition: MsgPack.h:24
llvm::lltok::Error
@ Error
Definition: LLToken.h:21
endianness
llvm::support::endianness endianness
Definition: BinaryStreamReader.cpp:16
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:249
llvm::getPGOFuncNameVarInitializer
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:441
offsetof
#define offsetof(TYPE, MEMBER)
Definition: AMDHSAKernelDescriptor.h:23
llvm::IndexedInstrProf::ComputeHash
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:995
llvm::ConstantExpr::getBitCast
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2207
intrinsic
QP Compare Ordered outs ins xscmpudp No intrinsic
Definition: README_P9.txt:303
INSTR_PROF_RAW_HEADER
#define INSTR_PROF_RAW_HEADER(Type, Name, Init)
Definition: InstrProf.h:1132
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:195
llvm::Data
@ Data
Definition: SIMachineScheduler.h:56
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:197
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:133
COVMAP_FUNC_RECORD
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init)
Definition: CoverageMapping.h:917
llvm::AMDGPU::HSAMD::ValueKind
ValueKind
Value kinds.
Definition: AMDGPUMetadata.h:73
llvm::coverage::CurrentVersion
@ CurrentVersion
Definition: CoverageMapping.h:1008
VALUE_PROF_KIND
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
Definition: InstrProf.h:239
COVMAP_HEADER
#define COVMAP_HEADER(Type, LLVMType, Name, Init)
Definition: CoverageMapping.h:970
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:885
llvm::ConstantDataArray::getRaw
static Constant * getRaw(StringRef Data, uint64_t NumElements, Type *ElementTy)
getRaw() constructor - Return a constant with array type with an element count and element type match...
Definition: Constants.h:709
llvm::IndexedInstrProf::Version
const uint64_t Version
Definition: InstrProf.h:991
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
INSTR_PROF_VALUE_NODE
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init)
llvm::COFF::NameSize
@ NameSize
Definition: COFF.h:58
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
__DATA
the multiplication has a latency of four as opposed to two cycles for the movl lea variant It appears gcc place string data with linkonce linkage in section coalesced instead of section __DATA
Definition: README.txt:260
I
#define I(x, y, z)
Definition: MD5.cpp:59
Magic
const char Magic[]
Definition: Archive.cpp:41
size
i< reg-> size
Definition: README.txt:166
llvm::ArrayType::get
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:598
uint32_t
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:198
uint16_t
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
This
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical This
Definition: README.txt:418
llvm::msgpack::Type
Type
MessagePack types as defined in the standard, with the exception of Integer being divided into a sign...
Definition: MsgPackReader.h:49
INSTR_PROF_SECT_ENTRY
#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix)
Definition: InstrProf.h:58