61#ifndef INSTR_PROF_VISIBILITY
62#define INSTR_PROF_VISIBILITY
67#ifndef INSTR_PROF_DATA
68#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
70#define INSTR_PROF_DATA_DEFINED
76 ConstantInt::get(
llvm::
Type::getInt64Ty(Ctx), \
77 Inc->getHash()->getZExtValue()))
89 ConstantInt::get(
llvm::
Type::getInt32Ty(Ctx), NumCounters))
91 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \
93 ConstantInt::get(
llvm::
Type::getInt32Ty(Ctx), NumBitmapBytes))
110#ifndef INSTR_PROF_VALUE_NODE
111#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
113#define INSTR_PROF_DATA_DEFINED
116 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
118 ConstantInt::get(
llvm::
Type::GetInt64Ty(Ctx), 0))
120 ConstantInt::get(
llvm::
Type::GetInt8PtrTy(Ctx), 0))
121#undef INSTR_PROF_VALUE_NODE
126#ifndef INSTR_PROF_RAW_HEADER
127#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
129#define INSTR_PROF_DATA_DEFINED
142 (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
144 (uintptr_t)BitmapBegin - (uintptr_t)DataBegin)
147#undef INSTR_PROF_RAW_HEADER
154#ifndef VALUE_PROF_FUNC_PARAM
155#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
156#define INSTR_PROF_COMMA
158#define INSTR_PROF_DATA_DEFINED
159#define INSTR_PROF_COMMA ,
165#undef VALUE_PROF_FUNC_PARAM
166#undef INSTR_PROF_COMMA
170#ifndef VALUE_PROF_KIND
171#define VALUE_PROF_KIND(Enumerator, Value, Descr)
173#define INSTR_PROF_DATA_DEFINED
195#undef VALUE_PROF_KIND
198#undef COVMAP_V2_OR_V3
200#define COVMAP_V2_OR_V3
203#define COVMAP_V2_OR_V3
210#ifndef COVMAP_FUNC_RECORD
211#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
213#define INSTR_PROF_DATA_DEFINED
223#ifdef COVMAP_V2_OR_V3
232 llvm::ConstantInt::get( \
233 llvm::
Type::getInt64Ty(Ctx), FuncHash))
240 CoverageMapping.size()), \
243 CoverageMapping, CoverageMapping.size(), \
246#undef COVMAP_FUNC_RECORD
253#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
255#define INSTR_PROF_DATA_DEFINED
269#ifdef INSTR_PROF_SECT_ENTRY
270#define INSTR_PROF_DATA_DEFINED
272 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
273 INSTR_PROF_DATA_COFF,
"__DATA,")
275 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
276 INSTR_PROF_CNTS_COFF, "__DATA,")
278 INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \
279 INSTR_PROF_BITS_COFF, "__DATA,")
281 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
282 INSTR_PROF_NAME_COFF, "__DATA,")
284 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
285 INSTR_PROF_VALS_COFF, "__DATA,")
287 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
288 INSTR_PROF_VNODES_COFF, "__DATA,")
290 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
291 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
293 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
294 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
296 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
297 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
299#undef INSTR_PROF_SECT_ENTRY
303#ifdef INSTR_PROF_VALUE_PROF_DATA
304#define INSTR_PROF_DATA_DEFINED
306#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
311typedef struct ValueProfRecord {
325 uint8_t SiteCountArray[1];
342 uint32_t getNumValueSites()
const {
return NumValueSites; }
346 void deserializeTo(InstrProfRecord &Record,
347 InstrProfSymtab *SymTab);
361typedef struct ValueProfData {
394 static std::unique_ptr<ValueProfData>
395 serializeFrom(
const InstrProfRecord &Record);
399 Error checkIntegrity();
405 static Expected<std::unique_ptr<ValueProfData>>
406 getValueProfData(
const unsigned char *SrcBuffer,
407 const unsigned char *
const SrcBufferEnd,
424 void deserializeTo(InstrProfRecord &Record,
425 InstrProfSymtab *SymTab);
426 void operator delete(
void *ptr) { ::operator
delete(ptr); }
444typedef struct ValueProfRecordClosure {
446 uint32_t (*GetNumValueKinds)(
const void *Record);
457 void (*GetValueForSite)(
const void *
R, InstrProfValueData *Dst,
uint32_t K,
459 ValueProfData *(*AllocValueProfData)(
size_t TotalSizeInBytes);
460} ValueProfRecordClosure;
462INSTR_PROF_VISIBILITY ValueProfRecord *
463getFirstValueProfRecord(ValueProfData *VPD);
464INSTR_PROF_VISIBILITY ValueProfRecord *
465getValueProfRecordNext(ValueProfRecord *VPR);
466INSTR_PROF_VISIBILITY InstrProfValueData *
467getValueProfRecordValueData(ValueProfRecord *VPR);
469getValueProfRecordHeaderSize(
uint32_t NumValueSites);
471#undef INSTR_PROF_VALUE_PROF_DATA
475#ifdef INSTR_PROF_COMMON_API_IMPL
476#define INSTR_PROF_DATA_DEFINED
478#define INSTR_PROF_INLINE inline
479#define INSTR_PROF_NULLPTR nullptr
481#define INSTR_PROF_INLINE
482#define INSTR_PROF_NULLPTR NULL
486#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
493INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
496 sizeof(uint8_t) * NumValueSites;
506INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
509 return getValueProfRecordHeaderSize(NumValueSites) +
510 sizeof(InstrProfValueData) * NumValueData;
516INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
517InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
518 return (InstrProfValueData *)((
char *)This + getValueProfRecordHeaderSize(
519 This->NumValueSites));
525INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
526uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
529 for (
I = 0;
I <
This->NumValueSites;
I++)
530 NumValueData +=
This->SiteCountArray[
I];
537INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
538ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
539 uint32_t NumValueData = getValueProfRecordNumValueData(This);
540 return (ValueProfRecord *)((
char *)This +
541 getValueProfRecordSize(
This->NumValueSites,
548INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
549ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
550 return (ValueProfRecord *)((
char *)This +
sizeof(ValueProfData));
560getValueProfDataSize(ValueProfRecordClosure *Closure) {
562 uint32_t TotalSize =
sizeof(ValueProfData);
563 const void *Record = Closure->Record;
565 for (Kind = IPVK_First;
Kind <= IPVK_Last;
Kind++) {
566 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
569 TotalSize += getValueProfRecordSize(NumValueSites,
570 Closure->GetNumValueData(Record, Kind));
579INSTR_PROF_VISIBILITY
void
580serializeValueProfRecordFrom(ValueProfRecord *This,
581 ValueProfRecordClosure *Closure,
584 const void *Record = Closure->Record;
586 This->NumValueSites = NumValueSites;
587 InstrProfValueData *DstVD = getValueProfRecordValueData(This);
589 for (S = 0; S < NumValueSites; S++) {
590 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
591 This->SiteCountArray[S] = ND;
592 Closure->GetValueForSite(Record, DstVD, ValueKind, S);
604INSTR_PROF_VISIBILITY ValueProfData *
605serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
606 ValueProfData *DstData) {
609 DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
612 DstData ? DstData : Closure->AllocValueProfData(TotalSize);
614 VPD->TotalSize = TotalSize;
615 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
616 ValueProfRecord *VR = getFirstValueProfRecord(VPD);
617 for (Kind = IPVK_First;
Kind <= IPVK_Last;
Kind++) {
618 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
621 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
622 VR = getValueProfRecordNext(VR);
627#undef INSTR_PROF_COMMON_API_IMPL
632#ifndef INSTR_PROF_DATA_DEFINED
634#ifndef INSTR_PROF_DATA_INC
635#define INSTR_PROF_DATA_INC
638#define INSTR_PROF_SIMPLE_QUOTE(x) #x
639#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
640#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
641#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
650#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
651 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
652 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
653#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
654 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
655 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
658#define INSTR_PROF_RAW_VERSION 9
660#define INSTR_PROF_INDEX_VERSION 11
662#define INSTR_PROF_COVMAP_VERSION 6
675#define VARIANT_MASKS_ALL 0xffffffff00000000ULL
676#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
677#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
678#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
679#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
680#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
681#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
682#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
683#define VARIANT_MASK_MEMPROF (0x1ULL << 62)
684#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63)
685#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
686#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
687#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
688#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
692#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
696#define INSTR_PROF_DATA_COMMON __llvm_prf_data
697#define INSTR_PROF_NAME_COMMON __llvm_prf_names
698#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
699#define INSTR_PROF_BITS_COMMON __llvm_prf_bits
700#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
701#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
702#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
703#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
704#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
708#define INSTR_PROF_DATA_COFF ".lprfd$M"
709#define INSTR_PROF_NAME_COFF ".lprfn$M"
710#define INSTR_PROF_CNTS_COFF ".lprfc$M"
711#define INSTR_PROF_BITS_COFF ".lprfb$M"
712#define INSTR_PROF_VALS_COFF ".lprfv$M"
713#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
714#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
715#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
716#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
720#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
721#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
722#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
723#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF
727#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
729#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
730#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
731#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
732#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
735#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
736#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
737#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
738#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON)
742#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
744#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
745#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
746#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
748#define INSTR_PROF_ORDERFILE_SECT_NAME \
749 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
752#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
753#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
754 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
755#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
756#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
757 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
764#define INSTR_PROF_SECT_START(Sect) \
765 INSTR_PROF_CONCAT(__start_,Sect)
766#define INSTR_PROF_SECT_STOP(Sect) \
767 INSTR_PROF_CONCAT(__stop_,Sect)
770#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
771#define INSTR_PROF_VALUE_PROF_FUNC_STR \
772 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
773#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
774#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
775 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
778#define INSTR_PROF_DATA_ALIGNMENT 8
783typedef struct InstrProfValueData {
792#ifndef INSTR_ORDER_FILE_INC
794#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
795#define INSTR_ORDER_FILE_BUFFER_BITS 17
796#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
799#undef INSTR_PROF_DATA_DEFINED
802#undef COVMAP_V2_OR_V3
804#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
807#define INSTR_PROF_INLINE inline
809#define INSTR_PROF_INLINE
842#define INSTR_PROF_NUM_BUCKETS 22
849#if defined(_MSC_VER) && !defined(__clang__)
852INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
853int InstProfClzll(
unsigned long long X) {
854 unsigned long LeadZeroIdx = 0;
855#if !defined(_M_ARM64) && !defined(_M_X64)
857 if (_BitScanReverse(&LeadZeroIdx, (
unsigned long)(
X >> 32)))
858 return (
int)(63 - (LeadZeroIdx + 32));
861 if (_BitScanReverse(&LeadZeroIdx, (
unsigned long)(
X)))
862 return (
int)(63 - LeadZeroIdx);
864 if (_BitScanReverse64(&LeadZeroIdx,
X))
return 63 - LeadZeroIdx;
868INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
869int InstProfPopcountll(
unsigned long long X) {
871 unsigned long long v =
X;
872 v =
v - ((
v >> 1) & 0x5555555555555555ULL);
873 v = (
v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
874 v = (
v + (
v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
875 return (
int)((
unsigned long long)(v * 0x0101010101010101ULL) >> 56);
880INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
881int InstProfClzll(
unsigned long long X) {
return __builtin_clzll(
X); }
882INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
883int InstProfPopcountll(
unsigned long long X) {
return __builtin_popcountll(
X); }
889INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
uint64_t
890InstrProfGetRangeRepValue(
uint64_t Value) {
894 else if (Value >= 513)
897 else if (InstProfPopcountll(Value) == 1)
902 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
908INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
unsigned
909InstrProfIsSingleValRange(
uint64_t Value) {
913 else if (InstProfPopcountll(Value) == 1)
Lower uses of LDS variables from non kernel functions
#define offsetof(TYPE, MEMBER)
#define COVMAP_HEADER(Type, LLVMType, Name, Init)
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
#define INSTR_PROF_RAW_HEADER(Type, Name, Init)
#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix)
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init)
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init)
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType)
static unsigned getSize(unsigned Kind)
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
static Constant * getRaw(StringRef Data, uint64_t NumElements, Type *ElementTy)
getRaw() constructor - Return a constant with array type with an element count and element type match...
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
std::enable_if_t< std::is_arithmetic< T >::value, void > swapBytes(T &value)
This is an optimization pass for GlobalISel generic memory operations.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.