LLVM 20.0.0git
Classes | Namespaces | Macros | Functions | Variables
MemProfiler.cpp File Reference
#include "llvm/Transforms/Instrumentation/MemProfiler.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryProfileInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/Support/BLAKE3.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/HashBuilder.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/TargetParser/Triple.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <map>
#include <set>

Go to the source code of this file.

Classes

struct  AllocMatchInfo
 

Namespaces

namespace  llvm
 This is an optimization pass for GlobalISel generic memory operations.
 

Macros

#define DEBUG_TYPE   "memprof"
 

Functions

 STATISTIC (NumInstrumentedReads, "Number of instrumented reads")
 
 STATISTIC (NumInstrumentedWrites, "Number of instrumented writes")
 
 STATISTIC (NumSkippedStackReads, "Number of non-instrumented stack reads")
 
 STATISTIC (NumSkippedStackWrites, "Number of non-instrumented stack writes")
 
 STATISTIC (NumOfMemProfMissing, "Number of functions without memory profile.")
 
 STATISTIC (NumOfMemProfMismatch, "Number of functions having mismatched memory profile hash.")
 
 STATISTIC (NumOfMemProfFunc, "Number of functions having valid memory profile.")
 
 STATISTIC (NumOfMemProfAllocContextProfiles, "Number of alloc contexts in memory profile.")
 
 STATISTIC (NumOfMemProfCallSiteProfiles, "Number of callsites in memory profile.")
 
 STATISTIC (NumOfMemProfMatchedAllocContexts, "Number of matched memory profile alloc contexts.")
 
 STATISTIC (NumOfMemProfMatchedAllocs, "Number of matched memory profile allocs.")
 
 STATISTIC (NumOfMemProfMatchedCallSites, "Number of matched memory profile callsites.")
 
void createProfileFileNameVar (Module &M)
 
void createMemprofHistogramFlagVar (Module &M)
 
static void addCallsiteMetadata (Instruction &I, std::vector< uint64_t > &InlinedCallStack, LLVMContext &Ctx)
 
static uint64_t computeStackId (GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column)
 
static uint64_t computeStackId (const memprof::Frame &Frame)
 
static uint64_t computeFullStackId (const std::vector< memprof::Frame > &CallStack)
 
static AllocationType addCallStack (CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo)
 
static bool stackFrameIncludesInlinedCallStack (ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack, unsigned StartIndex=0)
 
static bool isAllocationWithHotColdVariant (Function *Callee, const TargetLibraryInfo &TLI)
 
static void readMemprof (Module &M, Function &F, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI, std::map< uint64_t, AllocMatchInfo > &FullStackIdToAllocMatchInfo)
 

Variables

cl::opt< boolllvm::PGOWarnMissing
 
cl::opt< boolllvm::NoPGOWarnMismatchComdatWeak
 
constexpr int LLVM_MEM_PROFILER_VERSION = 1
 
constexpr uint64_t DefaultMemGranularity = 64
 
constexpr uint64_t HistogramGranularity = 8
 
constexpr uint64_t DefaultShadowScale = 3
 
constexpr char MemProfModuleCtorName [] = "memprof.module_ctor"
 
constexpr uint64_t MemProfCtorAndDtorPriority = 1
 
constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50
 
constexpr char MemProfInitName [] = "__memprof_init"
 
constexpr char MemProfVersionCheckNamePrefix []
 
constexpr char MemProfShadowMemoryDynamicAddress []
 
constexpr char MemProfFilenameVar [] = "__memprof_profile_filename"
 
constexpr char MemProfHistogramFlagVar [] = "__memprof_histogram"
 
static cl::opt< boolClInsertVersionCheck ("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
 
static cl::opt< boolClInstrumentReads ("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
 
static cl::opt< boolClInstrumentWrites ("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
 
static cl::opt< boolClInstrumentAtomics ("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
 
static cl::opt< boolClUseCalls ("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false))
 
static cl::opt< std::string > ClMemoryAccessCallbackPrefix ("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_"))
 
static cl::opt< int > ClMappingScale ("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale))
 
static cl::opt< int > ClMappingGranularity ("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultMemGranularity))
 
static cl::opt< boolClStack ("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false))
 
static cl::opt< int > ClDebug ("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0))
 
static cl::opt< std::string > ClDebugFunc ("memprof-debug-func", cl::Hidden, cl::desc("Debug func"))
 
static cl::opt< int > ClDebugMin ("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
 
static cl::opt< int > ClDebugMax ("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
 
static cl::opt< boolClMemProfMatchHotColdNew ("memprof-match-hot-cold-new", cl::desc("Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false))
 
static cl::opt< boolClHistogram ("memprof-histogram", cl::desc("Collect access count histograms"), cl::Hidden, cl::init(false))
 
static cl::opt< boolClPrintMemProfMatchInfo ("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false))
 
cl::opt< boolMemProfReportHintedSizes
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "memprof"

Definition at line 52 of file MemProfiler.cpp.

Function Documentation

◆ addCallsiteMetadata()

static void addCallsiteMetadata ( Instruction I,
std::vector< uint64_t > &  InlinedCallStack,
LLVMContext Ctx 
)
static

Definition at line 682 of file MemProfiler.cpp.

References llvm::memprof::buildCallstackMetadata(), and I.

Referenced by readMemprof().

◆ addCallStack()

static AllocationType addCallStack ( CallStackTrie AllocTrie,
const AllocationInfo AllocInfo 
)
static

◆ computeFullStackId()

static uint64_t computeFullStackId ( const std::vector< memprof::Frame > &  CallStack)
static

◆ computeStackId() [1/2]

static uint64_t computeStackId ( const memprof::Frame Frame)
static

◆ computeStackId() [2/2]

static uint64_t computeStackId ( GlobalValue::GUID  Function,
uint32_t  LineOffset,
uint32_t  Column 
)
static

◆ createMemprofHistogramFlagVar()

void createMemprofHistogramFlagVar ( Module M)

◆ createProfileFileNameVar()

void createProfileFileNameVar ( Module M)

◆ isAllocationWithHotColdVariant()

static bool isAllocationWithHotColdVariant ( Function Callee,
const TargetLibraryInfo TLI 
)
static

Definition at line 757 of file MemProfiler.cpp.

References ClMemProfMatchHotColdNew, and llvm::TargetLibraryInfo::getLibFunc().

Referenced by readMemprof().

◆ readMemprof()

static void readMemprof ( Module M,
Function F,
IndexedInstrProfReader MemProfReader,
const TargetLibraryInfo TLI,
std::map< uint64_t, AllocMatchInfo > &  FullStackIdToAllocMatchInfo 
)
static

◆ stackFrameIncludesInlinedCallStack()

static bool stackFrameIncludesInlinedCallStack ( ArrayRef< Frame ProfileCallStack,
ArrayRef< uint64_t InlinedCallStack,
unsigned  StartIndex = 0 
)
static

Definition at line 740 of file MemProfiler.cpp.

References llvm::ArrayRef< T >::begin(), computeStackId(), and llvm::ArrayRef< T >::end().

Referenced by readMemprof().

◆ STATISTIC() [1/12]

STATISTIC ( NumInstrumentedReads  ,
"Number of instrumented reads"   
)

◆ STATISTIC() [2/12]

STATISTIC ( NumInstrumentedWrites  ,
"Number of instrumented writes"   
)

◆ STATISTIC() [3/12]

STATISTIC ( NumOfMemProfAllocContextProfiles  ,
"Number of alloc contexts in memory profile."   
)

◆ STATISTIC() [4/12]

STATISTIC ( NumOfMemProfCallSiteProfiles  ,
"Number of callsites in memory profile."   
)

◆ STATISTIC() [5/12]

STATISTIC ( NumOfMemProfFunc  ,
"Number of functions having valid memory profile."   
)

◆ STATISTIC() [6/12]

STATISTIC ( NumOfMemProfMatchedAllocContexts  ,
"Number of matched memory profile alloc contexts."   
)

◆ STATISTIC() [7/12]

STATISTIC ( NumOfMemProfMatchedAllocs  ,
"Number of matched memory profile allocs."   
)

◆ STATISTIC() [8/12]

STATISTIC ( NumOfMemProfMatchedCallSites  ,
"Number of matched memory profile callsites."   
)

◆ STATISTIC() [9/12]

STATISTIC ( NumOfMemProfMismatch  ,
"Number of functions having mismatched memory profile hash."   
)

◆ STATISTIC() [10/12]

STATISTIC ( NumOfMemProfMissing  ,
"Number of functions without memory profile."   
)

◆ STATISTIC() [11/12]

STATISTIC ( NumSkippedStackReads  ,
"Number of non-instrumented stack reads"   
)

◆ STATISTIC() [12/12]

STATISTIC ( NumSkippedStackWrites  ,
"Number of non-instrumented stack writes"   
)

Variable Documentation

◆ ClDebug

cl::opt< int > ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0)) ( "memprof-debug"  ,
cl::desc("debug")  ,
cl::Hidden  ,
cl::init(0)   
)
static

◆ ClDebugFunc

cl::opt< std::string > ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func")) ( "memprof-debug-func"  ,
cl::Hidden  ,
cl::desc("Debug func")   
)
static

◆ ClDebugMax

cl::opt< int > ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1)) ( "memprof-debug-max"  ,
cl::desc("Debug max inst")  ,
cl::Hidden  ,
cl::init(-1)   
)
static

◆ ClDebugMin

cl::opt< int > ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1)) ( "memprof-debug-min"  ,
cl::desc("Debug min inst")  ,
cl::Hidden  ,
cl::init(-1)   
)
static

◆ ClHistogram

cl::opt< bool > ClHistogram("memprof-histogram", cl::desc("Collect access count histograms"), cl::Hidden, cl::init(false)) ( "memprof-histogram"  ,
cl::desc("Collect access count histograms")  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ ClInsertVersionCheck

cl::opt< bool > ClInsertVersionCheck("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true)) ( "memprof-guard-against-version-mismatch"  ,
cl::desc("Guard against compiler/runtime version mismatch.")  ,
cl::Hidden  ,
cl::init(true  
)
static

◆ ClInstrumentAtomics

cl::opt< bool > ClInstrumentAtomics("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true)) ( "memprof-instrument-atomics"  ,
cl::desc("instrument atomic instructions (rmw, cmpxchg)")  ,
cl::Hidden  ,
cl::init(true  
)
static

◆ ClInstrumentReads

cl::opt< bool > ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true)) ( "memprof-instrument-reads"  ,
cl::desc("instrument read instructions")  ,
cl::Hidden  ,
cl::init(true  
)
static

◆ ClInstrumentWrites

cl::opt< bool > ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true)) ( "memprof-instrument-writes"  ,
cl::desc("instrument write instructions")  ,
cl::Hidden  ,
cl::init(true  
)
static

◆ ClMappingGranularity

cl::opt< int > ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultMemGranularity)) ( "memprof-mapping-granularity"  ,
cl::desc("granularity of memprof shadow mapping")  ,
cl::Hidden  ,
cl::init(DefaultMemGranularity  
)
static

◆ ClMappingScale

cl::opt< int > ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale)) ( "memprof-mapping-scale"  ,
cl::desc("scale of memprof shadow mapping")  ,
cl::Hidden  ,
cl::init(DefaultShadowScale  
)
static

◆ ClMemoryAccessCallbackPrefix

cl::opt< std::string > ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_")) ( "memprof-memory-access-callback-prefix"  ,
cl::desc("Prefix for memory access callbacks")  ,
cl::Hidden  ,
cl::init("__memprof_")   
)
static

◆ ClMemProfMatchHotColdNew

cl::opt< bool > ClMemProfMatchHotColdNew("memprof-match-hot-cold-new", cl::desc( "Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false)) ( "memprof-match-hot-cold-new"  ,
cl::desc( "Match allocation profiles onto existing hot/cold operator new calls")  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ ClPrintMemProfMatchInfo

cl::opt< bool > ClPrintMemProfMatchInfo("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false)) ( "memprof-print-match-info"  ,
cl::desc("Print matching stats for each allocation " "context in this module's profiles")  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ ClStack

cl::opt< bool > ClStack("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false)) ( "memprof-instrument-stack"  ,
cl::desc("Instrument scalar stack variables")  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ ClUseCalls

cl::opt< bool > ClUseCalls("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false)) ( "memprof-use-callbacks"  ,
cl::desc("Use callbacks instead of inline instrumentation sequences.")  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ DefaultMemGranularity

constexpr uint64_t DefaultMemGranularity = 64
constexpr

Definition at line 63 of file MemProfiler.cpp.

Referenced by llvm::MemProfilerPass::run().

◆ DefaultShadowScale

constexpr uint64_t DefaultShadowScale = 3
constexpr

Definition at line 69 of file MemProfiler.cpp.

◆ HistogramGranularity

constexpr uint64_t HistogramGranularity = 8
constexpr

Definition at line 66 of file MemProfiler.cpp.

◆ LLVM_MEM_PROFILER_VERSION

constexpr int LLVM_MEM_PROFILER_VERSION = 1
constexpr

Definition at line 60 of file MemProfiler.cpp.

◆ MemProfCtorAndDtorPriority

constexpr uint64_t MemProfCtorAndDtorPriority = 1
constexpr

Definition at line 72 of file MemProfiler.cpp.

◆ MemProfEmscriptenCtorAndDtorPriority

constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50
constexpr

Definition at line 74 of file MemProfiler.cpp.

◆ MemProfFilenameVar

constexpr char MemProfFilenameVar[] = "__memprof_profile_filename"
constexpr

Definition at line 82 of file MemProfiler.cpp.

Referenced by createProfileFileNameVar().

◆ MemProfHistogramFlagVar

constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram"
constexpr

Definition at line 84 of file MemProfiler.cpp.

Referenced by createMemprofHistogramFlagVar().

◆ MemProfInitName

constexpr char MemProfInitName[] = "__memprof_init"
constexpr

Definition at line 75 of file MemProfiler.cpp.

◆ MemProfModuleCtorName

constexpr char MemProfModuleCtorName[] = "memprof.module_ctor"
constexpr

Definition at line 71 of file MemProfiler.cpp.

◆ MemProfReportHintedSizes

cl::opt<bool> MemProfReportHintedSizes
extern

◆ MemProfShadowMemoryDynamicAddress

constexpr char MemProfShadowMemoryDynamicAddress[]
constexpr
Initial value:
=
"__memprof_shadow_memory_dynamic_address"

Definition at line 79 of file MemProfiler.cpp.

◆ MemProfVersionCheckNamePrefix

constexpr char MemProfVersionCheckNamePrefix[]
constexpr
Initial value:
=
"__memprof_version_mismatch_check_v"

Definition at line 76 of file MemProfiler.cpp.