MemProfUse.cpp File Reference

#include "llvm/Transforms/Instrumentation/MemProfUse.h"
#include "InstrumentationOptions.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/MemoryProfileInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/StaticDataProfileInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Module.h"
#include "llvm/ProfileData/DataAccessProf.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/ProfileData/MemProfCommon.h"
#include "llvm/Support/BLAKE3.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/HashBuilder.h"
#include "llvm/Support/MD5.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Transforms/Utils/LongestCommonSequence.h"
#include <map>
#include <set>

Go to the source code of this file.

Classes
struct	AllocMatchInfo
struct	CallSiteEntry

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	DEBUG_TYPE   "memprof"

Functions
	STATISTIC (NumOfMemProfMissing, "Number of functions without memory profile.")
	STATISTIC (NumOfMemProfMismatch, "Number of functions having mismatched memory profile hash.")
	STATISTIC (NumOfMemProfFunc, "Number of functions having valid memory profile.")
	STATISTIC (NumOfMemProfAllocContextProfiles, "Number of alloc contexts in memory profile.")
	STATISTIC (NumOfMemProfCallSiteProfiles, "Number of callsites in memory profile.")
	STATISTIC (NumOfMemProfMatchedAllocContexts, "Number of matched memory profile alloc contexts.")
	STATISTIC (NumOfMemProfMatchedAllocs, "Number of matched memory profile allocs.")
	STATISTIC (NumOfMemProfMatchedCallSites, "Number of matched memory profile callsites.")
	STATISTIC (NumOfMemProfHotGlobalVars, "Number of global vars annotated with 'hot' section prefix.")
	STATISTIC (NumOfMemProfColdGlobalVars, "Number of global vars annotated with 'unlikely' section prefix.")
	STATISTIC (NumOfMemProfUnknownGlobalVars, "Number of global vars with unknown hotness (no section prefix).")
	STATISTIC (NumOfMemProfExplicitSectionGlobalVars, "Number of global vars with user-specified section (not annotated).")
static void	addCallsiteMetadata (Instruction &I, ArrayRef< uint64_t > InlinedCallStack, LLVMContext &Ctx)
static uint64_t	computeStackId (GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column)
static uint64_t	computeStackId (const memprof::Frame &Frame)
static uint32_t	getLineOffset (const DILocation *DIL)
static AllocationType	getAllocType (const AllocationInfo *AllocInfo)
static AllocationType	addCallStack (CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo, uint64_t FullStackId)
static bool	stackFrameIncludesInlinedCallStack (ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack)
static bool	isAllocationWithHotColdVariant (const InstrumentationOptions &Opts, const Function *Callee, const TargetLibraryInfo &TLI)
static void	HandleUnsupportedAnnotationKinds (GlobalVariable &GVar, AnnotationKind Kind)
static std::optional< uint64_t >	getStringContentHash (const GlobalVariable &GVar)
static void	undriftMemProfRecord (const DenseMap< uint64_t, LocToLocMap > &UndriftMaps, memprof::MemProfRecord &MemProfRec)
static void	addVPMetadata (const InstrumentationOptions &Opts, Module &M, Instruction &I, ArrayRef< GlobalValue::GUID > CalleeGuids)
static void	handleAllocSite (const InstrumentationOptions &Opts, Instruction &I, CallBase *CI, ArrayRef< uint64_t > InlinedCallStack, LLVMContext &Ctx, OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize, const std::set< const AllocationInfo * > &AllocInfoSet, std::map< uint64_t, AllocMatchInfo > &FullStackIdToAllocMatchInfo)
static void	handleCallSite (const InstrumentationOptions &Opts, Instruction &I, const Function *CalledFunction, ArrayRef< uint64_t > InlinedCallStack, const std::vector< CallSiteEntry > &CallSiteEntries, Module &M, std::set< std::vector< uint64_t > > &MatchedCallSites, OptimizationRemarkEmitter &ORE)
static void	dumpInlineCallStack (Instruction &I, CallBase *CI, OptimizationRemarkEmitter &ORE, DenseSet< uint64_t > &SeenFrames, DenseSet< uint64_t > &SeenStacks, bool ProfileHasColumns)
static void	readMemprof (const InstrumentationOptions &Opts, Module &M, Function &F, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI, std::map< uint64_t, AllocMatchInfo > &FullStackIdToAllocMatchInfo, std::set< std::vector< uint64_t > > &MatchedCallSites, DenseMap< uint64_t, LocToLocMap > &UndriftMaps, OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize, DenseSet< uint64_t > &SeenStacks, DenseSet< uint64_t > &SeenFrames)

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "memprof"

Definition at line 45 of file MemProfUse.cpp.

Function Documentation

addCallsiteMetadata()

void addCallsiteMetadata ( Instruction & I, ArrayRef< uint64_t > InlinedCallStack, LLVMContext & Ctx )

static

Definition at line 76 of file MemProfUse.cpp.

References llvm::memprof::buildCallstackMetadata(), and I.

Referenced by handleAllocSite(), and handleCallSite().

addCallStack()

AllocationType addCallStack ( CallStackTrie & AllocTrie, const AllocationInfo * AllocInfo, uint64_t FullStackId )

static

Definition at line 115 of file MemProfUse.cpp.

References llvm::memprof::CallStackTrie::addCallStack(), assert(), computeStackId(), llvm::memprof::getAllocType(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::memprof::recordContextSizeInfoForAnalysis(), and uint64_t.

Referenced by handleAllocSite().

addVPMetadata()

void addVPMetadata ( const InstrumentationOptions & Opts, Module & M, Instruction & I, ArrayRef< GlobalValue::GUID > CalleeGuids )

static

Definition at line 374 of file MemProfUse.cpp.

References llvm::annotateValueSite(), assert(), llvm::ArrayRef< T >::empty(), llvm::from_range, G, llvm::getValueProfDataFromInst(), I, llvm::map_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::reserve(), llvm::ArrayRef< T >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and uint64_t.

Referenced by handleCallSite().

computeStackId() [1/2]

uint64_t computeStackId ( const memprof::Frame & Frame )

static

Definition at line 94 of file MemProfUse.cpp.

References llvm::memprof::Frame::Column, computeStackId(), llvm::memprof::Frame::Function, llvm::memprof::Frame::LineOffset, and uint64_t.

computeStackId() [2/2]

uint64_t computeStackId ( GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column )

static

Definition at line 83 of file MemProfUse.cpp.

References llvm::HashBuilder< HasherT, Endianness >::add(), llvm::HashBuilderBase< HasherT >::final(), llvm::little, and uint64_t.

Referenced by addCallStack(), computeStackId(), dumpInlineCallStack(), readMemprof(), and llvm::MemProfUsePass::run().

dumpInlineCallStack()

void dumpInlineCallStack ( Instruction & I, CallBase * CI, OptimizationRemarkEmitter & ORE, DenseSet< uint64_t > & SeenFrames, DenseSet< uint64_t > & SeenStacks, bool ProfileHasColumns )

static

Definition at line 601 of file MemProfUse.cpp.

References computeStackId(), DEBUG_TYPE, llvm::OptimizationRemarkEmitter::emit(), llvm::First, llvm::GlobalValue::getGUIDAssumingExternalLinkage(), getLineOffset(), I, llvm::detail::DenseSetImpl< ValueT, MapTy >::insert(), llvm::sampleprof::MD5Hash(), Msg, llvm::raw_string_ostream::str(), and uint64_t.

Referenced by readMemprof().

getAllocType()

AllocationType getAllocType ( const AllocationInfo * AllocInfo )

static

Definition at line 109 of file MemProfUse.cpp.

References llvm::memprof::getAllocType().

getLineOffset()

uint32_t getLineOffset ( const DILocation * DIL )

static

Definition at line 101 of file MemProfUse.cpp.

Referenced by dumpInlineCallStack(), llvm::memprof::extractCallsFromIR(), and readMemprof().

getStringContentHash()

std::optional< uint64_t > getStringContentHash ( const GlobalVariable & GVar )

static

Definition at line 208 of file MemProfUse.cpp.

References C(), llvm::dyn_cast(), llvm::GlobalVariable::getInitializer(), and llvm::sampleprof::MD5Hash().

handleAllocSite()

void handleAllocSite ( const InstrumentationOptions & Opts, Instruction & I, CallBase * CI, ArrayRef< uint64_t > InlinedCallStack, LLVMContext & Ctx, OptimizationRemarkEmitter & ORE, uint64_t MaxColdSize, const std::set< const AllocationInfo * > & AllocInfoSet, std::map< uint64_t, AllocMatchInfo > & FullStackIdToAllocMatchInfo )

static

Definition at line 433 of file MemProfUse.cpp.

References addCallsiteMetadata(), addCallStack(), llvm::memprof::CallStackTrie::addSingleAllocTypeAttribute(), assert(), llvm::memprof::CallStackTrie::buildAndAttachMIBMetadata(), llvm::Cold, llvm::memprof::computeFullStackId(), DEBUG_TYPE, llvm::OptimizationRemarkEmitter::emit(), llvm::memprof::CallStackTrie::empty(), llvm::memprof::getAllocType(), llvm::memprof::getAllocTypeAttributeString(), llvm::Instruction::getFunction(), I, llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::insert(), llvm::NotCold, llvm::ArrayRef< T >::size(), stackFrameIncludesInlinedCallStack(), and uint64_t.

Referenced by readMemprof().

handleCallSite()

void handleCallSite ( const InstrumentationOptions & Opts, Instruction & I, const Function * CalledFunction, ArrayRef< uint64_t > InlinedCallStack, const std::vector< CallSiteEntry > & CallSiteEntries, Module & M, std::set< std::vector< uint64_t > > & MatchedCallSites, OptimizationRemarkEmitter & ORE )

static

Definition at line 542 of file MemProfUse.cpp.

References addCallsiteMetadata(), addVPMetadata(), llvm::append_range(), assert(), llvm::ArrayRef< T >::begin(), CallSiteEntry::CalleeGuids, DEBUG_TYPE, llvm::OptimizationRemarkEmitter::emit(), llvm::ArrayRef< T >::end(), CallSiteEntry::Frames, llvm::SetVector< T, Vector, Set, N >::getArrayRef(), I, llvm::SetVector< T, Vector, Set, N >::insert(), llvm::Remark, llvm::ArrayRef< T >::size(), stackFrameIncludesInlinedCallStack(), and uint64_t.

Referenced by readMemprof().

HandleUnsupportedAnnotationKinds()

void HandleUnsupportedAnnotationKinds ( GlobalVariable & GVar, AnnotationKind Kind )

static

Definition at line 182 of file MemProfUse.cpp.

References llvm::memprof::AnnotationOK, llvm::SmallString< InternalLen >::append(), assert(), llvm::dbgs(), llvm::memprof::DeclForLinker, llvm::memprof::ExplicitSection, llvm::Value::getName(), LLVM_DEBUG, llvm_unreachable, and llvm::memprof::ReservedName.

isAllocationWithHotColdVariant()

bool isAllocationWithHotColdVariant ( const InstrumentationOptions & Opts, const Function * Callee, const TargetLibraryInfo & TLI )

static

Definition at line 146 of file MemProfUse.cpp.

References llvm::TargetLibraryInfo::getLibFunc().

Referenced by llvm::memprof::extractCallsFromIR(), and readMemprof().

readMemprof()

void readMemprof ( const InstrumentationOptions & Opts, Module & M, Function & F, IndexedInstrProfReader * MemProfReader, const TargetLibraryInfo & TLI, std::map< uint64_t, AllocMatchInfo > & FullStackIdToAllocMatchInfo, std::set< std::vector< uint64_t > > & MatchedCallSites, DenseMap< uint64_t, LocToLocMap > & UndriftMaps, OptimizationRemarkEmitter & ORE, uint64_t MaxColdSize, DenseSet< uint64_t > & SeenStacks, DenseSet< uint64_t > & SeenFrames )

static

Definition at line 646 of file MemProfUse.cpp.

References llvm::OptimizationRemarkEmitter::allowExtraAnalysis(), llvm::ArrayRef(), assert(), llvm::GlobalValue::AvailableExternallyLinkage, computeStackId(), llvm::dbgs(), DEBUG_TYPE, llvm::ArrayRef< T >::drop_front(), llvm::DS_Warning, dumpInlineCallStack(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::errs(), F, llvm::InstrProfError::get(), llvm::GlobalValue::getGUIDAssumingExternalLinkage(), getLineOffset(), llvm::handleAllErrors(), handleAllocSite(), handleCallSite(), llvm::hash_mismatch, I, isAllocationWithHotColdVariant(), LLVM_DEBUG, llvm::InstrProfError::message(), Msg, llvm::NoPGOWarnMismatch, llvm::SmallVectorTemplateBase< T, bool >::push_back(), uint64_t, undriftMemProfRecord(), and llvm::unknown_function.

Referenced by llvm::MemProfUsePass::run().

stackFrameIncludesInlinedCallStack()

bool stackFrameIncludesInlinedCallStack ( ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack )

static

Definition at line 137 of file MemProfUse.cpp.

References llvm::equal(), F, llvm::ArrayRef< T >::size(), llvm::ArrayRef< T >::take_front(), and uint64_t.

Referenced by handleAllocSite(), and handleCallSite().

STATISTIC() [1/12]

STATISTIC	(	NumOfMemProfAllocContextProfiles	,
		"Number of alloc contexts in memory profile."	)

STATISTIC() [2/12]

STATISTIC	(	NumOfMemProfCallSiteProfiles	,
		"Number of callsites in memory profile."	)

STATISTIC() [3/12]

STATISTIC	(	NumOfMemProfColdGlobalVars	,
		"Number of global vars annotated with 'unlikely' section prefix."	)

STATISTIC() [4/12]

STATISTIC	(	NumOfMemProfExplicitSectionGlobalVars	,
		"Number of global vars with user-specified section (not annotated)."	)

STATISTIC() [5/12]

STATISTIC	(	NumOfMemProfFunc	,
		"Number of functions having valid memory profile."	)

STATISTIC() [6/12]

STATISTIC	(	NumOfMemProfHotGlobalVars	,
		"Number of global vars annotated with 'hot' section prefix."	)

STATISTIC() [7/12]

STATISTIC	(	NumOfMemProfMatchedAllocContexts	,
		"Number of matched memory profile alloc contexts."	)

STATISTIC() [8/12]

STATISTIC	(	NumOfMemProfMatchedAllocs	,
		"Number of matched memory profile allocs."	)

STATISTIC() [9/12]

STATISTIC	(	NumOfMemProfMatchedCallSites	,
		"Number of matched memory profile callsites."	)

STATISTIC() [10/12]

STATISTIC	(	NumOfMemProfMismatch	,
		"Number of functions having mismatched memory profile hash."	)

STATISTIC() [11/12]

STATISTIC	(	NumOfMemProfMissing	,
		"Number of functions without memory profile."	)

STATISTIC() [12/12]

STATISTIC	(	NumOfMemProfUnknownGlobalVars	,
		"Number of global vars with unknown hotness (no section prefix)."	)

undriftMemProfRecord()

void undriftMemProfRecord ( const DenseMap< uint64_t, LocToLocMap > & UndriftMaps, memprof::MemProfRecord & MemProfRec )

static

Definition at line 349 of file MemProfUse.cpp.

References llvm::memprof::MemProfRecord::AllocSites, llvm::memprof::MemProfRecord::CallSites, llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::end(), F, llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::find(), and I.

Referenced by readMemprof().