#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
#include "ValueProfileCollector.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Comdat.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/EHPersonalities.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/ProfileSummary.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/CRC.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/TargetParser/Triple.h"
#include "llvm/Transforms/Instrumentation/BlockCoverageInference.h"
#include "llvm/Transforms/Instrumentation/CFGMST.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Instrumentation.h"
#include "llvm/Transforms/Utils/MisExpect.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <memory>
#include <numeric>
#include <optional>
#include <stack>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include "llvm/ProfileData/InstrProfData.inc"

Classes
struct	llvm::GraphTraits< PGOUseFunc * >
struct	llvm::DOTGraphTraits< PGOUseFunc * >

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	DEBUG_TYPE "pgo-instrumentation"
#define	VALUE_PROF_KIND(Enumerator, Value, Descr)
#define	INSTR_PROF_VISIBILITY
#define	INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
#define	INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
#define	INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
#define	INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
#define	VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
#define	INSTR_PROF_COMMA
#define	INSTR_PROF_DATA_DEFINED
#define	COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
#define	COVMAP_HEADER(Type, LLVMType, Name, Initializer)
#define	COVINIT_FUNC(Type, LLVMType, Name, Initializer)

Typedefs
using	VPCandidateInfo = ValueProfileCollector::CandidateInfo

Functions
	STATISTIC (NumOfPGOInstrument, "Number of edges instrumented.")
	STATISTIC (NumOfPGOSelectInsts, "Number of select instruction instrumented.")
	STATISTIC (NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented.")
	STATISTIC (NumOfPGOEdge, "Number of edges.")
	STATISTIC (NumOfPGOBB, "Number of basic-blocks.")
	STATISTIC (NumOfPGOSplit, "Number of critical edge splits.")
	STATISTIC (NumOfPGOFunc, "Number of functions having valid profile counts.")
	STATISTIC (NumOfPGOMismatch, "Number of functions having mismatch profile.")
	STATISTIC (NumOfPGOMissing, "Number of functions without profile.")
	STATISTIC (NumOfPGOICall, "Number of indirect call value instrumentations.")
	STATISTIC (NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO.")
	STATISTIC (NumOfCSPGOSelectInsts, "Number of select instruction instrumented in CSPGO.")
	STATISTIC (NumOfCSPGOMemIntrinsics, "Number of mem intrinsics instrumented in CSPGO.")
	STATISTIC (NumOfCSPGOEdge, "Number of edges in CSPGO.")
	STATISTIC (NumOfCSPGOBB, "Number of basic-blocks in CSPGO.")
	STATISTIC (NumOfCSPGOSplit, "Number of critical edge splits in CSPGO.")
	STATISTIC (NumOfCSPGOFunc, "Number of functions having valid profile counts in CSPGO.")
	STATISTIC (NumOfCSPGOMismatch, "Number of functions having mismatch profile in CSPGO.")
	STATISTIC (NumOfCSPGOMissing, "Number of functions without profile in CSPGO.")
	STATISTIC (NumCoveredBlocks, "Number of basic blocks that were executed")
cl::opt< bool >	llvm::PGOWarnMissing ("pgo-warn-missing-function", cl::init(false), cl::Hidden, cl::desc("Use this option to turn on/off " "warnings about missing profile data for " "functions."))
cl::opt< bool >	llvm::NoPGOWarnMismatch ("no-pgo-warn-mismatch", cl::init(false), cl::Hidden, cl::desc("Use this option to turn off/on " "warnings about profile cfg mismatch."))
cl::opt< bool >	llvm::NoPGOWarnMismatchComdatWeak ("no-pgo-warn-mismatch-comdat-weak", cl::init(true), cl::Hidden, cl::desc("The option is used to turn on/off " "warnings about hash mismatch for comdat " "or weak functions."))
cl::opt< bool >	llvm::PGOInstrumentColdFunctionOnly ("pgo-instrument-cold-function-only", cl::init(false), cl::Hidden, cl::desc("Enable cold function only instrumentation."))
static std::string	getBranchCondString (Instruction *TI)
static GlobalVariable *	createIRLevelProfileFlagVar (Module &M, PGOInstrumentationType InstrumentationType)
static bool	canRenameComdat (Function &F, std::unordered_multimap< Comdat , GlobalValue > &ComdatMembers)
static void	populateEHOperandBundle (VPCandidateInfo &Cand, DenseMap< BasicBlock *, ColorVector > &BlockColors, SmallVectorImpl< OperandBundleDef > &OpBundles)
static uint64_t	sumEdgeCount (const ArrayRef< PGOUseEdge * > Edges)
static void	setupBBInfoEdges (const FuncPGOInstrumentation< PGOUseEdge, PGOUseBBInfo > &FuncInfo)
	Set up InEdges/OutEdges for all BBs in the MST.
static void	annotateFunctionWithHashMismatch (Function &F, LLVMContext &ctx)
static bool	isIndirectBrTarget (BasicBlock *BB)
static uint32_t	getMaxNumAnnotations (InstrProfValueKind ValueProfKind)
static void	collectComdatMembers (Module &M, std::unordered_multimap< Comdat , GlobalValue > &ComdatMembers)
static bool	skipPGOUse (const Function &F)
static bool	skipPGOGen (const Function &F)
static bool	InstrumentAllFunctions (Module &M, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo (Function &)> LookupBPI, function_ref< BlockFrequencyInfo (Function &)> LookupBFI, function_ref< LoopInfo *(Function &)> LookupLI, PGOInstrumentationType InstrumentationType)
static void	fixFuncEntryCount (PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI)
static void	verifyFuncBFI (PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI, uint64_t HotCountThreshold, uint64_t ColdCountThreshold)
static bool	annotateAllFunctions (Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, vfs::FileSystem &FS, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo (Function &)> LookupBPI, function_ref< BlockFrequencyInfo (Function &)> LookupBFI, function_ref< LoopInfo (Function &)> LookupLI, ProfileSummaryInfo PSI, bool IsCS)
static std::string	getSimpleNodeName (const BasicBlock *Node)
LLVM_ABI void	llvm::setIrrLoopHeaderMetadata (Module M, Instruction TI, uint64_t Count)

Variables
static cl::opt< std::string >	PGOTestProfileFile ("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is " "mainly for test purpose."))
static cl::opt< std::string >	PGOTestProfileRemappingFile ("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))
static cl::opt< bool >	DisableValueProfiling ("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
static cl::opt< unsigned >	MaxNumAnnotations ("icp-max-annotations", cl::init(3), cl::Hidden, cl::desc("Max number of annotations for a single indirect " "call callsite"))
static cl::opt< unsigned >	MaxNumMemOPAnnotations ("memop-max-annotations", cl::init(4), cl::Hidden, cl::desc("Max number of precise value annotations for a single memop" "intrinsic"))
static cl::opt< bool >	DoComdatRenaming ("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))
static cl::opt< bool >	llvm::PGOInstrSelect ("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))
static cl::opt< PGOViewCountsType >	llvm::PGOViewRawCounts ("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
static cl::opt< bool >	llvm::PGOInstrMemOP ("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))
static cl::opt< bool >	llvm::EmitBranchProbability ("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass\|" "pass-remarks}=pgo-instrumentation"))
static cl::opt< bool >	llvm::PGOInstrumentEntry ("pgo-instrument-entry", cl::init(false), cl::Hidden, cl::desc("Force to instrument function entry basicblock."))
static cl::opt< bool >	llvm::PGOInstrumentLoopEntries ("pgo-instrument-loop-entries", cl::init(false), cl::Hidden, cl::desc("Force to instrument loop entries."))
static cl::opt< bool >	llvm::PGOFunctionEntryCoverage ("pgo-function-entry-coverage", cl::Hidden, cl::desc("Use this option to enable function entry coverage instrumentation."))
static cl::opt< bool >	llvm::PGOBlockCoverage ("pgo-block-coverage", cl::desc("Use this option to enable basic block coverage instrumentation"))
static cl::opt< bool >	llvm::PGOViewBlockCoverageGraph ("pgo-view-block-coverage-graph", cl::desc("Create a dot file of CFGs with block " "coverage inference information"))
static cl::opt< bool >	llvm::PGOTemporalInstrumentation ("pgo-temporal-instrumentation", cl::desc("Use this option to enable temporal instrumentation"))
static cl::opt< bool >	llvm::PGOFixEntryCount ("pgo-fix-entry-count", cl::init(true), cl::Hidden, cl::desc("Fix function entry count in profile use."))
static cl::opt< bool >	llvm::PGOVerifyHotBFI ("pgo-verify-hot-bfi", cl::init(false), cl::Hidden, cl::desc("Print out the non-match BFI count if a hot raw profile count " "becomes non-hot, or a cold raw profile count becomes hot. " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remarks-analysis=pgo."))
static cl::opt< bool >	llvm::PGOVerifyBFI ("pgo-verify-bfi", cl::init(false), cl::Hidden, cl::desc("Print out mismatched BFI counts after setting profile metadata " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remarks-analysis=pgo."))
static cl::opt< unsigned >	llvm::PGOVerifyBFIRatio ("pgo-verify-bfi-ratio", cl::init(2), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: only print out " "mismatched BFI if the difference percentage is greater than " "this value (in percentage)."))
static cl::opt< unsigned >	llvm::PGOVerifyBFICutoff ("pgo-verify-bfi-cutoff", cl::init(5), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose " "profile count value is below."))
static cl::opt< std::string >	llvm::PGOTraceFuncHash ("pgo-trace-func-hash", cl::init("-"), cl::Hidden, cl::value_desc("function name"), cl::desc("Trace the hash of the function with this name."))
static cl::opt< unsigned >	llvm::PGOFunctionSizeThreshold ("pgo-function-size-threshold", cl::Hidden, cl::desc("Do not instrument functions smaller than this threshold."))
static cl::opt< unsigned >	llvm::PGOFunctionCriticalEdgeThreshold ("pgo-critical-edge-threshold", cl::init(20000), cl::Hidden, cl::desc("Do not instrument functions with the number of critical edges " " greater than this threshold."))
static cl::opt< uint64_t >	llvm::PGOColdInstrumentEntryThreshold ("pgo-cold-instrument-entry-threshold", cl::init(0), cl::Hidden, cl::desc("For cold function instrumentation, skip instrumenting functions " "whose entry count is above the given value."))
static cl::opt< bool >	llvm::PGOTreatUnknownAsCold ("pgo-treat-unknown-as-cold", cl::init(false), cl::Hidden, cl::desc("For cold function instrumentation, treat count unknown(e.g. " "unprofiled) functions as cold."))
cl::list< std::string >	llvm::CtxPGOSkipCallsiteInstrument ("ctx-prof-skip-callsite-instr", cl::Hidden, cl::desc("Do not instrument callsites to functions in this list. Intended " "for testing."))
static const char *	ValueProfKindDescr []

Macro Definition Documentation

◆ COVINIT_FUNC

#define COVINIT_FUNC	(	Type,
		LLVMType,
		Name,
		Initializer )

◆ COVMAP_FUNC_RECORD

#define COVMAP_FUNC_RECORD	(	Type,
		LLVMType,
		Name,
		Initializer )

◆ COVMAP_HEADER

#define COVMAP_HEADER	(	Type,
		LLVMType,
		Name,
		Initializer )

◆ DEBUG_TYPE

#define DEBUG_TYPE "pgo-instrumentation"

Definition at line 134 of file PGOInstrumentation.cpp.

◆ INSTR_PROF_COMMA

#define INSTR_PROF_COMMA

◆ INSTR_PROF_DATA

#define INSTR_PROF_DATA	(	Type,
		LLVMType,
		Name,
		Initializer )

◆ INSTR_PROF_DATA_DEFINED

#define INSTR_PROF_DATA_DEFINED

◆ INSTR_PROF_RAW_HEADER

#define INSTR_PROF_RAW_HEADER	(	Type,
		Name,
		Initializer )

◆ INSTR_PROF_VALUE_NODE

#define INSTR_PROF_VALUE_NODE	(	Type,
		LLVMType,
		Name,
		Initializer )

◆ INSTR_PROF_VISIBILITY

#define INSTR_PROF_VISIBILITY

◆ INSTR_PROF_VTABLE_DATA

#define INSTR_PROF_VTABLE_DATA	(	Type,
		LLVMType,
		Name,
		Initializer )

◆ VALUE_PROF_FUNC_PARAM

#define VALUE_PROF_FUNC_PARAM	(	ArgType,
		ArgName,
		ArgLLVMType )

◆ VALUE_PROF_KIND

#define VALUE_PROF_KIND	(	Enumerator,
		Value,
		Descr )

Value:

Descr,

Typedef Documentation

◆ VPCandidateInfo

using VPCandidateInfo = ValueProfileCollector::CandidateInfo

Definition at line 132 of file PGOInstrumentation.cpp.

Function Documentation

◆ annotateAllFunctions()

bool annotateAllFunctions	(	Module &	M,
		StringRef	ProfileFileName,
		StringRef	ProfileRemappingFileName,
		vfs::FileSystem &	FS,
		function_ref< TargetLibraryInfo &(Function &)>	LookupTLI,
		function_ref< BranchProbabilityInfo *(Function &)>	LookupBPI,
		function_ref< BlockFrequencyInfo *(Function &)>	LookupBFI,
		function_ref< LoopInfo *(Function &)>	LookupLI,
		ProfileSummaryInfo *	PSI,
		bool	IsCS )

static

Definition at line 2152 of file PGOInstrumentation.cpp.

References ColdCountThreshold, collectComdatMembers(), llvm::IndexedInstrProfReader::create(), llvm::createPGONameMetadata(), llvm::StringRef::data(), llvm::dbgs(), llvm::DS_Warning, E(), llvm::EnableVTableProfileUse, F, fixFuncEntryCount(), G, llvm::ProfileSummaryInfo::getOrCompColdCountThreshold(), llvm::ProfileSummaryInfo::getOrCompHotCountThreshold(), llvm::getPGOName(), llvm::handleAllErrors(), LLVM_DEBUG, llvm::ErrorInfoBase::message(), llvm::InstrProfRecord::NotPseudo, llvm::Function::PCT_Real, llvm::PGOFixEntryCount, llvm::PGOInstrumentEntry, llvm::PGOInstrumentLoopEntries, llvm::PGOVCT_Graph, llvm::PGOVCT_None, llvm::PGOVCT_Text, llvm::PGOVerifyBFI, llvm::PGOVerifyHotBFI, llvm::PGOViewCounts, llvm::PGOViewRawCounts, llvm::InstrProfRecord::PseudoHot, llvm::ProfileSummary::PSK_CSInstr, llvm::ProfileSummary::PSK_Instr, llvm::ProfileSummaryInfo::refresh(), skipPGOUse(), llvm::SplitIndirectBrCriticalEdges(), verifyFuncBFI(), llvm::ViewBlockFreqFuncName, llvm::ViewGraph(), and llvm::WriteGraph().

Referenced by llvm::PGOInstrumentationUse::run().

◆ annotateFunctionWithHashMismatch()

void annotateFunctionWithHashMismatch	(	Function &	F,
		LLVMContext &	ctx )

static

Definition at line 1387 of file PGOInstrumentation.cpp.

References llvm::cast(), llvm::MDBuilder::createString(), F, llvm::MDTuple::get(), N, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

◆ canRenameComdat()

bool canRenameComdat	(	Function &	F,
		std::unordered_multimap< Comdat , GlobalValue > &	ComdatMembers )

static

Definition at line 755 of file PGOInstrumentation.cpp.

References assert(), llvm::CallingConv::C, llvm::canRenameComdatFunc(), DoComdatRenaming, llvm::dyn_cast(), F, llvm::isa(), and llvm::make_range().

◆ collectComdatMembers()

void collectComdatMembers	(	Module &	M,
		std::unordered_multimap< Comdat , GlobalValue > &	ComdatMembers )

static

Definition at line 1894 of file PGOInstrumentation.cpp.

References llvm::CallingConv::C, DoComdatRenaming, and F.

Referenced by annotateAllFunctions(), and InstrumentAllFunctions().

◆ createIRLevelProfileFlagVar()

GlobalVariable * createIRLevelProfileFlagVar	(	Module &	M,
		PGOInstrumentationType	InstrumentationType )

static

Definition at line 447 of file PGOInstrumentation.cpp.

References llvm::CSFDO, llvm::CTXPROF, llvm::InstrProfCorrelator::DEBUG_INFO, llvm::DebugInfoCorrelate, llvm::GlobalValue::ExternalLinkage, llvm::Type::getInt64Ty(), llvm::Constant::getIntegerValue(), llvm::GlobalValue::HiddenVisibility, INSTR_PROF_QUOTE, INSTR_PROF_RAW_VERSION, INSTR_PROF_RAW_VERSION_VAR, llvm::isGPUProfTarget(), llvm::PGOBlockCoverage, llvm::PGOFunctionEntryCoverage, llvm::PGOInstrumentEntry, llvm::PGOInstrumentLoopEntries, llvm::PGOTemporalInstrumentation, llvm::ProfileCorrelate, llvm::GlobalValue::ProtectedVisibility, VARIANT_MASK_BYTE_COVERAGE, VARIANT_MASK_CSIR_PROF, VARIANT_MASK_DBG_CORRELATE, VARIANT_MASK_FUNCTION_ENTRY_ONLY, VARIANT_MASK_INSTR_ENTRY, VARIANT_MASK_INSTR_LOOP_ENTRIES, VARIANT_MASK_IR_PROF, VARIANT_MASK_TEMPORAL_PROF, and llvm::GlobalValue::WeakAnyLinkage.

Referenced by InstrumentAllFunctions(), and llvm::PGOInstrumentationGenCreateVar::run().

◆ fixFuncEntryCount()

void fixFuncEntryCount	(	PGOUseFunc &	Func,
		LoopInfo &	LI,
		BranchProbabilityInfo &	NBPI )

static

Definition at line 2030 of file PGOInstrumentation.cpp.

References assert(), llvm::APFloatBase::cmpEqual, llvm::APFloatBase::cmpGreaterThan, llvm::dbgs(), F, FuncEntryCount, llvm::BlockFrequencyInfo::getBlockProfileCount(), llvm::APFloat::getZero(), llvm::APFloatBase::IEEEdouble(), LLVM_DEBUG, llvm::Function::PCT_Real, and llvm::APFloatBase::rmNearestTiesToEven.

Referenced by annotateAllFunctions().

◆ getBranchCondString()

std::string getBranchCondString ( Instruction * TI )

static

Definition at line 409 of file PGOInstrumentation.cpp.

References Cond, llvm::dyn_cast(), llvm::BranchInst::getCondition(), llvm::User::getOperand(), llvm::CmpInst::getPredicate(), llvm::Value::getType(), llvm::BranchInst::isConditional(), llvm::ConstantInt::isMinusOne(), llvm::ConstantInt::isOne(), llvm::ConstantInt::isZero(), llvm::Type::print(), and RHS.

Referenced by llvm::setProfMetadata().

◆ getMaxNumAnnotations()

uint32_t getMaxNumAnnotations ( InstrProfValueKind ValueProfKind )

static

Definition at line 1827 of file PGOInstrumentation.cpp.

References MaxNumAnnotations, MaxNumMemOPAnnotations, and llvm::MaxNumVTableAnnotations.

◆ getSimpleNodeName()

std::string getSimpleNodeName ( const BasicBlock * Node )

static

Definition at line 2398 of file PGOInstrumentation.cpp.

References Node::printAsOperand().

Referenced by llvm::DOTGraphTraits< PGOUseFunc * >::getNodeLabel().

◆ InstrumentAllFunctions()

bool InstrumentAllFunctions	(	Module &	M,
		function_ref< TargetLibraryInfo &(Function &)>	LookupTLI,
		function_ref< BranchProbabilityInfo *(Function &)>	LookupBPI,
		function_ref< BlockFrequencyInfo *(Function &)>	LookupBFI,
		function_ref< LoopInfo *(Function &)>	LookupLI,
		PGOInstrumentationType	InstrumentationType )

static

Definition at line 1954 of file PGOInstrumentation.cpp.

References collectComdatMembers(), createIRLevelProfileFlagVar(), llvm::LLVMContext::diagnose(), llvm::DS_Warning, llvm::EnableVTableValueProfiling, F, llvm::FDO, and skipPGOGen().

Referenced by llvm::PGOInstrumentationGen::run().

◆ isIndirectBrTarget()

bool isIndirectBrTarget ( BasicBlock * BB )

static

Definition at line 1744 of file PGOInstrumentation.cpp.

References llvm::isa(), and llvm::predecessors().

◆ populateEHOperandBundle()

void populateEHOperandBundle	(	VPCandidateInfo &	Cand,
		DenseMap< BasicBlock *, ColorVector > &	BlockColors,
		SmallVectorImpl< OperandBundleDef > &	OpBundles )

static

Definition at line 902 of file PGOInstrumentation.cpp.

References llvm::ValueProfileCollector::CandidateInfo::AnnotatedInst, assert(), llvm::dyn_cast(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::empty(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::TinyPtrVector< EltTy >::front(), llvm::BasicBlock::getFirstNonPHIIt(), llvm::isa(), llvm::LLVMContext::OB_funclet, and llvm::TinyPtrVector< EltTy >::size().

◆ setupBBInfoEdges()

void setupBBInfoEdges ( const FuncPGOInstrumentation< PGOUseEdge, PGOUseBBInfo > & FuncInfo )

static

Set up InEdges/OutEdges for all BBs in the MST.

Definition at line 1283 of file PGOInstrumentation.cpp.

References E().

◆ skipPGOGen()

bool skipPGOGen ( const Function & F )

static

Definition at line 1935 of file PGOInstrumentation.cpp.

References F, llvm::PGOColdInstrumentEntryThreshold, llvm::PGOFunctionSizeThreshold, llvm::PGOInstrumentColdFunctionOnly, llvm::PGOTreatUnknownAsCold, and skipPGOUse().

Referenced by InstrumentAllFunctions().

◆ skipPGOUse()

bool skipPGOUse ( const Function & F )

static

Definition at line 1911 of file PGOInstrumentation.cpp.

References llvm::dbgs(), E(), F, llvm::Instruction::getNumSuccessors(), I, llvm::isCriticalEdge(), LLVM_DEBUG, and llvm::PGOFunctionCriticalEdgeThreshold.

Referenced by annotateAllFunctions(), and skipPGOGen().

◆ STATISTIC() [1/20]

STATISTIC	(	NumCoveredBlocks	,
		"Number of basic blocks that were executed"	)

◆ STATISTIC() [2/20]

STATISTIC	(	NumOfCSPGOBB	,
		"Number of basic-blocks in CSPGO."	)

◆ STATISTIC() [3/20]

STATISTIC	(	NumOfCSPGOEdge	,
		"Number of edges in CSPGO."	)

◆ STATISTIC() [4/20]

STATISTIC	(	NumOfCSPGOFunc	,
		"Number of functions having valid profile counts in CSPGO."	)

◆ STATISTIC() [5/20]

STATISTIC	(	NumOfCSPGOInstrument	,
		"Number of edges instrumented in CSPGO."	)

◆ STATISTIC() [6/20]

STATISTIC	(	NumOfCSPGOMemIntrinsics	,
		"Number of mem intrinsics instrumented in CSPGO."	)

◆ STATISTIC() [7/20]

STATISTIC	(	NumOfCSPGOMismatch	,
		"Number of functions having mismatch profile in CSPGO."	)

◆ STATISTIC() [8/20]

STATISTIC	(	NumOfCSPGOMissing	,
		"Number of functions without profile in CSPGO."	)

◆ STATISTIC() [9/20]

STATISTIC	(	NumOfCSPGOSelectInsts	,
		"Number of select instruction instrumented in CSPGO."	)

◆ STATISTIC() [10/20]

STATISTIC	(	NumOfCSPGOSplit	,
		"Number of critical edge splits in CSPGO."	)

◆ STATISTIC() [11/20]

STATISTIC	(	NumOfPGOBB	,
		"Number of basic-blocks."	)

◆ STATISTIC() [12/20]

STATISTIC	(	NumOfPGOEdge	,
		"Number of edges."	)

◆ STATISTIC() [13/20]

STATISTIC	(	NumOfPGOFunc	,
		"Number of functions having valid profile counts."	)

◆ STATISTIC() [14/20]

STATISTIC	(	NumOfPGOICall	,
		"Number of indirect call value instrumentations."	)

◆ STATISTIC() [15/20]

STATISTIC	(	NumOfPGOInstrument	,
		"Number of edges instrumented."	)

◆ STATISTIC() [16/20]

STATISTIC	(	NumOfPGOMemIntrinsics	,
		"Number of mem intrinsics instrumented."	)

◆ STATISTIC() [17/20]

STATISTIC	(	NumOfPGOMismatch	,
		"Number of functions having mismatch profile."	)

◆ STATISTIC() [18/20]

STATISTIC	(	NumOfPGOMissing	,
		"Number of functions without profile."	)

◆ STATISTIC() [19/20]

STATISTIC	(	NumOfPGOSelectInsts	,
		"Number of select instruction instrumented."	)

◆ STATISTIC() [20/20]

STATISTIC	(	NumOfPGOSplit	,
		"Number of critical edge splits."	)

◆ sumEdgeCount()

uint64_t sumEdgeCount ( const ArrayRef< PGOUseEdge * > Edges )

static

Definition at line 1148 of file PGOInstrumentation.cpp.

References E(), and llvm::Total.

◆ verifyFuncBFI()

void verifyFuncBFI	(	PGOUseFunc &	Func,
		LoopInfo &	LI,
		BranchProbabilityInfo &	NBPI,
		uint64_t	HotCountThreshold,
		uint64_t	ColdCountThreshold )

static

Definition at line 2077 of file PGOInstrumentation.cpp.

References ColdCountThreshold, DEBUG_TYPE, llvm::OptimizationRemarkEmitter::emit(), llvm::StringRef::empty(), F, llvm::BlockFrequencyInfo::getBlockProfileCount(), llvm::PGOVerifyBFICutoff, llvm::PGOVerifyBFIRatio, llvm::PGOVerifyHotBFI, and llvm::Remark.

Referenced by annotateAllFunctions().

Variable Documentation

◆ DisableValueProfiling

cl::opt< bool > DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))	(	"disable-vp"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Disable Value Profiling")	)

static

◆ DoComdatRenaming

cl::opt< bool > DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))	(	"do-comdat-renaming"	,
		cl::init(false)	,
		cl::Hidden	,
		cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner")	)

static

Referenced by canRenameComdat(), and collectComdatMembers().

◆ MaxNumAnnotations

cl::opt< unsigned > MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::desc("Max number of annotations for a single indirect " "call callsite"))	(	"icp-max-annotations"	,
		cl::init(3)	,
		cl::Hidden	,
		cl::desc("Max number of annotations for a single indirect " "call callsite")	)

static

Referenced by getMaxNumAnnotations().

◆ MaxNumMemOPAnnotations

cl::opt< unsigned > MaxNumMemOPAnnotations("memop-max-annotations", cl::init(4), cl::Hidden, cl::desc("Max number of precise value annotations for a single memop" "intrinsic"))	(	"memop-max-annotations"	,
		cl::init(4)	,
		cl::Hidden	,
		cl::desc("Max number of precise value annotations for a single memop" "intrinsic")	)

static

Referenced by getMaxNumAnnotations().

◆ PGOTestProfileFile

cl::opt< std::string > PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is " "mainly for test purpose."))	(	"pgo-test-profile-file"	,
		cl::init("")	,
		cl::Hidden	,
		cl::value_desc("filename")	,
		cl::desc("Specify the path of profile data file. This is " "mainly for test purpose.")	)

static

Referenced by llvm::PGOInstrumentationUse::PGOInstrumentationUse().

◆ PGOTestProfileRemappingFile

cl::opt< std::string > PGOTestProfileRemappingFile("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))	(	"pgo-test-profile-remapping-file"	,
		cl::init("")	,
		cl::Hidden	,
		cl::value_desc("filename")	,
		cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose.")	)

static

Referenced by llvm::PGOInstrumentationUse::PGOInstrumentationUse().

◆ ValueProfKindDescr

const char* ValueProfKindDescr[]

static

Definition at line 439 of file PGOInstrumentation.cpp.

Classes

Namespaces

Macros

Typedefs

Functions

Variables

Macro Definition Documentation

◆ COVINIT_FUNC

◆ COVMAP_FUNC_RECORD

◆ COVMAP_HEADER

◆ DEBUG_TYPE

◆ INSTR_PROF_COMMA

◆ INSTR_PROF_DATA

◆ INSTR_PROF_DATA_DEFINED

◆ INSTR_PROF_RAW_HEADER

◆ INSTR_PROF_VALUE_NODE

◆ INSTR_PROF_VISIBILITY

◆ INSTR_PROF_VTABLE_DATA

◆ VALUE_PROF_FUNC_PARAM

◆ VALUE_PROF_KIND

Typedef Documentation

◆ VPCandidateInfo

Function Documentation

◆ annotateAllFunctions()

◆ annotateFunctionWithHashMismatch()

◆ canRenameComdat()

◆ collectComdatMembers()

◆ createIRLevelProfileFlagVar()

◆ fixFuncEntryCount()

◆ getBranchCondString()

◆ getMaxNumAnnotations()

◆ getSimpleNodeName()

◆ InstrumentAllFunctions()

◆ isIndirectBrTarget()

◆ populateEHOperandBundle()

◆ setupBBInfoEdges()

◆ skipPGOGen()

◆ skipPGOUse()

◆ STATISTIC() [1/20]

◆ STATISTIC() [2/20]

◆ STATISTIC() [3/20]

◆ STATISTIC() [4/20]

◆ STATISTIC() [5/20]

◆ STATISTIC() [6/20]

◆ STATISTIC() [7/20]

◆ STATISTIC() [8/20]

◆ STATISTIC() [9/20]

◆ STATISTIC() [10/20]

◆ STATISTIC() [11/20]

◆ STATISTIC() [12/20]

◆ STATISTIC() [13/20]

◆ STATISTIC() [14/20]

◆ STATISTIC() [15/20]

◆ STATISTIC() [16/20]

◆ STATISTIC() [17/20]

◆ STATISTIC() [18/20]

◆ STATISTIC() [19/20]

◆ STATISTIC() [20/20]

◆ sumEdgeCount()

◆ verifyFuncBFI()

Variable Documentation

◆ DisableValueProfiling

◆ DoComdatRenaming

◆ MaxNumAnnotations

◆ MaxNumMemOPAnnotations

◆ PGOTestProfileFile

◆ PGOTestProfileRemappingFile

◆ ValueProfKindDescr