LLVM  13.0.0git
Macros | Typedefs | Functions | Variables
SampleProfile.cpp File Reference
#include "llvm/Transforms/IPO/SampleProfile.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/PriorityQueue.h"
#include "llvm/ADT/SCCIterator.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BlockFrequencyInfoImpl.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/CallGraphSCCPass.h"
#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/InlineCost.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/PostDominators.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ReplayInlineAdvisor.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ValueSymbolTable.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/SampleProf.h"
#include "llvm/ProfileData/SampleProfReader.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/GenericDomTree.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/IPO/ProfiledCallGraph.h"
#include "llvm/Transforms/IPO/SampleContextTracker.h"
#include "llvm/Transforms/IPO/SampleProfileProbe.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/CallPromotionUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h"
#include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <functional>
#include <limits>
#include <map>
#include <memory>
#include <queue>
#include <string>
#include <system_error>
#include <utility>
#include <vector>

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "sample-profile"
 
#define CSINLINE_DEBUG   DEBUG_TYPE "-inline"
 

Typedefs

using ProfileCount = Function::ProfileCount
 

Functions

 STATISTIC (NumCSInlined, "Number of functions inlined with context sensitive profile")
 
 STATISTIC (NumCSNotInlined, "Number of functions not inlined with context sensitive profile")
 
 STATISTIC (NumMismatchedProfile, "Number of functions with CFG mismatched profile")
 
 STATISTIC (NumMatchedProfile, "Number of functions with CFG matched profile")
 
 STATISTIC (NumDuplicatedInlinesite, "Number of inlined callsites with a partial distribution factor")
 
 STATISTIC (NumCSInlinedHitMinLimit, "Number of functions with FDO inline stopped due to min size limit")
 
 STATISTIC (NumCSInlinedHitMaxLimit, "Number of functions with FDO inline stopped due to max size limit")
 
 STATISTIC (NumCSInlinedHitGrowthLimit, "Number of functions with FDO inline stopped due to growth size limit")
 
static bool doesHistoryAllowICP (const Instruction &Inst, StringRef Candidate)
 Check whether the indirect call promotion history of Inst allows the promotion for Candidate. More...
 
static void updateIDTMetaData (Instruction &Inst, const SmallVectorImpl< InstrProfValueData > &CallTargets, uint64_t Sum)
 Update indirect call target profile metadata for Inst. More...
 
static SmallVector< InstrProfValueData, 2 > GetSortedValueDataFromCallTargets (const SampleRecord::CallTargetMap &M)
 Returns the sorted CallTargetMap M by count in descending order. More...
 
 INITIALIZE_PASS_BEGIN (SampleProfileLoaderLegacyPass, "sample-profile", "Sample Profile loader", false, false) INITIALIZE_PASS_END(SampleProfileLoaderLegacyPass
 

Variables

static cl::opt< std::string > SampleProfileFile ("sample-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile file loaded by -sample-profile"), cl::Hidden)
 
static cl::opt< std::string > SampleProfileRemappingFile ("sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden)
 
static cl::opt< bool > ProfileSampleAccurate ("profile-sample-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " "callsite and function as having 0 samples. Otherwise, treat " "un-sampled callsites and functions conservatively as unknown. "))
 
static cl::opt< bool > ProfileAccurateForSymsInList ("profile-accurate-for-symsinlist", cl::Hidden, cl::ZeroOrMore, cl::init(true), cl::desc("For symbols in profile symbol list, regard their profiles to " "be accurate. It may be overriden by profile-sample-accurate. "))
 
static cl::opt< bool > ProfileMergeInlinee ("sample-profile-merge-inlinee", cl::Hidden, cl::init(true), cl::desc("Merge past inlinee's profile to outline version if sample " "profile loader decided not to inline a call site. It will " "only be enabled when top-down order of profile loading is " "enabled. "))
 
static cl::opt< bool > ProfileTopDownLoad ("sample-profile-top-down-load", cl::Hidden, cl::init(true), cl::desc("Do profile annotation and inlining for functions in top-down " "order of call graph during sample profile loading. It only " "works for new pass manager. "))
 
static cl::opt< bool > UseProfiledCallGraph ("use-profiled-call-graph", cl::init(true), cl::Hidden, cl::desc("Process functions in a top-down order " "defined by the profiled call graph when " "-sample-profile-top-down-load is on."))
 
static cl::opt< bool > ProfileSizeInline ("sample-profile-inline-size", cl::Hidden, cl::init(false), cl::desc("Inline cold call sites in profile loader if it's beneficial " "for code size."))
 
cl::opt< intProfileInlineGrowthLimit ("sample-profile-inline-growth-limit", cl::Hidden, cl::init(12), cl::desc("The size growth ratio limit for proirity-based sample profile " "loader inlining."))
 
cl::opt< intProfileInlineLimitMin ("sample-profile-inline-limit-min", cl::Hidden, cl::init(100), cl::desc("The lower bound of size growth limit for " "proirity-based sample profile loader inlining."))
 
cl::opt< intProfileInlineLimitMax ("sample-profile-inline-limit-max", cl::Hidden, cl::init(10000), cl::desc("The upper bound of size growth limit for " "proirity-based sample profile loader inlining."))
 
cl::opt< intSampleHotCallSiteThreshold ("sample-profile-hot-inline-threshold", cl::Hidden, cl::init(3000), cl::desc("Hot callsite threshold for proirity-based sample profile loader " "inlining."))
 
cl::opt< intSampleColdCallSiteThreshold ("sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45), cl::desc("Threshold for inlining cold callsites"))
 
static cl::opt< unsigned > ProfileICPRelativeHotness ("sample-profile-icp-relative-hotness", cl::Hidden, cl::init(25), cl::desc("Relative hotness percentage threshold for indirect " "call promotion in proirity-based sample profile loader inlining."))
 
static cl::opt< unsigned > ProfileICPRelativeHotnessSkip ("sample-profile-icp-relative-hotness-skip", cl::Hidden, cl::init(1), cl::desc("Skip relative hotness check for ICP up to given number of targets."))
 
static cl::opt< bool > CallsitePrioritizedInline ("sample-profile-prioritized-inline", cl::Hidden, cl::ZeroOrMore, cl::init(false), cl::desc("Use call site prioritized inlining for sample profile loader." "Currently only CSSPGO is supported."))
 
static cl::opt< std::string > ProfileInlineReplayFile ("sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by inlining from sample profile loader."), cl::Hidden)
 
static cl::opt< unsigned > MaxNumPromotions ("sample-profile-icp-max-prom", cl::init(3), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of promotions for a single indirect " "call callsite in sample profile loader"))
 
static cl::opt< bool > OverwriteExistingWeights ("overwrite-existing-weights", cl::Hidden, cl::init(false), cl::desc("Ignore existing branch weights on IR and always overwrite."))
 
sample profile
 
sample Sample Profile loader
 
sample Sample Profile false
 

Macro Definition Documentation

◆ CSINLINE_DEBUG

#define CSINLINE_DEBUG   DEBUG_TYPE "-inline"

Definition at line 107 of file SampleProfile.cpp.

◆ DEBUG_TYPE

#define DEBUG_TYPE   "sample-profile"

Definition at line 106 of file SampleProfile.cpp.

Typedef Documentation

◆ ProfileCount

Definition at line 105 of file SampleProfile.cpp.

Function Documentation

◆ doesHistoryAllowICP()

static bool doesHistoryAllowICP ( const Instruction Inst,
StringRef  Candidate 
)
static

Check whether the indirect call promotion history of Inst allows the promotion for Candidate.

If the profile count for the promotion candidate Candidate is NOMORE_ICP_MAGICNUM, it means Candidate has already been promoted for Inst. If we already have at least MaxNumPromotions NOMORE_ICP_MAGICNUM count values in the value profile of Inst, we cannot promote for Inst anymore.

Definition at line 735 of file SampleProfile.cpp.

References llvm::GlobalValue::getGUID(), llvm::getValueProfDataFromInst(), I, MaxNumPromotions, and llvm::NOMORE_ICP_MAGICNUM.

◆ GetSortedValueDataFromCallTargets()

static SmallVector<InstrProfValueData, 2> GetSortedValueDataFromCallTargets ( const SampleRecord::CallTargetMap M)
static

Returns the sorted CallTargetMap M by count in descending order.

Definition at line 1427 of file SampleProfile.cpp.

References llvm::sampleprof::FunctionSamples::getGUID(), I, M, and llvm::sampleprof::SampleRecord::SortCallTargets().

◆ INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN ( SampleProfileLoaderLegacyPass  ,
"sample-profile ,
"Sample Profile loader ,
false  ,
false   
)

◆ STATISTIC() [1/8]

STATISTIC ( NumCSInlined  ,
"Number of functions inlined with context sensitive profile  
)

◆ STATISTIC() [2/8]

STATISTIC ( NumCSInlinedHitGrowthLimit  ,
"Number of functions with FDO inline stopped due to growth size limit"   
)

◆ STATISTIC() [3/8]

STATISTIC ( NumCSInlinedHitMaxLimit  ,
"Number of functions with FDO inline stopped due to max size limit"   
)

◆ STATISTIC() [4/8]

STATISTIC ( NumCSInlinedHitMinLimit  ,
"Number of functions with FDO inline stopped due to min size limit"   
)

◆ STATISTIC() [5/8]

STATISTIC ( NumCSNotInlined  ,
"Number of functions not inlined with context sensitive profile  
)

◆ STATISTIC() [6/8]

STATISTIC ( NumDuplicatedInlinesite  ,
"Number of inlined callsites with a partial distribution factor"   
)

◆ STATISTIC() [7/8]

STATISTIC ( NumMatchedProfile  ,
"Number of functions with CFG matched profile  
)

◆ STATISTIC() [8/8]

STATISTIC ( NumMismatchedProfile  ,
"Number of functions with CFG mismatched profile  
)

◆ updateIDTMetaData()

static void updateIDTMetaData ( Instruction Inst,
const SmallVectorImpl< InstrProfValueData > &  CallTargets,
uint64_t  Sum 
)
static

Update indirect call target profile metadata for Inst.

Usually Sum is the sum of counts of all the targets for Inst. If it is 0, it means updateIDTMetaData is used to mark a certain target to be promoted already. If it is not zero, we expect to use it to update the total count in the value profile.

Definition at line 772 of file SampleProfile.cpp.

References llvm::annotateValueSite(), assert(), llvm::Data, llvm::SmallVectorImpl< T >::emplace_back(), llvm::Instruction::getParent(), llvm::BasicBlock::getParent(), llvm::GlobalValue::getParent(), llvm::getValueProfDataFromInst(), I, MaxNumPromotions, llvm::min(), llvm::NOMORE_ICP_MAGICNUM, llvm::sort(), and llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::try_emplace().

Variable Documentation

◆ CallsitePrioritizedInline

cl::opt<bool> CallsitePrioritizedInline("sample-profile-prioritized-inline", cl::Hidden, cl::ZeroOrMore, cl::init(false), cl::desc("Use call site prioritized inlining for sample profile loader." "Currently only CSSPGO is supported."))
static

◆ false

sample Sample Profile false

Definition at line 1627 of file SampleProfile.cpp.

◆ loader

sample Sample Profile loader

Definition at line 1627 of file SampleProfile.cpp.

◆ MaxNumPromotions

cl::opt<unsigned> MaxNumPromotions("sample-profile-icp-max-prom", cl::init(3), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of promotions for a single indirect " "call callsite in sample profile loader"))
static

◆ OverwriteExistingWeights

cl::opt<bool> OverwriteExistingWeights("overwrite-existing-weights", cl::Hidden, cl::init(false), cl::desc("Ignore existing branch weights on IR and always overwrite."))
static

◆ profile

sample profile

Definition at line 1626 of file SampleProfile.cpp.

◆ ProfileAccurateForSymsInList

cl::opt<bool> ProfileAccurateForSymsInList("profile-accurate-for-symsinlist", cl::Hidden, cl::ZeroOrMore, cl::init(true), cl::desc("For symbols in profile symbol list, regard their profiles to " "be accurate. It may be overriden by profile-sample-accurate. "))
static

◆ ProfileICPRelativeHotness

cl::opt<unsigned> ProfileICPRelativeHotness("sample-profile-icp-relative-hotness", cl::Hidden, cl::init(25), cl::desc( "Relative hotness percentage threshold for indirect " "call promotion in proirity-based sample profile loader inlining."))
static

◆ ProfileICPRelativeHotnessSkip

cl::opt<unsigned> ProfileICPRelativeHotnessSkip("sample-profile-icp-relative-hotness-skip", cl::Hidden, cl::init(1), cl::desc( "Skip relative hotness check for ICP up to given number of targets."))
static

◆ ProfileInlineGrowthLimit

cl::opt<int> ProfileInlineGrowthLimit("sample-profile-inline-growth-limit", cl::Hidden, cl::init(12), cl::desc("The size growth ratio limit for proirity-based sample profile " "loader inlining."))

◆ ProfileInlineLimitMax

cl::opt<int> ProfileInlineLimitMax("sample-profile-inline-limit-max", cl::Hidden, cl::init(10000), cl::desc("The upper bound of size growth limit for " "proirity-based sample profile loader inlining."))

◆ ProfileInlineLimitMin

cl::opt<int> ProfileInlineLimitMin("sample-profile-inline-limit-min", cl::Hidden, cl::init(100), cl::desc("The lower bound of size growth limit for " "proirity-based sample profile loader inlining."))

◆ ProfileInlineReplayFile

cl::opt<std::string> ProfileInlineReplayFile("sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc( "Optimization remarks file containing inline remarks to be replayed " "by inlining from sample profile loader."), cl::Hidden)
static

◆ ProfileMergeInlinee

cl::opt<bool> ProfileMergeInlinee("sample-profile-merge-inlinee", cl::Hidden, cl::init(true), cl::desc("Merge past inlinee's profile to outline version if sample " "profile loader decided not to inline a call site. It will " "only be enabled when top-down order of profile loading is " "enabled. "))
static

◆ ProfileSampleAccurate

cl::opt<bool> ProfileSampleAccurate("profile-sample-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " "callsite and function as having 0 samples. Otherwise, treat " "un-sampled callsites and functions conservatively as unknown. "))
static

◆ ProfileSizeInline

cl::opt<bool> ProfileSizeInline("sample-profile-inline-size", cl::Hidden, cl::init(false), cl::desc("Inline cold call sites in profile loader if it's beneficial " "for code size."))
static

◆ ProfileTopDownLoad

cl::opt<bool> ProfileTopDownLoad("sample-profile-top-down-load", cl::Hidden, cl::init(true), cl::desc("Do profile annotation and inlining for functions in top-down " "order of call graph during sample profile loading. It only " "works for new pass manager. "))
static

◆ SampleColdCallSiteThreshold

cl::opt<int> SampleColdCallSiteThreshold("sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45), cl::desc("Threshold for inlining cold callsites"))

◆ SampleHotCallSiteThreshold

cl::opt<int> SampleHotCallSiteThreshold("sample-profile-hot-inline-threshold", cl::Hidden, cl::init(3000), cl::desc("Hot callsite threshold for proirity-based sample profile loader " "inlining."))

◆ SampleProfileFile

cl::opt<std::string> SampleProfileFile("sample-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile file loaded by -sample-profile"), cl::Hidden)
static

◆ SampleProfileRemappingFile

cl::opt<std::string> SampleProfileRemappingFile("sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden)
static

◆ UseProfiledCallGraph

cl::opt<bool> UseProfiledCallGraph("use-profiled-call-graph", cl::init(true), cl::Hidden, cl::desc("Process functions in a top-down order " "defined by the profiled call graph when " "-sample-profile-top-down-load is on."))
static