LLVM  15.0.0git
Macros | Typedefs | Functions | Variables
Inliner.cpp File Reference
#include "llvm/Transforms/IPO/Inliner.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PriorityWorklist.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/CGSCCPassManager.h"
#include "llvm/Analysis/CallGraph.h"
#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/InlineCost.h"
#include "llvm/Analysis/InlineOrder.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ReplayInlineAdvisor.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/CallPromotionUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <algorithm>
#include <cassert>
#include <functional>
#include <utility>
#include <vector>

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "inline"
 

Typedefs

using InlinedArrayAllocasTy = DenseMap< ArrayType *, std::vector< AllocaInst * > >
 

Functions

 STATISTIC (NumInlined, "Number of functions inlined")
 
 STATISTIC (NumCallsDeleted, "Number of call sites deleted, not inlined")
 
 STATISTIC (NumDeleted, "Number of functions deleted because all callers found")
 
 STATISTIC (NumMergedAllocas, "Number of allocas merged together")
 
static void mergeInlinedArrayAllocas (Function *Caller, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory)
 Look at all of the allocas that we inlined through this call site. More...
 
static InlineResult inlineCallIfPossible (CallBase &CB, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory, bool InsertLifetime, function_ref< AAResults &(Function &)> &AARGetter, ImportedFunctionsInliningStatistics &ImportedFunctionsStats)
 If it is possible to inline the specified call site, do so and update the CallGraph for this operation. More...
 
static bool inlineHistoryIncludes (Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int >> &InlineHistory)
 Return true if the specified inline history ID indicates an inline history that includes the specified function. More...
 
static bool inlineCallsImpl (CallGraphSCC &SCC, CallGraph &CG, std::function< AssumptionCache &(Function &)> GetAssumptionCache, ProfileSummaryInfo *PSI, std::function< const TargetLibraryInfo &(Function &)> GetTLI, bool InsertLifetime, function_ref< InlineCost(CallBase &CB)> GetInlineCost, function_ref< AAResults &(Function &)> AARGetter, ImportedFunctionsInliningStatistics &ImportedFunctionsStats)
 

Variables

static cl::opt< bool > DisableInlinedAllocaMerging ("disable-inlined-alloca-merging", cl::init(false), cl::Hidden)
 Flag to disable manual alloca merging. More...
 
static cl::opt< intIntraSCCCostMultiplier ("intra-scc-cost-multiplier", cl::init(2), cl::Hidden, cl::desc("Cost multiplier to multiply onto inlined call sites where the " "new call was previously an intra-SCC call (not relevant when the " "original call was already intra-SCC). This can accumulate over " "multiple inlinings (e.g. if a call site already had a cost " "multiplier and one of its inlined calls was also subject to " "this, the inlined call would have the original multiplier " "multiplied by intra-scc-cost-multiplier). This is to prevent tons of " "inlining through a child SCC which can cause terrible compile times"))
 
static cl::opt< bool > KeepAdvisorForPrinting ("keep-inline-advisor-for-printing", cl::init(false), cl::Hidden)
 A flag for test, so we can print the content of the advisor when running it as part of the default (e.g. More...
 
static cl::opt< bool > EnablePostSCCAdvisorPrinting ("enable-scc-inline-advisor-printing", cl::init(false), cl::Hidden)
 Allows printing the contents of the advisor after each SCC inliner pass. More...
 
cl::opt< InlinerFunctionImportStatsOpts > InlinerFunctionImportStats
 
static cl::opt< std::string > CGSCCInlineReplayFile ("cgscc-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by cgscc inlining."), cl::Hidden)
 
static cl::opt< ReplayInlinerSettings::ScopeCGSCCInlineReplayScope ("cgscc-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during cgscc inlining."), cl::Hidden)
 
static cl::opt< ReplayInlinerSettings::FallbackCGSCCInlineReplayFallback ("cgscc-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How cgscc inline replay treats sites that don't come from the replay. " "Original: defers to original advisor, AlwaysInline: inline all sites " "not in replay, NeverInline: inline no sites not in replay"), cl::Hidden)
 
static cl::opt< CallSiteFormat::FormatCGSCCInlineReplayFormat ("cgscc-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden)
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "inline"

Definition at line 73 of file Inliner.cpp.

Typedef Documentation

◆ InlinedArrayAllocasTy

using InlinedArrayAllocasTy = DenseMap<ArrayType *, std::vector<AllocaInst *> >

Definition at line 182 of file Inliner.cpp.

Function Documentation

◆ inlineCallIfPossible()

static InlineResult inlineCallIfPossible ( CallBase CB,
InlineFunctionInfo IFI,
InlinedArrayAllocasTy InlinedArrayAllocas,
int  InlineHistory,
bool  InsertLifetime,
function_ref< AAResults &(Function &)> &  AARGetter,
ImportedFunctionsInliningStatistics ImportedFunctionsStats 
)
static

If it is possible to inline the specified call site, do so and update the CallGraph for this operation.

This function also does some basic book-keeping to update the IR. The InlinedArrayAllocas map keeps track of any allocas that are already available from other functions inlined into the caller. If we are able to inline this call site we attempt to reuse already available allocas or add any new allocas to the set if not possible.

Definition at line 307 of file Inliner.cpp.

References Callee, DisableInlinedAllocaMerging, llvm::CallBase::getCalledFunction(), llvm::CallBase::getCaller(), llvm::InlineFunction(), InlinerFunctionImportStats, IR, llvm::AttributeFuncs::mergeAttributesForInlining(), mergeInlinedArrayAllocas(), llvm::No, and llvm::ImportedFunctionsInliningStatistics::recordInline().

◆ inlineCallsImpl()

static bool inlineCallsImpl ( CallGraphSCC SCC,
CallGraph CG,
std::function< AssumptionCache &(Function &)>  GetAssumptionCache,
ProfileSummaryInfo PSI,
std::function< const TargetLibraryInfo &(Function &)>  GetTLI,
bool  InsertLifetime,
function_ref< InlineCost(CallBase &CB)>  GetInlineCost,
function_ref< AAResults &(Function &)>  AARGetter,
ImportedFunctionsInliningStatistics ImportedFunctionsStats 
)
static

Definition at line 362 of file Inliner.cpp.

References llvm::dbgs(), and LLVM_DEBUG.

Referenced by llvm::LegacyInlinerBase::inlineCalls().

◆ inlineHistoryIncludes()

static bool inlineHistoryIncludes ( Function F,
int  InlineHistoryID,
const SmallVectorImpl< std::pair< Function *, int >> &  InlineHistory 
)
static

Return true if the specified inline history ID indicates an inline history that includes the specified function.

Definition at line 336 of file Inliner.cpp.

References assert(), and F.

◆ mergeInlinedArrayAllocas()

static void mergeInlinedArrayAllocas ( Function Caller,
InlineFunctionInfo IFI,
InlinedArrayAllocasTy InlinedArrayAllocas,
int  InlineHistory 
)
static

Look at all of the allocas that we inlined through this call site.

If we have already inlined other allocas through other calls into this function, then we know that they have disjoint lifetimes and that we can merge them.

There are many heuristics possible for merging these allocas, and the different options have different tradeoffs. One thing that we really don't want to hurt is SRoA: once inlining happens, often allocas are no longer address taken and so they can be promoted.

Our "solution" for that is to only merge allocas whose outermost type is an array type. These are usually not promoted because someone is using a variable index into them. These are also often the most important ones to merge.

A better solution would be to have real memory lifetime markers in the IR and not have the inliner do any merging of allocas at all. This would allow the backend to do proper stack slot coloring of all allocas that actually make it to the backend, which is really what we want.

Because we don't have this information, we do this simple and useful hack.

Definition at line 204 of file Inliner.cpp.

References llvm::dbgs(), E, llvm::Instruction::eraseFromParent(), llvm::AllocaInst::getAlign(), llvm::AllocaInst::getAllocatedType(), llvm::Value::getContext(), llvm::MetadataAsValue::getIfExists(), llvm::LocalAsMetadata::getIfExists(), llvm::Instruction::getParent(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::AllocaInst::isArrayAllocation(), LLVM_DEBUG, llvm::Value::replaceAllUsesWith(), llvm::InlineFunctionInfo::StaticAllocas, and llvm::Value::users().

Referenced by inlineCallIfPossible().

◆ STATISTIC() [1/4]

STATISTIC ( NumCallsDeleted  ,
"Number of call sites  deleted,
not inlined"   
)

◆ STATISTIC() [2/4]

STATISTIC ( NumDeleted  ,
"Number of functions deleted because all callers found"   
)

◆ STATISTIC() [3/4]

STATISTIC ( NumInlined  ,
"Number of functions inlined"   
)

◆ STATISTIC() [4/4]

STATISTIC ( NumMergedAllocas  ,
"Number of allocas merged together"   
)

Variable Documentation

◆ CGSCCInlineReplayFallback

cl::opt<ReplayInlinerSettings::Fallback> CGSCCInlineReplayFallback("cgscc-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values( clEnumValN( ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc( "How cgscc inline replay treats sites that don't come from the replay. " "Original: defers to original advisor, AlwaysInline: inline all sites " "not in replay, NeverInline: inline no sites not in replay"), cl::Hidden)
static

◆ CGSCCInlineReplayFile

cl::opt<std::string> CGSCCInlineReplayFile("cgscc-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc( "Optimization remarks file containing inline remarks to be replayed " "by cgscc inlining."), cl::Hidden)
static

◆ CGSCCInlineReplayFormat

cl::opt<CallSiteFormat::Format> CGSCCInlineReplayFormat("cgscc-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values( clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden)
static

◆ CGSCCInlineReplayScope

cl::opt<ReplayInlinerSettings::Scope> CGSCCInlineReplayScope("cgscc-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during cgscc inlining."), cl::Hidden)
static

◆ DisableInlinedAllocaMerging

cl::opt<bool> DisableInlinedAllocaMerging("disable-inlined-alloca-merging", cl::init(false), cl::Hidden)
static

Flag to disable manual alloca merging.

Merging of allocas was originally done as a stack-size saving technique prior to LLVM's code generator having support for stack coloring based on lifetime markers. It is now in the process of being removed. To experiment with disabling it and relying fully on lifetime marker based stack coloring, you can pass this flag to LLVM.

Referenced by inlineCallIfPossible().

◆ EnablePostSCCAdvisorPrinting

cl::opt<bool> EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing", cl::init(false), cl::Hidden)
static

Allows printing the contents of the advisor after each SCC inliner pass.

Referenced by llvm::ModuleInlinerWrapperPass::ModuleInlinerWrapperPass().

◆ InlinerFunctionImportStats

cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats

◆ IntraSCCCostMultiplier

cl::opt<int> IntraSCCCostMultiplier("intra-scc-cost-multiplier", cl::init(2), cl::Hidden, cl::desc( "Cost multiplier to multiply onto inlined call sites where the " "new call was previously an intra-SCC call (not relevant when the " "original call was already intra-SCC). This can accumulate over " "multiple inlinings (e.g. if a call site already had a cost " "multiplier and one of its inlined calls was also subject to " "this, the inlined call would have the original multiplier " "multiplied by intra-scc-cost-multiplier). This is to prevent tons of " "inlining through a child SCC which can cause terrible compile times"))
static

◆ KeepAdvisorForPrinting

cl::opt<bool> KeepAdvisorForPrinting("keep-inline-advisor-for-printing", cl::init(false), cl::Hidden)
static

A flag for test, so we can print the content of the advisor when running it as part of the default (e.g.

-O3) pipeline.