23#define DEBUG_TYPE "inline-order"
29 cl::desc(
"Choose the priority mode to use in module inline"),
31 "Use callee size priority."),
33 "Use inline cost priority."),
35 "Use cost-benefit ratio."),
40 cl::desc(
"The cost threshold for call sites that get inlined without the "
41 "cost-benefit analysis"));
51 .getCachedResult<ProfileSummaryAnalysis>(
68 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
70 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
71 GetBFI, PSI, RemarksEnabled ? &ORE :
nullptr);
76 SizePriority() =
default;
83 static bool isMoreDesirable(
const SizePriority &P1,
const SizePriority &P2) {
84 return P1.Size < P2.Size;
88 unsigned Size = UINT_MAX;
93 CostPriority() =
default;
96 auto IC = getInlineCostWrapper(
const_cast<CallBase &
>(*CB),
FAM, Params);
100 Cost = IC.isNever() ? INT_MAX : INT_MIN;
103 static bool isMoreDesirable(
const CostPriority &P1,
const CostPriority &P2) {
104 return P1.Cost < P2.Cost;
111class CostBenefitPriority {
113 CostBenefitPriority() =
default;
116 auto IC = getInlineCostWrapper(
const_cast<CallBase &
>(*CB),
FAM, Params);
118 StaticBonusApplied = IC.getStaticBonusApplied();
122 static bool isMoreDesirable(
const CostBenefitPriority &P1,
123 const CostBenefitPriority &P2) {
139 bool P1ReducesCallerSize =
141 bool P2ReducesCallerSize =
143 if (P1ReducesCallerSize || P2ReducesCallerSize) {
146 if (P1ReducesCallerSize != P2ReducesCallerSize)
147 return P1ReducesCallerSize;
151 return P1.Cost < P2.Cost;
154 bool P1HasCB = P1.CostBenefit.has_value();
155 bool P2HasCB = P2.CostBenefit.has_value();
156 if (P1HasCB || P2HasCB) {
159 if (P1HasCB != P2HasCB)
164 APInt LHS = P1.CostBenefit->getBenefit() * P2.CostBenefit->getCost();
165 APInt RHS = P2.CostBenefit->getBenefit() * P1.CostBenefit->getCost();
170 return P1.Cost < P2.Cost;
175 int StaticBonusApplied = 0;
181 MLPriority() =
default;
184 auto IC = getInlineCostWrapper(
const_cast<CallBase &
>(*CB),
FAM, Params);
188 Cost = IC.isNever() ? INT_MAX : INT_MIN;
191 static bool isMoreDesirable(
const MLPriority &P1,
const MLPriority &P2) {
192 return P1.Cost < P2.Cost;
199template <
typename PriorityT>
200class PriorityInlineOrder :
public InlineOrder<std::pair<CallBase *, int>> {
201 using T = std::pair<CallBase *, int>;
204 const auto I1 = Priorities.find(L);
205 const auto I2 = Priorities.find(R);
206 assert(I1 != Priorities.end() && I2 != Priorities.end());
207 return PriorityT::isMoreDesirable(I2->second,
I1->second);
210 bool updateAndCheckDecreased(
const CallBase *CB) {
211 auto It = Priorities.find(CB);
212 const auto OldPriority = It->second;
213 It->second = PriorityT(CB, FAM, Params);
214 const auto NewPriority = It->second;
215 return PriorityT::isMoreDesirable(OldPriority, NewPriority);
226 while (updateAndCheckDecreased(Heap.front())) {
227 std::pop_heap(Heap.begin(), Heap.end(), isLess);
228 std::push_heap(Heap.begin(), Heap.end(), isLess);
234 :
FAM(
FAM), Params(Params) {
236 return hasLowerPriority(L, R);
240 size_t size()
override {
return Heap.size(); }
242 void push(
const T &Elt)
override {
244 const int InlineHistoryID = Elt.second;
247 Priorities[CB] = PriorityT(CB, FAM, Params);
248 std::push_heap(Heap.begin(), Heap.end(), isLess);
249 InlineHistoryMap[CB] = InlineHistoryID;
257 T Result = std::make_pair(CB, InlineHistoryMap[CB]);
258 InlineHistoryMap.erase(CB);
259 std::pop_heap(Heap.begin(), Heap.end(), isLess);
265 auto PredWrapper = [=](
CallBase *CB) ->
bool {
266 return Pred(std::make_pair(CB, 0));
269 std::make_heap(Heap.begin(), Heap.end(), isLess);
284bool llvm::PluginInlineOrderAnalysis::HasBeenRegistered;
286std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
292 LLVM_DEBUG(
dbgs() <<
" Current used priority: Size priority ---- \n");
293 return std::make_unique<PriorityInlineOrder<SizePriority>>(
FAM, Params);
296 LLVM_DEBUG(
dbgs() <<
" Current used priority: Cost priority ---- \n");
297 return std::make_unique<PriorityInlineOrder<CostPriority>>(
FAM, Params);
301 dbgs() <<
" Current used priority: cost-benefit priority ---- \n");
302 return std::make_unique<PriorityInlineOrder<CostBenefitPriority>>(
FAM,
305 LLVM_DEBUG(
dbgs() <<
" Current used priority: ML priority ---- \n");
306 return std::make_unique<PriorityInlineOrder<MLPriority>>(
FAM, Params);
311std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This is the interface for a simple mod/ref and alias analysis over globals.
static cl::opt< int > ModuleInlinerTopPriorityThreshold("module-inliner-top-priority-threshold", cl::Hidden, cl::init(0), cl::desc("The cost threshold for call sites that get inlined without the " "cost-benefit analysis"))
static cl::opt< InlinePriorityMode > UseInlinePriority("inline-priority-mode", cl::init(InlinePriorityMode::Size), cl::Hidden, cl::desc("Choose the priority mode to use in module inline"), cl::values(clEnumValN(InlinePriorityMode::Size, "size", "Use callee size priority."), clEnumValN(InlinePriorityMode::Cost, "cost", "Use inline cost priority."), clEnumValN(InlinePriorityMode::CostBenefit, "cost-benefit", "Use cost-benefit ratio."), clEnumValN(InlinePriorityMode::ML, "ml", "Use ML.")))
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Class for arbitrary precision integers.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
const Function * getParent() const
Return the enclosing method, or null if none.
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Function * getCaller()
Helper to get the caller (the parent function).
Module * getParent()
Get the module that this global value is contained inside of...
Represents the cost of inlining a function.
virtual void erase_if(function_ref< bool(T)> Pred)=0
virtual void push(const T &Elt)=0
const BasicBlock * getParent() const
A Module instance is used to store all the information related to an LLVM module.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Used for dynamically loading instances of InlineOrder as plugins.
static bool isRegistered()
Analysis providing profile information.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
An efficient, type-erasing, non-owning reference to a callable.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
std::unique_ptr< InlineOrder< std::pair< CallBase *, int > > > getDefaultInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params, ModuleAnalysisManager &MAM, Module &M)
std::unique_ptr< InlineOrder< std::pair< CallBase *, int > > > getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params, ModuleAnalysisManager &MAM, Module &M)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
A special type used by analysis passes to provide an address that identifies that particular analysis...
Thresholds to tune inline cost analysis.