48#define DEBUG_TYPE "pgo-icall-prom"
50STATISTIC(NumOfPGOICallPromotion,
"Number of indirect call promotions.");
51STATISTIC(NumOfPGOICallsites,
"Number of indirect call candidate sites.");
56 cl::desc(
"Disable indirect call promotion"));
64 cl::desc(
"Max number of promotions for this compilation"));
70 cl::desc(
"Skip Callsite up to this number for this compilation"));
76 cl::desc(
"Run indirect-call promotion in LTO "
83 cl::desc(
"Run indirect-call promotion in SamplePGO mode"));
89 cl::desc(
"Run indirect-call promotion for call instructions "
96 cl::desc(
"Run indirect-call promotion for "
97 "invoke instruction only"));
103 cl::desc(
"Dump IR after transformation happens"));
109class IndirectCallPromoter {
117 const bool SamplePGO;
122 struct PromotionCandidate {
134 std::vector<PromotionCandidate> getPromotionCandidatesForCallSite(
141 const std::vector<PromotionCandidate> &Candidates,
147 :
F(
Func), Symtab(Symtab), SamplePGO(SamplePGO), ORE(ORE) {}
148 IndirectCallPromoter(
const IndirectCallPromoter &) =
delete;
149 IndirectCallPromoter &operator=(
const IndirectCallPromoter &) =
delete;
158std::vector<IndirectCallPromoter::PromotionCandidate>
159IndirectCallPromoter::getPromotionCandidatesForCallSite(
162 std::vector<PromotionCandidate>
Ret;
164 LLVM_DEBUG(
dbgs() <<
" \nWork on callsite #" << NumOfPGOICallsites << CB
165 <<
" Num_targets: " << ValueDataRef.
size()
166 <<
" Num_candidates: " << NumCandidates <<
"\n");
167 NumOfPGOICallsites++;
175 assert(Count <= TotalCount);
179 <<
" Target_func: " <<
Target <<
"\n");
185 <<
" Not promote: User options";
193 <<
" Not promote: User options";
201 <<
" Not promote: Cutoff reached";
214 if (TargetFunction ==
nullptr || TargetFunction->
isDeclaration()) {
218 <<
"Cannot promote indirect call: target with md5sum "
224 const char *Reason =
nullptr;
230 <<
"Cannot promote indirect call to "
231 <<
NV(
"TargetFunction", TargetFunction) <<
" with count of "
232 <<
NV(
"Count", Count) <<
": " << Reason;
237 Ret.push_back(PromotionCandidate(TargetFunction, Count));
245 bool AttachProfToDirectCall,
248 uint64_t ElseCount = TotalCount - Count;
249 uint64_t MaxCount = (Count >= ElseCount ? Count : ElseCount);
258 if (AttachProfToDirectCall) {
261 LLVMContext::MD_prof,
270 <<
"Promote indirect call to " << NV(
"DirectCallee", DirectCallee)
271 <<
" with count " << NV(
"Count", Count) <<
" out of "
272 << NV(
"TotalCount", TotalCount);
278uint32_t IndirectCallPromoter::tryToPromote(
279 CallBase &CB,
const std::vector<PromotionCandidate> &Candidates,
283 for (
const auto &
C : Candidates) {
287 assert(TotalCount >= Count);
289 NumOfPGOICallPromotion++;
298 bool Changed =
false;
304 CB, NumVals, TotalCount, NumCandidates);
305 if (!NumCandidates ||
308 auto PromotionCandidates = getPromotionCandidatesForCallSite(
309 *CB, ICallProfDataRef, TotalCount, NumCandidates);
310 uint32_t NumPromoted = tryToPromote(*CB, PromotionCandidates, TotalCount);
311 if (NumPromoted == 0)
318 if (TotalCount == 0 || NumPromoted == NumVals)
322 TotalCount, IPVK_IndirectCallTarget, NumCandidates);
334 std::string SymtabFailure =
toString(std::move(
E));
335 M.getContext().emitError(
"Failed to create symtab: " + SymtabFailure);
338 bool Changed =
false;
340 if (
F.isDeclaration() ||
F.hasOptNone())
347 IndirectCallPromoter CallPromoter(
F, &Symtab, SamplePGO, ORE);
348 bool FuncChanged = CallPromoter.processFunction(PSI);
353 Changed |= FuncChanged;
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Lightweight error class with error context and mandatory checking.
const Function & getFunction() const
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
A Module instance is used to store all the information related to an LLVM module.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
bool hasProfileSummary() const
Returns true if profile summary is available.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
Target - Wrapper for Target specific information.
LLVMContext & getContext() const
All values hold a context through their type.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
DiagnosticInfoOptimizationBase::Argument NV
CallBase & promoteIndirectCall(CallBase &CB, Function *F, uint64_t Count, uint64_t TotalCount, bool AttachProfToDirectCall, OptimizationRemarkEmitter *ORE)
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
std::vector< CallBase * > findIndirectCalls(Function &F)
CallBase & promoteCallWithIfThenElse(CallBase &CB, Function *Callee, MDNode *BranchWeights=nullptr)
Promote the given indirect call site to conditionally call Callee.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
static uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.