48#define DEBUG_TYPE "pgo-icall-prom"
50STATISTIC(NumOfPGOICallPromotion,
"Number of indirect call promotions.");
51STATISTIC(NumOfPGOICallsites,
"Number of indirect call candidate sites.");
56 cl::desc(
"Disable indirect call promotion"));
64 cl::desc(
"Max number of promotions for this compilation"));
70 cl::desc(
"Skip Callsite up to this number for this compilation"));
76 cl::desc(
"Run indirect-call promotion in LTO "
83 cl::desc(
"Run indirect-call promotion in SamplePGO mode"));
89 cl::desc(
"Run indirect-call promotion for call instructions "
96 cl::desc(
"Run indirect-call promotion for "
97 "invoke instruction only"));
103 cl::desc(
"Dump IR after transformation happens"));
109class ICallPromotionFunc {
123 struct PromotionCandidate {
135 std::vector<PromotionCandidate> getPromotionCandidatesForCallSite(
142 const std::vector<PromotionCandidate> &Candidates,
148 :
F(
Func),
M(Modu), Symtab(Symtab), SamplePGO(SamplePGO), ORE(ORE) {}
149 ICallPromotionFunc(
const ICallPromotionFunc &) =
delete;
150 ICallPromotionFunc &operator=(
const ICallPromotionFunc &) =
delete;
159std::vector<ICallPromotionFunc::PromotionCandidate>
160ICallPromotionFunc::getPromotionCandidatesForCallSite(
163 std::vector<PromotionCandidate>
Ret;
165 LLVM_DEBUG(
dbgs() <<
" \nWork on callsite #" << NumOfPGOICallsites << CB
166 <<
" Num_targets: " << ValueDataRef.
size()
167 <<
" Num_candidates: " << NumCandidates <<
"\n");
168 NumOfPGOICallsites++;
176 assert(Count <= TotalCount);
180 <<
" Target_func: " <<
Target <<
"\n");
186 <<
" Not promote: User options";
194 <<
" Not promote: User options";
202 <<
" Not promote: Cutoff reached";
215 if (TargetFunction ==
nullptr || TargetFunction->
isDeclaration()) {
219 <<
"Cannot promote indirect call: target with md5sum "
225 const char *Reason =
nullptr;
231 <<
"Cannot promote indirect call to "
232 <<
NV(
"TargetFunction", TargetFunction) <<
" with count of "
233 <<
NV(
"Count", Count) <<
": " << Reason;
238 Ret.push_back(PromotionCandidate(TargetFunction, Count));
246 bool AttachProfToDirectCall,
249 uint64_t ElseCount = TotalCount - Count;
250 uint64_t MaxCount = (Count >= ElseCount ? Count : ElseCount);
259 if (AttachProfToDirectCall) {
262 LLVMContext::MD_prof,
271 <<
"Promote indirect call to " << NV(
"DirectCallee", DirectCallee)
272 <<
" with count " << NV(
"Count", Count) <<
" out of "
273 << NV(
"TotalCount", TotalCount);
279uint32_t ICallPromotionFunc::tryToPromote(
280 CallBase &CB,
const std::vector<PromotionCandidate> &Candidates,
284 for (
const auto &
C : Candidates) {
288 assert(TotalCount >= Count);
290 NumOfPGOICallPromotion++;
299 bool Changed =
false;
305 CB, NumVals, TotalCount, NumCandidates);
306 if (!NumCandidates ||
309 auto PromotionCandidates = getPromotionCandidatesForCallSite(
310 *CB, ICallProfDataRef, TotalCount, NumCandidates);
311 uint32_t NumPromoted = tryToPromote(*CB, PromotionCandidates, TotalCount);
312 if (NumPromoted == 0)
319 if (TotalCount == 0 || NumPromoted == NumVals)
323 IPVK_IndirectCallTarget, NumCandidates);
330 bool InLTO,
bool SamplePGO,
336 std::string SymtabFailure =
toString(std::move(
E));
337 M.getContext().emitError(
"Failed to create symtab: " + SymtabFailure);
340 bool Changed =
false;
342 if (
F.isDeclaration() ||
F.hasOptNone())
345 std::unique_ptr<OptimizationRemarkEmitter> OwnedORE;
352 OwnedORE = std::make_unique<OptimizationRemarkEmitter>(&
F);
353 ORE = OwnedORE.get();
356 ICallPromotionFunc ICallPromotion(
F, &M, &Symtab, SamplePGO, *ORE);
357 bool FuncChanged = ICallPromotion.processFunction(PSI);
362 Changed |= FuncChanged;
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
FunctionAnalysisManager FAM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Lightweight error class with error context and mandatory checking.
const Function & getFunction() const
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
A symbol table used for function PGO name look-up with keys (such as pointers, md5hash values) to the...
Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
A Module instance is used to store all the information related to an LLVM module.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
bool hasProfileSummary() const
Returns true if profile summary is available.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
Target - Wrapper for Target specific information.
LLVMContext & getContext() const
All values hold a context through their type.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
DiagnosticInfoOptimizationBase::Argument NV
CallBase & promoteIndirectCall(CallBase &CB, Function *F, uint64_t Count, uint64_t TotalCount, bool AttachProfToDirectCall, OptimizationRemarkEmitter *ORE)
This is an optimization pass for GlobalISel generic memory operations.
bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
std::vector< CallBase * > findIndirectCalls(Function &F)
CallBase & promoteCallWithIfThenElse(CallBase &CB, Function *Callee, MDNode *BranchWeights=nullptr)
Promote the given indirect call site to conditionally call Callee.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
static uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.