125#include <unordered_map>
133#define DEBUG_TYPE "pgo-instrumentation"
135STATISTIC(NumOfPGOInstrument,
"Number of edges instrumented.");
136STATISTIC(NumOfPGOSelectInsts,
"Number of select instruction instrumented.");
137STATISTIC(NumOfPGOMemIntrinsics,
"Number of mem intrinsics instrumented.");
140STATISTIC(NumOfPGOSplit,
"Number of critical edge splits.");
141STATISTIC(NumOfPGOFunc,
"Number of functions having valid profile counts.");
142STATISTIC(NumOfPGOMismatch,
"Number of functions having mismatch profile.");
143STATISTIC(NumOfPGOMissing,
"Number of functions without profile.");
144STATISTIC(NumOfPGOICall,
"Number of indirect call value instrumentations.");
145STATISTIC(NumOfCSPGOInstrument,
"Number of edges instrumented in CSPGO.");
147 "Number of select instruction instrumented in CSPGO.");
149 "Number of mem intrinsics instrumented in CSPGO.");
151STATISTIC(NumOfCSPGOBB,
"Number of basic-blocks in CSPGO.");
152STATISTIC(NumOfCSPGOSplit,
"Number of critical edge splits in CSPGO.");
154 "Number of functions having valid profile counts in CSPGO.");
156 "Number of functions having mismatch profile in CSPGO.");
157STATISTIC(NumOfCSPGOMissing,
"Number of functions without profile in CSPGO.");
158STATISTIC(NumCoveredBlocks,
"Number of basic blocks that were executed");
165 cl::desc(
"Specify the path of profile data file. This is"
166 "mainly for test purpose."));
170 cl::desc(
"Specify the path of profile remapping file. This is mainly for "
177 cl::desc(
"Disable Value Profiling"));
183 cl::desc(
"Max number of annotations for a single indirect "
190 cl::desc(
"Max number of preicise value annotations for a single memop"
197 cl::desc(
"Append function hash to the name of COMDAT function to avoid "
198 "function hash mismatch due to the preinliner"));
205 cl::desc(
"Use this option to turn on/off "
206 "warnings about missing profile data for "
213 cl::desc(
"Use this option to turn off/on "
214 "warnings about profile cfg mismatch."));
221 cl::desc(
"The option is used to turn on/off "
222 "warnings about hash mismatch for comdat "
223 "or weak functions."));
229 cl::desc(
"Use this option to turn on/off SELECT "
230 "instruction instrumentation. "));
235 cl::desc(
"A boolean option to show CFG dag or text "
236 "with raw profile counts from "
237 "profile data. See also option "
238 "-pgo-view-counts. To limit graph "
239 "display to only one function, use "
240 "filtering option -view-bfi-func-name."),
248 cl::desc(
"Use this option to turn on/off "
249 "memory intrinsic size profiling."));
254 cl::desc(
"When this option is on, the annotated "
255 "branch probability will be emitted as "
256 "optimization remarks: -{Rpass|"
257 "pass-remarks}=pgo-instrumentation"));
261 cl::desc(
"Force to instrument function entry basicblock."));
266 "Use this option to enable function entry coverage instrumentation."));
269 "pgo-block-coverage",
270 cl::desc(
"Use this option to enable basic block coverage instrumentation"));
274 cl::desc(
"Create a dot file of CFGs with block "
275 "coverage inference information"));
278 "pgo-temporal-instrumentation",
279 cl::desc(
"Use this option to enable temporal instrumentation"));
283 cl::desc(
"Fix function entry count in profile use."));
287 cl::desc(
"Print out the non-match BFI count if a hot raw profile count "
288 "becomes non-hot, or a cold raw profile count becomes hot. "
289 "The print is enabled under -Rpass-analysis=pgo, or "
290 "internal option -pass-remakrs-analysis=pgo."));
294 cl::desc(
"Print out mismatched BFI counts after setting profile metadata "
295 "The print is enabled under -Rpass-analysis=pgo, or "
296 "internal option -pass-remakrs-analysis=pgo."));
300 cl::desc(
"Set the threshold for pgo-verify-bfi: only print out "
301 "mismatched BFI if the difference percentage is greater than "
302 "this value (in percentage)."));
306 cl::desc(
"Set the threshold for pgo-verify-bfi: skip the counts whose "
307 "profile count value is below."));
312 cl::desc(
"Trace the hash of the function with this name."));
316 cl::desc(
"Do not instrument functions smaller than this threshold."));
320 cl::desc(
"Do not instrument functions with the number of critical edges "
321 " greater than this threshold."));
360 return std::string();
365 return std::string();
377 else if (CV->
isOne())
389#define VALUE_PROF_KIND(Enumerator, Value, Descr) Descr,
396 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
398 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
400 ProfileVersion |= VARIANT_MASK_CSIR_PROF;
402 ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
404 ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
407 VARIANT_MASK_BYTE_COVERAGE | VARIANT_MASK_FUNCTION_ENTRY_ONLY;
409 ProfileVersion |= VARIANT_MASK_BYTE_COVERAGE;
411 ProfileVersion |= VARIANT_MASK_TEMPORAL_PROF;
416 Triple TT(M.getTargetTriple());
417 if (TT.supportsCOMDAT()) {
419 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
421 return IRLevelVersionVariable;
431enum VisitMode { VM_counting, VM_instrument, VM_annotate };
435struct SelectInstVisitor :
public InstVisitor<SelectInstVisitor> {
438 VisitMode
Mode = VM_counting;
439 unsigned *CurCtrIdx =
nullptr;
440 unsigned TotalNumCtrs = 0;
443 PGOUseFunc *UseFunc =
nullptr;
444 bool HasSingleByteCoverage;
446 SelectInstVisitor(
Function &Func,
bool HasSingleByteCoverage)
447 :
F(
Func), HasSingleByteCoverage(HasSingleByteCoverage) {}
449 void countSelects() {
459 void instrumentSelects(
unsigned *Ind,
unsigned TotalNC,
GlobalVariable *FNV,
461 Mode = VM_instrument;
463 TotalNumCtrs = TotalNC;
470 void annotateSelects(PGOUseFunc *UF,
unsigned *Ind) {
485 unsigned getNumOfSelectInsts()
const {
return NSIs; }
497 bool Removed =
false;
498 bool IsCritical =
false;
501 : SrcBB(Src), DestBB(Dest), Weight(
W) {}
504 std::string infoString()
const {
505 return (
Twine(Removed ?
"-" :
" ") + (InMST ?
" " :
"*") +
506 (IsCritical ?
"c" :
" ") +
" W=" +
Twine(Weight))
517 PGOBBInfo(
unsigned IX) : Group(this),
Index(IX) {}
520 std::string infoString()
const {
526template <
class Edge,
class BBInfo>
class FuncPGOInstrumentation {
534 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
538 void computeCFGHash();
539 void renameComdatFunction();
543 std::vector<std::vector<VPCandidateInfo>> ValueSites;
544 SelectInstVisitor SIVisitor;
545 std::string FuncName;
546 std::string DeprecatedFuncName;
555 const std::optional<BlockCoverageInference> BCI;
557 static std::optional<BlockCoverageInference>
558 constructBCI(
Function &Func,
bool HasSingleByteCoverage,
559 bool InstrumentFuncEntry) {
560 if (HasSingleByteCoverage)
567 void getInstrumentBBs(std::vector<BasicBlock *> &InstrumentBBs);
580 void dumpInfo(
StringRef Str =
"")
const {
582 " Hash: " +
Twine(FunctionHash) +
"\t" + Str);
585 FuncPGOInstrumentation(
587 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
590 bool InstrumentFuncEntry =
true,
bool HasSingleByteCoverage =
false)
591 :
F(
Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(
Func, TLI),
592 TLI(TLI), ValueSites(IPVK_Last + 1),
593 SIVisitor(
Func, HasSingleByteCoverage),
594 MST(
F, InstrumentFuncEntry, BPI,
BFI),
595 BCI(constructBCI(
Func, HasSingleByteCoverage, InstrumentFuncEntry)) {
597 BCI->viewBlockCoverageGraph();
599 SIVisitor.countSelects();
600 ValueSites[IPVK_MemOPSize] = VPC.
get(IPVK_MemOPSize);
602 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
603 NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
605 ValueSites[IPVK_IndirectCallTarget] = VPC.
get(IPVK_IndirectCallTarget);
607 ValueSites[IPVK_VTableTarget] = VPC.
get(IPVK_VTableTarget);
609 NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
610 NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
617 if (!ComdatMembers.empty())
618 renameComdatFunction();
621 for (
const auto &E : MST.
allEdges()) {
624 IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++;
626 IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++;
639template <
class Edge,
class BBInfo>
640void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
641 std::vector<uint8_t> Indexes;
645 auto BI = findBBInfo(Succ);
649 for (
int J = 0; J < 4; J++)
650 Indexes.push_back((uint8_t)(
Index >> (J * 8)));
657 auto updateJCH = [&JCH](
uint64_t Num) {
662 updateJCH((
uint64_t)SIVisitor.getNumOfSelectInsts());
663 updateJCH((
uint64_t)ValueSites[IPVK_IndirectCallTarget].
size());
666 updateJCH(BCI->getInstrumentedBlocksHash());
676 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
679 LLVM_DEBUG(
dbgs() <<
"Function Hash Computation for " <<
F.getName() <<
":\n"
680 <<
" CRC = " << JC.
getCRC()
681 <<
", Selects = " << SIVisitor.getNumOfSelectInsts()
682 <<
", Edges = " << MST.
numEdges() <<
", ICSites = "
683 << ValueSites[IPVK_IndirectCallTarget].size()
684 <<
", Memops = " << ValueSites[IPVK_MemOPSize].size()
685 <<
", High32 CRC = " << JCH.
getCRC()
686 <<
", Hash = " << FunctionHash <<
"\n";);
689 dbgs() <<
"Funcname=" <<
F.getName() <<
", Hash=" << FunctionHash
690 <<
" in building " <<
F.getParent()->getSourceFileName() <<
"\n";
696 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
708 for (
auto &&CM :
make_range(ComdatMembers.equal_range(
C))) {
709 assert(!isa<GlobalAlias>(CM.second));
710 Function *FM = dyn_cast<Function>(CM.second);
718template <
class Edge,
class BBInfo>
719void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
722 std::string OrigName =
F.getName().str();
723 std::string NewFuncName =
725 F.setName(
Twine(NewFuncName));
727 FuncName =
Twine(FuncName +
"." +
Twine(FunctionHash)).
str();
733 if (!
F.hasComdat()) {
735 NewComdat =
M->getOrInsertComdat(
StringRef(NewFuncName));
737 F.setComdat(NewComdat);
742 Comdat *OrigComdat =
F.getComdat();
743 std::string NewComdatName =
745 NewComdat =
M->getOrInsertComdat(
StringRef(NewComdatName));
748 for (
auto &&CM :
make_range(ComdatMembers.equal_range(OrigComdat))) {
750 cast<Function>(CM.second)->setComdat(NewComdat);
756template <
class Edge,
class BBInfo>
757void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(
758 std::vector<BasicBlock *> &InstrumentBBs) {
761 if (BCI->shouldInstrumentBlock(BB))
762 InstrumentBBs.push_back(&BB);
767 std::vector<Edge *> EdgeList;
769 for (
const auto &E : MST.
allEdges())
770 EdgeList.push_back(E.get());
772 for (
auto &E : EdgeList) {
775 InstrumentBBs.push_back(InstrBB);
781template <
class Edge,
class BBInfo>
782BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
783 if (E->InMST || E->Removed)
789 if (SrcBB ==
nullptr)
791 if (DestBB ==
nullptr)
806 return canInstrument(SrcBB);
808 return canInstrument(DestBB);
817 dbgs() <<
"Fail to split critical edge: not instrument this edge.\n");
822 IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;
824 <<
" --> " << getBBInfo(DestBB).
Index <<
"\n");
826 MST.
addEdge(SrcBB, InstrBB, 0);
828 Edge &NewEdge1 = MST.
addEdge(InstrBB, DestBB, 0);
829 NewEdge1.InMST =
true;
832 return canInstrument(InstrBB);
848 if (!isa<IntrinsicInst>(OrigCall)) {
851 std::optional<OperandBundleUse> ParentFunclet =
859 if (!BlockColors.
empty()) {
860 const ColorVector &CV = BlockColors.
find(OrigCall->getParent())->second;
861 assert(CV.
size() == 1 &&
"non-unique color for block!");
874 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
882 FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(
886 auto Name = FuncInfo.FuncNameVar;
888 FuncInfo.FunctionHash);
890 auto &EntryBB =
F.getEntryBlock();
891 IRBuilder<> Builder(&EntryBB, EntryBB.getFirstInsertionPt());
900 std::vector<BasicBlock *> InstrumentBBs;
901 FuncInfo.getInstrumentBBs(InstrumentBBs);
902 unsigned NumCounters =
903 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
918 for (
auto &Instr : BB)
919 if (
auto *CS = dyn_cast<CallBase>(&Instr)) {
920 if ((CS->getCalledFunction() &&
921 CS->getCalledFunction()->isIntrinsic()) ||
922 dyn_cast<InlineAsm>(CS->getCalledOperand()))
929 Visit([&TotalNrCallsites](
auto *) { ++TotalNrCallsites; });
933 Visit([&](
auto *CB) {
945 auto &EntryBB =
F.getEntryBlock();
946 IRBuilder<> Builder(&EntryBB, EntryBB.getFirstInsertionPt());
955 for (
auto *InstrBB : InstrumentBBs) {
958 "Cannot get the Instrumentation point");
963 ? Intrinsic::instrprof_cover
964 : Intrinsic::instrprof_increment),
969 FuncInfo.SIVisitor.instrumentSelects(&
I, NumCounters, FuncInfo.FuncNameVar,
970 FuncInfo.FunctionHash);
976 NumOfPGOICall += FuncInfo.ValueSites[IPVK_IndirectCallTarget].size();
983 if (
F.hasPersonalityFn() &&
988 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
989 unsigned SiteIndex = 0;
995 <<
" site: CallSite Index = " << SiteIndex <<
"\n");
999 "Cannot get the Instrumentation point");
1001 Value *ToProfile =
nullptr;
1002 if (Cand.V->getType()->isIntegerTy())
1004 else if (Cand.V->getType()->isPointerTy())
1006 assert(ToProfile &&
"value profiling Value is of unexpected type");
1012 {FuncInfo.FuncNameVar, Builder.
getInt64(FuncInfo.FunctionHash),
1022struct PGOUseEdge :
public PGOEdge {
1023 using PGOEdge::PGOEdge;
1025 std::optional<uint64_t> Count;
1031 std::string infoString()
const {
1033 return PGOEdge::infoString();
1034 return (
Twine(PGOEdge::infoString()) +
" Count=" +
Twine(*Count)).str();
1041struct PGOUseBBInfo :
public PGOBBInfo {
1042 std::optional<uint64_t> Count;
1043 int32_t UnknownCountInEdge = 0;
1044 int32_t UnknownCountOutEdge = 0;
1045 DirectEdges InEdges;
1046 DirectEdges OutEdges;
1048 PGOUseBBInfo(
unsigned IX) : PGOBBInfo(IX) {}
1054 std::string infoString()
const {
1056 return PGOBBInfo::infoString();
1057 return (
Twine(PGOBBInfo::infoString()) +
" Count=" +
Twine(*Count)).str();
1061 void addOutEdge(PGOUseEdge *E) {
1062 OutEdges.push_back(E);
1063 UnknownCountOutEdge++;
1067 void addInEdge(PGOUseEdge *E) {
1068 InEdges.push_back(E);
1069 UnknownCountInEdge++;
1078 for (
const auto &E : Edges) {
1092 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
1095 bool HasSingleByteCoverage)
1096 :
F(
Func),
M(Modu),
BFI(BFIin), PSI(PSI),
1097 FuncInfo(
Func, TLI, ComdatMembers,
false, BPI, BFIin, IsCS,
1098 InstrumentFuncEntry, HasSingleByteCoverage),
1099 FreqAttr(FFA_Normal), IsCS(IsCS), VPC(
Func, TLI) {}
1101 void handleInstrProfError(
Error Err,
uint64_t MismatchedFuncSum);
1108 void populateCounters();
1117 void annotateValueSites();
1120 void annotateValueSites(
uint32_t Kind);
1123 void annotateIrrLoopHeaderWeights();
1126 enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };
1129 FuncFreqAttr getFuncFreqAttr()
const {
return FreqAttr; }
1138 PGOUseBBInfo &getBBInfo(
const BasicBlock *BB)
const {
1139 return FuncInfo.getBBInfo(BB);
1143 PGOUseBBInfo *findBBInfo(
const BasicBlock *BB)
const {
1144 return FuncInfo.findBBInfo(BB);
1149 void dumpInfo(
StringRef Str =
"")
const { FuncInfo.dumpInfo(Str); }
1151 uint64_t getProgramMaxCount()
const {
return ProgramMaxCount; }
1160 FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> FuncInfo;
1176 FuncFreqAttr FreqAttr;
1184 bool setInstrumentedCounts(
const std::vector<uint64_t> &CountFromProfile);
1197 FreqAttr = FFA_Cold;
1205 const FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> &FuncInfo) {
1209 for (
const auto &E : FuncInfo.MST.allEdges()) {
1214 PGOUseBBInfo &SrcInfo = FuncInfo.getBBInfo(SrcBB);
1215 PGOUseBBInfo &DestInfo = FuncInfo.getBBInfo(DestBB);
1216 SrcInfo.addOutEdge(E.get());
1217 DestInfo.addInEdge(E.get());
1223bool PGOUseFunc::setInstrumentedCounts(
1224 const std::vector<uint64_t> &CountFromProfile) {
1226 std::vector<BasicBlock *> InstrumentBBs;
1227 FuncInfo.getInstrumentBBs(InstrumentBBs);
1231 unsigned NumCounters =
1232 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
1235 if (NumCounters != CountFromProfile.size()) {
1238 auto *FuncEntry = &*
F.begin();
1243 uint64_t CountValue = CountFromProfile[
I++];
1244 PGOUseBBInfo &
Info = getBBInfo(InstrBB);
1248 if (InstrBB == FuncEntry && CountValue == 0)
1250 Info.setBBInfoCount(CountValue);
1252 ProfileCountSize = CountFromProfile.size();
1256 auto setEdgeCount = [
this](PGOUseEdge *E,
uint64_t Value) ->
void {
1257 E->setEdgeCount(
Value);
1258 this->getBBInfo(E->SrcBB).UnknownCountOutEdge--;
1259 this->getBBInfo(E->DestBB).UnknownCountInEdge--;
1265 for (
const auto &E : FuncInfo.MST.allEdges()) {
1266 if (E->Removed || E->InMST)
1269 PGOUseBBInfo &SrcInfo = getBBInfo(SrcBB);
1273 if (SrcInfo.Count && SrcInfo.OutEdges.size() == 1)
1274 setEdgeCount(E.get(), *SrcInfo.Count);
1277 PGOUseBBInfo &DestInfo = getBBInfo(DestBB);
1280 if (DestInfo.Count && DestInfo.InEdges.size() == 1)
1281 setEdgeCount(E.get(), *DestInfo.Count);
1287 setEdgeCount(E.get(), 0);
1294void PGOUseFunc::setEdgeCount(DirectEdges &Edges,
uint64_t Value) {
1295 for (
auto &E : Edges) {
1298 E->setEdgeCount(
Value);
1300 getBBInfo(E->SrcBB).UnknownCountOutEdge--;
1301 getBBInfo(E->DestBB).UnknownCountInEdge--;
1309 const char MetadataName[] =
"instr_prof_hash_mismatch";
1312 auto *Existing =
F.getMetadata(LLVMContext::MD_annotation);
1314 MDTuple *Tuple = cast<MDTuple>(Existing);
1315 for (
const auto &
N : Tuple->
operands()) {
1316 if (
N.equalsStr(MetadataName))
1325 F.setMetadata(LLVMContext::MD_annotation, MD);
1328void PGOUseFunc::handleInstrProfError(
Error Err,
uint64_t MismatchedFuncSum) {
1330 auto &Ctx =
M->getContext();
1331 auto Err = IPE.
get();
1332 bool SkipWarning =
false;
1334 << FuncInfo.FuncName <<
": ");
1335 if (Err == instrprof_error::unknown_function) {
1336 IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++;
1339 }
else if (Err == instrprof_error::hash_mismatch ||
1340 Err == instrprof_error::malformed) {
1341 IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++;
1347 LLVM_DEBUG(
dbgs() <<
"hash mismatch (hash= " << FuncInfo.FunctionHash
1348 <<
" skip=" << SkipWarning <<
")");
1358 IPE.
message() + std::string(
" ") +
F.getName().str() +
1359 std::string(
" Hash = ") + std::to_string(FuncInfo.FunctionHash) +
1360 std::string(
" up to ") + std::to_string(MismatchedFuncSum) +
1361 std::string(
" count discarded");
1373 auto &Ctx =
M->getContext();
1376 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,
1377 &MismatchedFuncSum);
1379 handleInstrProfError(std::move(E), MismatchedFuncSum);
1382 ProfileRecord = std::move(
Result.get());
1387 std::vector<uint64_t> &CountFromProfile = ProfileRecord.
Counts;
1389 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
1393 for (
unsigned I = 0, S = CountFromProfile.size();
I < S;
I++) {
1395 ValueSum += CountFromProfile[
I];
1397 AllZeros = (ValueSum == 0);
1401 getBBInfo(
nullptr).UnknownCountOutEdge = 2;
1402 getBBInfo(
nullptr).UnknownCountInEdge = 2;
1404 if (!setInstrumentedCounts(CountFromProfile)) {
1406 dbgs() <<
"Inconsistent number of counts, skipping this function");
1408 M->getName().data(),
1409 Twine(
"Inconsistent number of counts in ") +
F.getName().str() +
1410 Twine(
": the profile may be stale or there is a function name "
1422 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,
1423 &MismatchedFuncSum);
1424 if (
auto Err =
Result.takeError()) {
1425 handleInstrProfError(std::move(Err), MismatchedFuncSum);
1428 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
1430 std::vector<uint64_t> &CountsFromProfile =
Result.get().Counts;
1434 if (FuncInfo.BCI->shouldInstrumentBlock(BB))
1440 InverseDependencies;
1441 for (
auto &BB :
F) {
1442 for (
auto *Dep : FuncInfo.BCI->getDependencies(BB)) {
1444 InverseDependencies[Dep].
insert(&BB);
1449 std::stack<const BasicBlock *> CoveredBlocksToProcess;
1450 for (
auto &[BB, IsCovered] : Coverage)
1452 CoveredBlocksToProcess.push(BB);
1454 while (!CoveredBlocksToProcess.empty()) {
1455 auto *CoveredBlock = CoveredBlocksToProcess.top();
1456 assert(Coverage[CoveredBlock]);
1457 CoveredBlocksToProcess.pop();
1458 for (
auto *BB : InverseDependencies[CoveredBlock]) {
1463 CoveredBlocksToProcess.push(BB);
1471 F.setEntryCount(Coverage[&
F.getEntryBlock()] ? 10000 : 0);
1472 for (
auto &BB :
F) {
1481 Weights.
push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0);
1482 if (Weights.
size() >= 2)
1487 unsigned NumCorruptCoverage = 0;
1492 auto IsBlockDead = [&](
const BasicBlock &BB) -> std::optional<bool> {
1493 if (
auto C =
BFI.getBlockProfileCount(&BB))
1497 LLVM_DEBUG(
dbgs() <<
"Block Coverage: (Instrumented=*, Covered=X)\n");
1498 for (
auto &BB :
F) {
1499 LLVM_DEBUG(
dbgs() << (FuncInfo.BCI->shouldInstrumentBlock(BB) ?
"* " :
" ")
1500 << (Coverage[&BB] ?
"X " :
" ") <<
" " << BB.getName()
1506 if (Coverage[&BB] == IsBlockDead(BB).value_or(
false)) {
1508 dbgs() <<
"Found inconsistent block covearge for " << BB.getName()
1509 <<
": BCI=" << (Coverage[&BB] ?
"Covered" :
"Dead") <<
" BFI="
1510 << (IsBlockDead(BB).
value() ?
"Dead" :
"Covered") <<
"\n");
1511 ++NumCorruptCoverage;
1517 auto &Ctx =
M->getContext();
1519 M->getName().data(),
1520 Twine(
"Found inconsistent block coverage for function ") +
F.getName() +
1521 " in " +
Twine(NumCorruptCoverage) +
" blocks.",
1525 FuncInfo.BCI->viewBlockCoverageGraph(&Coverage);
1530void PGOUseFunc::populateCounters() {
1531 bool Changes =
true;
1532 unsigned NumPasses = 0;
1540 PGOUseBBInfo *UseBBInfo = findBBInfo(&BB);
1541 if (UseBBInfo ==
nullptr)
1543 if (!UseBBInfo->Count) {
1544 if (UseBBInfo->UnknownCountOutEdge == 0) {
1547 }
else if (UseBBInfo->UnknownCountInEdge == 0) {
1552 if (UseBBInfo->Count) {
1553 if (UseBBInfo->UnknownCountOutEdge == 1) {
1559 if (*UseBBInfo->Count > OutSum)
1560 Total = *UseBBInfo->Count - OutSum;
1561 setEdgeCount(UseBBInfo->OutEdges,
Total);
1564 if (UseBBInfo->UnknownCountInEdge == 1) {
1567 if (*UseBBInfo->Count > InSum)
1568 Total = *UseBBInfo->Count - InSum;
1569 setEdgeCount(UseBBInfo->InEdges,
Total);
1576 LLVM_DEBUG(
dbgs() <<
"Populate counts in " << NumPasses <<
" passes.\n");
1580 for (
auto &BB :
F) {
1581 auto BI = findBBInfo(&BB);
1584 assert(BI->Count &&
"BB count is not valid");
1589 for (
auto &BB :
F) {
1590 auto BI = findBBInfo(&BB);
1593 FuncMaxCount = std::max(FuncMaxCount, *BI->Count);
1603 FuncInfo.SIVisitor.annotateSelects(
this, &CountPosition);
1604 assert(CountPosition == ProfileCountSize);
1606 LLVM_DEBUG(FuncInfo.dumpInfo(
"after reading profile."));
1610void PGOUseFunc::setBranchWeights() {
1612 LLVM_DEBUG(
dbgs() <<
"\nSetting branch weights for func " <<
F.getName()
1613 <<
" IsCS=" << IsCS <<
"\n");
1614 for (
auto &BB :
F) {
1618 if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
1619 isa<IndirectBrInst>(TI) || isa<InvokeInst>(TI) ||
1620 isa<CallBrInst>(TI)))
1623 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);
1624 if (!*BBCountInfo.Count)
1628 unsigned Size = BBCountInfo.OutEdges.size();
1631 for (
unsigned s = 0; s <
Size; s++) {
1632 const PGOUseEdge *E = BBCountInfo.OutEdges[s];
1635 if (DestBB ==
nullptr)
1639 if (EdgeCount > MaxCount)
1640 MaxCount = EdgeCount;
1641 EdgeCounts[SuccNum] = EdgeCount;
1650 auto &Ctx =
M->getContext();
1652 M->getName().data(),
1653 Twine(
"Profile in ") +
F.getName().str() +
1654 Twine(
" partially ignored") +
1655 Twine(
", possibly due to the lack of a return path."),
1663 if (isa<IndirectBrInst>(Pred->getTerminator()))
1669void PGOUseFunc::annotateIrrLoopHeaderWeights() {
1670 LLVM_DEBUG(
dbgs() <<
"\nAnnotating irreducible loop header weights.\n");
1672 for (
auto &BB :
F) {
1678 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);
1684void SelectInstVisitor::instrumentOneSelectInst(
SelectInst &SI) {
1688 auto *Step = Builder.CreateZExt(
SI.getCondition(), Int64Ty);
1691 {FuncNameVar, Builder.getInt64(FuncHash), Builder.getInt32(TotalNumCtrs),
1692 Builder.getInt32(*CurCtrIdx), Step});
1696void SelectInstVisitor::annotateOneSelectInst(
SelectInst &SI) {
1697 std::vector<uint64_t> &CountFromProfile = UseFunc->getProfileRecord().Counts;
1698 assert(*CurCtrIdx < CountFromProfile.size() &&
1699 "Out of bound access of counters");
1701 SCounts[0] = CountFromProfile[*CurCtrIdx];
1704 auto BI = UseFunc->findBBInfo(
SI.getParent());
1706 TotalCount = *BI->Count;
1708 SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);
1709 uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);
1714void SelectInstVisitor::visitSelectInst(
SelectInst &SI) {
1718 if (
SI.getCondition()->getType()->isVectorTy())
1726 instrumentOneSelectInst(SI);
1729 annotateOneSelectInst(SI);
1737 if (ValueProfKind == IPVK_MemOPSize)
1739 if (ValueProfKind == llvm::IPVK_VTableTarget)
1745void PGOUseFunc::annotateValueSites() {
1753 annotateValueSites(Kind);
1757void PGOUseFunc::annotateValueSites(
uint32_t Kind) {
1758 assert(Kind <= IPVK_Last);
1759 unsigned ValueSiteIndex = 0;
1772 if (NumValueSites > 0 && Kind == IPVK_VTableTarget &&
1773 NumValueSites != FuncInfo.ValueSites[IPVK_VTableTarget].size() &&
1775 FuncInfo.ValueSites[IPVK_VTableTarget] = VPC.
get(IPVK_VTableTarget);
1776 auto &ValueSites = FuncInfo.ValueSites[
Kind];
1777 if (NumValueSites != ValueSites.size()) {
1778 auto &Ctx =
M->getContext();
1780 M->getName().data(),
1781 Twine(
"Inconsistent number of value sites for ") +
1784 Twine(
"\", possibly due to the use of a stale profile."),
1790 LLVM_DEBUG(
dbgs() <<
"Read one value site profile (kind = " << Kind
1791 <<
"): Index = " << ValueSiteIndex <<
" out of "
1792 << NumValueSites <<
"\n");
1794 *M, *
I.AnnotatedInst, ProfileRecord,
1805 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
1810 ComdatMembers.insert(std::make_pair(
C, &
F));
1812 if (
Comdat *
C = GV.getComdat())
1813 ComdatMembers.insert(std::make_pair(
C, &GV));
1815 if (
Comdat *
C = GA.getComdat())
1816 ComdatMembers.insert(std::make_pair(
C, &GA));
1821 if (
F.isDeclaration())
1826 unsigned NumCriticalEdges = 0;
1827 for (
auto &BB :
F) {
1836 <<
", NumCriticalEdges=" << NumCriticalEdges
1837 <<
" exceed the threshold. Skip PGO.\n");
1847 if (
F.hasFnAttribute(llvm::Attribute::Naked))
1849 if (
F.hasFnAttribute(llvm::Attribute::NoProfile))
1851 if (
F.hasFnAttribute(llvm::Attribute::SkipProfile))
1867 Triple TT(M.getTargetTriple());
1872 Twine(
"VTable value profiling is presently not "
1873 "supported for non-ELF object formats"),
1875 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1881 auto &TLI = LookupTLI(
F);
1882 auto *BPI = LookupBPI(
F);
1883 auto *BFI = LookupBFI(
F);
1895 if (ProfileSampling)
1929 auto BFIEntryCount =
F.getEntryCount();
1930 assert(BFIEntryCount && (BFIEntryCount->getCount() > 0) &&
1931 "Invalid BFI Entrycount");
1935 for (
auto &BBI :
F) {
1938 if (!Func.findBBInfo(&BBI))
1941 CountValue = *Func.getBBInfo(&BBI).Count;
1942 BFICountValue = *BFICount;
1946 if (SumCount.isZero())
1950 "Incorrect sum of BFI counts");
1953 double Scale = (SumCount / SumBFICount).convertToDouble();
1954 if (Scale < 1.001 && Scale > 0.999)
1959 if (NewEntryCount == 0)
1965 << NewEntryCount <<
"\n");
1982 unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;
1983 for (
auto &BBI :
F) {
1987 CountValue = Func.getBBInfo(&BBI).Count.value_or(CountValue);
1994 BFICountValue = *BFICount;
1997 bool rawIsHot = CountValue >= HotCountThreshold;
1998 bool BFIIsHot = BFICountValue >= HotCountThreshold;
2000 bool ShowCount =
false;
2001 if (rawIsHot && !BFIIsHot) {
2002 Msg =
"raw-Hot to BFI-nonHot";
2004 }
else if (rawIsCold && BFIIsHot) {
2005 Msg =
"raw-Cold to BFI-Hot";
2014 uint64_t Diff = (BFICountValue >= CountValue)
2015 ? BFICountValue - CountValue
2016 : CountValue - BFICountValue;
2024 F.getSubprogram(), &BBI);
2026 <<
" Count=" <<
ore::NV(
"Count", CountValue)
2027 <<
" BFI_Count=" <<
ore::NV(
"Count", BFICountValue);
2029 Remark <<
" (" << Msg <<
")";
2036 F.getSubprogram(), &
F.getEntryBlock())
2037 <<
"In Func " <<
ore::NV(
"Function",
F.getName())
2038 <<
": Num_of_BB=" <<
ore::NV(
"Count", BBNum)
2039 <<
", Num_of_non_zerovalue_BB=" <<
ore::NV(
"Count", NonZeroBBNum)
2040 <<
", Num_of_mis_matching_BB=" <<
ore::NV(
"Count", BBMisMatchNum);
2052 auto &Ctx = M.getContext();
2055 ProfileRemappingFileName);
2056 if (
Error E = ReaderOrErr.takeError()) {
2064 std::unique_ptr<IndexedInstrProfReader> PGOReader =
2065 std::move(ReaderOrErr.get());
2071 if (!PGOReader->hasCSIRLevelProfile() && IsCS)
2075 if (!PGOReader->isIRLevelProfile()) {
2077 ProfileFileName.
data(),
"Not an IR level instrumentation profile"));
2080 if (PGOReader->functionEntryOnly()) {
2082 ProfileFileName.
data(),
2083 "Function entry profiles are not yet supported for optimization"));
2089 if (!
G.hasName() || !
G.hasMetadata(LLVMContext::MD_type))
2100 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
2105 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
2107 std::vector<Function *> HotFunctions;
2108 std::vector<Function *> ColdFunctions;
2112 bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
2117 bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();
2121 auto &TLI = LookupTLI(
F);
2122 auto *BPI = LookupBPI(
F);
2123 auto *BFI = LookupBFI(
F);
2124 if (!HasSingleByteCoverage) {
2130 PGOUseFunc Func(
F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS,
2131 InstrumentFuncEntry, HasSingleByteCoverage);
2132 if (HasSingleByteCoverage) {
2133 Func.populateCoverage(PGOReader.get());
2141 bool AllZeros =
false;
2142 if (!Func.readCounters(PGOReader.get(), AllZeros, PseudoKind))
2146 if (Func.getProgramMaxCount() != 0)
2147 ColdFunctions.push_back(&
F);
2152 if (
F.hasFnAttribute(Attribute::Cold))
2153 F.removeFnAttr(Attribute::Cold);
2156 F.addFnAttr(Attribute::Hot);
2159 Func.populateCounters();
2160 Func.setBranchWeights();
2161 Func.annotateValueSites();
2162 Func.annotateIrrLoopHeaderWeights();
2163 PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();
2164 if (FreqAttr == PGOUseFunc::FFA_Cold)
2165 ColdFunctions.push_back(&
F);
2166 else if (FreqAttr == PGOUseFunc::FFA_Hot)
2167 HotFunctions.push_back(&
F);
2172 std::unique_ptr<BranchProbabilityInfo> NewBPI =
2173 std::make_unique<BranchProbabilityInfo>(
F, LI);
2174 std::unique_ptr<BlockFrequencyInfo> NewBFI =
2175 std::make_unique<BlockFrequencyInfo>(
F, *NewBPI, LI);
2179 dbgs() <<
"pgo-view-counts: " << Func.getFunc().getName() <<
"\n";
2180 NewBFI->print(
dbgs());
2190 ViewGraph(&Func,
Twine(
"PGORawCounts_") + Func.getFunc().getName());
2192 dbgs() <<
"pgo-view-raw-counts: " << Func.getFunc().getName() <<
"\n";
2219 for (
auto &
F : HotFunctions) {
2220 F->addFnAttr(Attribute::InlineHint);
2221 LLVM_DEBUG(
dbgs() <<
"Set inline attribute to function: " <<
F->getName()
2224 for (
auto &
F : ColdFunctions) {
2227 if (
F->hasFnAttribute(Attribute::Hot)) {
2228 auto &Ctx = M.getContext();
2229 std::string Msg = std::string(
"Function ") +
F->getName().str() +
2230 std::string(
" is annotated as a hot function but"
2231 " the profile is cold");
2236 F->addFnAttr(Attribute::Cold);
2237 LLVM_DEBUG(
dbgs() <<
"Set cold attribute to function: " <<
F->getName()
2244 std::string Filename, std::string RemappingFilename,
bool IsCS,
2246 : ProfileFileName(
std::
move(Filename)),
2247 ProfileRemappingFileName(
std::
move(RemappingFilename)), IsCS(IsCS),
2273 LookupTLI, LookupBPI, LookupBFI, PSI, IsCS))
2280 if (!
Node->getName().empty())
2281 return Node->getName().str();
2283 std::string SimpleNodeName;
2286 return SimpleNodeName;
2291 assert(MaxCount > 0 &&
"Bad max count");
2294 for (
const auto &ECI : EdgeCounts)
2307 if (BrCondStr.empty())
2319 std::string BranchProbStr;
2322 OS <<
" (total count : " << TotalCount <<
")";
2328 << BrCondStr <<
" is true with probability : " << BranchProbStr;
2347 return &
G->getFunc().front();
2370 return std::string(
G->getFunc().getName());
2378 PGOUseBBInfo *BI = Graph->findBBInfo(Node);
2380 if (BI && BI->Count)
2381 OS << *BI->Count <<
"\\l";
2389 if (!isa<SelectInst>(&
I))
2392 OS <<
"SELECT : { T = ";
2396 OS <<
"Unknown, F = Unknown }\\l";
2398 OS << TC <<
", F = " << FC <<
" }\\l";
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
This file finds the minimum set of blocks on a CFG that must be instrumented to infer execution cover...
Analysis containing CSE Info
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Given that RA is a live value
static BasicBlock * getInstrBB(CFGMST< Edge, BBInfo > &MST, Edge &E, const DenseSet< const BasicBlock * > &ExecBlocks)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)
Module.h This file contains the declarations for the Module class.
static cl::opt< bool > PGOInstrumentEntry("pgo-instrument-entry", cl::init(false), cl::Hidden, cl::desc("Force to instrument function entry basicblock."))
static cl::opt< std::string > PGOTestProfileRemappingFile("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))
static cl::opt< bool > PGOFixEntryCount("pgo-fix-entry-count", cl::init(true), cl::Hidden, cl::desc("Fix function entry count in profile use."))
static void fixFuncEntryCount(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI)
static cl::opt< bool > PGOVerifyHotBFI("pgo-verify-hot-bfi", cl::init(false), cl::Hidden, cl::desc("Print out the non-match BFI count if a hot raw profile count " "becomes non-hot, or a cold raw profile count becomes hot. " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remakrs-analysis=pgo."))
static void annotateFunctionWithHashMismatch(Function &F, LLVMContext &ctx)
cl::opt< unsigned > MaxNumVTableAnnotations
static cl::opt< bool > PGOTemporalInstrumentation("pgo-temporal-instrumentation", cl::desc("Use this option to enable temporal instrumentation"))
static cl::opt< unsigned > PGOFunctionSizeThreshold("pgo-function-size-threshold", cl::Hidden, cl::desc("Do not instrument functions smaller than this threshold."))
static cl::opt< unsigned > MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::desc("Max number of annotations for a single indirect " "call callsite"))
static bool skipPGOGen(const Function &F)
static void collectComdatMembers(Module &M, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
static cl::opt< unsigned > PGOVerifyBFICutoff("pgo-verify-bfi-cutoff", cl::init(5), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose " "profile count value is below."))
static cl::opt< std::string > PGOTraceFuncHash("pgo-trace-func-hash", cl::init("-"), cl::Hidden, cl::value_desc("function name"), cl::desc("Trace the hash of the function with this name."))
static void instrumentOneFunc(Function &F, Module *M, TargetLibraryInfo &TLI, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers, bool IsCS)
bool isValueProfilingDisabled()
static void populateEHOperandBundle(VPCandidateInfo &Cand, DenseMap< BasicBlock *, ColorVector > &BlockColors, SmallVectorImpl< OperandBundleDef > &OpBundles)
static cl::opt< bool > PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))
static cl::opt< bool > PGOFunctionEntryCoverage("pgo-function-entry-coverage", cl::Hidden, cl::desc("Use this option to enable function entry coverage instrumentation."))
static bool InstrumentAllFunctions(Module &M, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS)
static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI, uint64_t HotCountThreshold, uint64_t ColdCountThreshold)
static cl::opt< unsigned > PGOVerifyBFIRatio("pgo-verify-bfi-ratio", cl::init(2), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: only print out " "mismatched BFI if the difference percentage is greater than " "this value (in percentage)."))
static cl::opt< bool > DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))
static cl::opt< unsigned > PGOFunctionCriticalEdgeThreshold("pgo-critical-edge-threshold", cl::init(20000), cl::Hidden, cl::desc("Do not instrument functions with the number of critical edges " " greater than this threshold."))
static void setupBBInfoEdges(const FuncPGOInstrumentation< PGOUseEdge, PGOUseBBInfo > &FuncInfo)
Set up InEdges/OutEdges for all BBs in the MST.
static cl::opt< std::string > PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose."))
static bool skipPGOUse(const Function &F)
static bool canRenameComdat(Function &F, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
static cl::opt< bool > PGOVerifyBFI("pgo-verify-bfi", cl::init(false), cl::Hidden, cl::desc("Print out mismatched BFI counts after setting profile metadata " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remakrs-analysis=pgo."))
static cl::opt< bool > PGOBlockCoverage("pgo-block-coverage", cl::desc("Use this option to enable basic block coverage instrumentation"))
static uint64_t sumEdgeCount(const ArrayRef< PGOUseEdge * > Edges)
static cl::opt< bool > PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))
static uint32_t getMaxNumAnnotations(InstrProfValueKind ValueProfKind)
Function::ProfileCount ProfileCount
static cl::opt< bool > EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))
static cl::opt< unsigned > MaxNumMemOPAnnotations("memop-max-annotations", cl::init(4), cl::Hidden, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic"))
static cl::opt< bool > DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
static std::string getSimpleNodeName(const BasicBlock *Node)
static cl::opt< bool > PGOViewBlockCoverageGraph("pgo-view-block-coverage-graph", cl::desc("Create a dot file of CFGs with block " "coverage inference information"))
static GlobalVariable * createIRLevelProfileFlagVar(Module &M, bool IsCS)
static bool isIndirectBrTarget(BasicBlock *BB)
static std::string getBranchCondString(Instruction *TI)
bool shouldInstrumentEntryBB()
static bool annotateAllFunctions(Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, vfs::FileSystem &FS, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, ProfileSummaryInfo *PSI, bool IsCS)
static cl::opt< PGOViewCountsType > PGOViewRawCounts("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
static const char * ValueProfKindDescr[]
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Defines the virtual file system interface vfs::FileSystem.
void printAsOperand(OutputBuffer &OB, Prec P=Prec::Default, bool StrictlyWorse=false) const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Class for arbitrary precision integers.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
std::optional< uint64_t > getBlockProfileCount(const BasicBlock *BB, bool AllowSynthetic=false) const
Returns the estimated profile count of BB.
Conditional or Unconditional Branch instruction.
bool isConditional() const
Value * getCondition() const
Analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
An union-find based Minimum Spanning Tree for CFG.
Edge & addEdge(BasicBlock *Src, BasicBlock *Dest, uint64_t W)
const std::vector< std::unique_ptr< Edge > > & allEdges() const
size_t bbInfoSize() const
BBInfo * findBBInfo(const BasicBlock *BB) const
BBInfo & getBBInfo(const BasicBlock *BB) const
void dumpEdges(raw_ostream &OS, const Twine &Message) const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Value * getCalledOperand() const
Predicate getPredicate() const
Return the predicate for this instruction.
StringRef getName() const
void setSelectionKind(SelectionKind Val)
SelectionKind getSelectionKind() const
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Base class for error info classes.
virtual std::string message() const
Return the error message as a string.
Lightweight error class with error context and mandatory checking.
Tagged union holding either a T or a Error.
Class to represent profile counts.
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
@ HiddenVisibility
The GV is hidden.
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
This instruction compares its operands according to the predicate given to the constructor.
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
BasicBlock::iterator GetInsertPoint() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Reader for the indexed binary instrprof format.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
uint64_t getMaximumFunctionCount(bool UseCS)
Return the maximum of all known function counts.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
RetTy visitSelectInst(SelectInst &I)
instrprof_error get() const
std::string message() const override
Return the error message as a string.
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
void update(ArrayRef< uint8_t > Data)
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
MDString * createString(StringRef Str)
Return the given string as metadata.
MDNode * createIrrLoopHeaderWeight(uint64_t Weight)
Return metadata containing an irreducible loop header weight.
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
static bool isContextualIRPGOEnabled()
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
PGOInstrumentationUse(std::string Filename="", std::string RemappingFilename="", bool IsCS=false, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
uint64_t getOrCompColdCountThreshold() const
Returns ColdCountThreshold if set.
void refresh()
If no summary is present, attempt to refresh.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
uint64_t getOrCompHotCountThreshold() const
Returns HotCountThreshold if set.
This class represents the LLVM 'select' instruction.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
TinyPtrVector - This class is specialized for cases where there are normally 0 or 1 element in a vect...
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
The instances of the Type class are immutable: once they are created, they are never changed.
void print(raw_ostream &O, bool IsForDebug=false, bool NoDetails=false) const
Print the current type.
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
Utility analysis that determines what values are worth profiling.
std::vector< CandidateInfo > get(InstrProfValueKind Kind) const
returns a list of value profiling candidates of the given kind
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
A raw_ostream that writes to an std::string.
The virtual file system interface.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
This file contains the declaration of the Comdat class, which represents a single COMDAT in LLVM.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
uint64_t getFuncHash(const FuncRecordTy *Record)
Return the structural hash associated with the function.
void checkExpectAnnotations(Instruction &I, const ArrayRef< uint32_t > ExistingWeights, bool IsFrontend)
checkExpectAnnotations - compares PGO counters to the thresholds used for llvm.expect and warns if th...
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< FuncNode * > Func
void write64le(void *P, uint64_t V)
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
Function::ProfileCount ProfileCount
auto successors(const MachineBasicBlock *BB)
void createProfileSamplingVar(Module &M)
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
cl::opt< InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate("profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate")))
DenseMap< BasicBlock *, ColorVector > colorEHFunclets(Function &F)
If an EH funclet personality is in use (see isFuncletEHPersonality), this will recompute which blocks...
void createPGONameMetadata(GlobalObject &GO, StringRef PGOName)
Create the PGOName metadata if a global object's PGO name is different from its mangled name.
cl::opt< bool > PGOWarnMissing
raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")
bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, BranchProbabilityInfo *BPI=nullptr, BlockFrequencyInfo *BFI=nullptr)
cl::opt< bool > EnableVTableProfileUse("enable-vtable-profile-use", cl::init(false), cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable " "profiles will be used by ICP pass for more efficient indirect " "call sequence. If false, type profiles won't be used."))
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
cl::opt< bool > DebugInfoCorrelate
OperandBundleDefT< Value * > OperandBundleDef
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
auto reverse(ContainerTy &&C)
void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
cl::opt< bool > NoPGOWarnMismatch
RNSuccIterator< NodeRef, BlockT, RegionT > succ_begin(NodeRef Node)
cl::opt< PGOViewCountsType > PGOViewCounts("pgo-view-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text with " "block profile counts and branch probabilities " "right after PGO profile annotation step. The " "profile counts are computed using branch " "probabilities from the runtime profile data and " "block frequency propagation algorithm. To view " "the raw counts from the profile, use option " "-pgo-view-raw-counts instead. To limit graph " "display to only one function, use filtering option " "-view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
RNSuccIterator< NodeRef, BlockT, RegionT > succ_end(NodeRef Node)
static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If this edge is a critical edge, insert a new node to split the critical edge.
void ViewGraph(const GraphType &G, const Twine &Name, bool ShortNames=false, const Twine &Title="", GraphProgram::Name Program=GraphProgram::DOT)
ViewGraph - Emit a dot graph, run 'dot', run gv on the postscript file, then cleanup.
bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges=false)
Return true if the specified edge is a critical edge.
static uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
auto predecessors(const MachineBasicBlock *BB)
void setProfMetadata(Module *M, Instruction *TI, ArrayRef< uint64_t > EdgeCounts, uint64_t MaxCount)
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
SuccIterator< const Instruction, const BasicBlock > const_succ_iterator
cl::opt< bool > NoPGOWarnMismatchComdatWeak
Implement std::hash so that hash_code can be used in STL containers.
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
DOTGraphTraits(bool isSimple=false)
static std::string getGraphName(const PGOUseFunc *G)
std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph)
DOTGraphTraits - Template class that can be specialized to customize how graphs are converted to 'dot...
DefaultDOTGraphTraits - This class provides the default implementations of all of the DOTGraphTraits ...
static ChildIteratorType child_end(const NodeRef N)
static NodeRef getEntryNode(const PGOUseFunc *G)
static ChildIteratorType child_begin(const NodeRef N)
static nodes_iterator nodes_end(const PGOUseFunc *G)
static nodes_iterator nodes_begin(const PGOUseFunc *G)
Profiling information for a single function.
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
static void setCSFlagInHash(uint64_t &FuncHash)
Instruction * AnnotatedInst