123#include <unordered_map>
131#define DEBUG_TYPE "pgo-instrumentation"
133STATISTIC(NumOfPGOInstrument,
"Number of edges instrumented.");
134STATISTIC(NumOfPGOSelectInsts,
"Number of select instruction instrumented.");
135STATISTIC(NumOfPGOMemIntrinsics,
"Number of mem intrinsics instrumented.");
138STATISTIC(NumOfPGOSplit,
"Number of critical edge splits.");
139STATISTIC(NumOfPGOFunc,
"Number of functions having valid profile counts.");
140STATISTIC(NumOfPGOMismatch,
"Number of functions having mismatch profile.");
141STATISTIC(NumOfPGOMissing,
"Number of functions without profile.");
142STATISTIC(NumOfPGOICall,
"Number of indirect call value instrumentations.");
143STATISTIC(NumOfCSPGOInstrument,
"Number of edges instrumented in CSPGO.");
145 "Number of select instruction instrumented in CSPGO.");
147 "Number of mem intrinsics instrumented in CSPGO.");
149STATISTIC(NumOfCSPGOBB,
"Number of basic-blocks in CSPGO.");
150STATISTIC(NumOfCSPGOSplit,
"Number of critical edge splits in CSPGO.");
152 "Number of functions having valid profile counts in CSPGO.");
154 "Number of functions having mismatch profile in CSPGO.");
155STATISTIC(NumOfCSPGOMissing,
"Number of functions without profile in CSPGO.");
156STATISTIC(NumCoveredBlocks,
"Number of basic blocks that were executed");
163 cl::desc(
"Specify the path of profile data file. This is"
164 "mainly for test purpose."));
168 cl::desc(
"Specify the path of profile remapping file. This is mainly for "
175 cl::desc(
"Disable Value Profiling"));
181 cl::desc(
"Max number of annotations for a single indirect "
188 cl::desc(
"Max number of preicise value annotations for a single memop"
195 cl::desc(
"Append function hash to the name of COMDAT function to avoid "
196 "function hash mismatch due to the preinliner"));
203 cl::desc(
"Use this option to turn on/off "
204 "warnings about missing profile data for "
211 cl::desc(
"Use this option to turn off/on "
212 "warnings about profile cfg mismatch."));
219 cl::desc(
"The option is used to turn on/off "
220 "warnings about hash mismatch for comdat "
221 "or weak functions."));
227 cl::desc(
"Use this option to turn on/off SELECT "
228 "instruction instrumentation. "));
233 cl::desc(
"A boolean option to show CFG dag or text "
234 "with raw profile counts from "
235 "profile data. See also option "
236 "-pgo-view-counts. To limit graph "
237 "display to only one function, use "
238 "filtering option -view-bfi-func-name."),
246 cl::desc(
"Use this option to turn on/off "
247 "memory intrinsic size profiling."));
252 cl::desc(
"When this option is on, the annotated "
253 "branch probability will be emitted as "
254 "optimization remarks: -{Rpass|"
255 "pass-remarks}=pgo-instrumentation"));
259 cl::desc(
"Force to instrument function entry basicblock."));
264 "Use this option to enable function entry coverage instrumentation."));
267 "pgo-block-coverage",
268 cl::desc(
"Use this option to enable basic block coverage instrumentation"));
272 cl::desc(
"Create a dot file of CFGs with block "
273 "coverage inference information"));
276 "pgo-temporal-instrumentation",
277 cl::desc(
"Use this option to enable temporal instrumentation"));
281 cl::desc(
"Fix function entry count in profile use."));
285 cl::desc(
"Print out the non-match BFI count if a hot raw profile count "
286 "becomes non-hot, or a cold raw profile count becomes hot. "
287 "The print is enabled under -Rpass-analysis=pgo, or "
288 "internal option -pass-remakrs-analysis=pgo."));
292 cl::desc(
"Print out mismatched BFI counts after setting profile metadata "
293 "The print is enabled under -Rpass-analysis=pgo, or "
294 "internal option -pass-remakrs-analysis=pgo."));
298 cl::desc(
"Set the threshold for pgo-verify-bfi: only print out "
299 "mismatched BFI if the difference percentage is greater than "
300 "this value (in percentage)."));
304 cl::desc(
"Set the threshold for pgo-verify-bfi: skip the counts whose "
305 "profile count value is below."));
310 cl::desc(
"Trace the hash of the function with this name."));
314 cl::desc(
"Do not instrument functions smaller than this threshold."));
318 cl::desc(
"Do not instrument functions with the number of critical edges "
319 " greater than this threshold."));
335 cl::desc(
"Use the old CFG function hashing"));
342 return std::string();
347 return std::string();
359 else if (CV->
isOne())
371#define VALUE_PROF_KIND(Enumerator, Value, Descr) Descr,
378 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
380 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
382 ProfileVersion |= VARIANT_MASK_CSIR_PROF;
384 ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
386 ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
389 VARIANT_MASK_BYTE_COVERAGE | VARIANT_MASK_FUNCTION_ENTRY_ONLY;
391 ProfileVersion |= VARIANT_MASK_BYTE_COVERAGE;
393 ProfileVersion |= VARIANT_MASK_TEMPORAL_PROF;
398 Triple TT(M.getTargetTriple());
399 if (TT.supportsCOMDAT()) {
401 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));
403 return IRLevelVersionVariable;
413enum VisitMode { VM_counting, VM_instrument, VM_annotate };
417struct SelectInstVisitor :
public InstVisitor<SelectInstVisitor> {
420 VisitMode
Mode = VM_counting;
421 unsigned *CurCtrIdx =
nullptr;
422 unsigned TotalNumCtrs = 0;
425 PGOUseFunc *UseFunc =
nullptr;
426 bool HasSingleByteCoverage;
428 SelectInstVisitor(
Function &Func,
bool HasSingleByteCoverage)
429 :
F(
Func), HasSingleByteCoverage(HasSingleByteCoverage) {}
431 void countSelects() {
441 void instrumentSelects(
unsigned *Ind,
unsigned TotalNC,
GlobalVariable *FNV,
443 Mode = VM_instrument;
445 TotalNumCtrs = TotalNC;
452 void annotateSelects(PGOUseFunc *UF,
unsigned *Ind) {
467 unsigned getNumOfSelectInsts()
const {
return NSIs; }
479 bool Removed =
false;
480 bool IsCritical =
false;
483 : SrcBB(Src), DestBB(Dest), Weight(
W) {}
486 std::string infoString()
const {
487 return (
Twine(Removed ?
"-" :
" ") + (InMST ?
" " :
"*") +
488 (IsCritical ?
"c" :
" ") +
" W=" +
Twine(Weight))
499 PGOBBInfo(
unsigned IX) : Group(this),
Index(IX) {}
502 std::string infoString()
const {
508template <
class Edge,
class BBInfo>
class FuncPGOInstrumentation {
516 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
520 void computeCFGHash();
521 void renameComdatFunction();
525 std::vector<std::vector<VPCandidateInfo>> ValueSites;
526 SelectInstVisitor SIVisitor;
527 std::string FuncName;
528 std::string DeprecatedFuncName;
537 const std::optional<BlockCoverageInference> BCI;
539 static std::optional<BlockCoverageInference>
540 constructBCI(
Function &Func,
bool HasSingleByteCoverage,
541 bool InstrumentFuncEntry) {
542 if (HasSingleByteCoverage)
549 void getInstrumentBBs(std::vector<BasicBlock *> &InstrumentBBs);
562 void dumpInfo(
StringRef Str =
"")
const {
564 " Hash: " +
Twine(FunctionHash) +
"\t" + Str);
567 FuncPGOInstrumentation(
569 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
572 bool InstrumentFuncEntry =
true,
bool HasSingleByteCoverage =
false)
573 :
F(
Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(
Func, TLI),
574 TLI(TLI), ValueSites(IPVK_Last + 1),
575 SIVisitor(
Func, HasSingleByteCoverage),
576 MST(
F, InstrumentFuncEntry, BPI,
BFI),
577 BCI(constructBCI(
Func, HasSingleByteCoverage, InstrumentFuncEntry)) {
579 BCI->viewBlockCoverageGraph();
581 SIVisitor.countSelects();
582 ValueSites[IPVK_MemOPSize] = VPC.
get(IPVK_MemOPSize);
584 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
585 NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
586 NumOfPGOBB += MST.
BBInfos.size();
587 ValueSites[IPVK_IndirectCallTarget] = VPC.
get(IPVK_IndirectCallTarget);
589 NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
590 NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();
591 NumOfCSPGOBB += MST.
BBInfos.size();
597 if (!ComdatMembers.empty())
598 renameComdatFunction();
604 IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++;
606 IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++;
619template <
class Edge,
class BBInfo>
620void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
621 std::vector<uint8_t> Indexes;
627 auto BI = findBBInfo(Succ);
631 for (
int J = 0; J < 4; J++)
632 Indexes.push_back((uint8_t)(
Index >> (J * 8)));
641 FunctionHash = (
uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
642 (
uint64_t)ValueSites[IPVK_IndirectCallTarget].
size() << 48 |
647 auto updateJCH = [&JCH](
uint64_t Num) {
652 updateJCH((
uint64_t)SIVisitor.getNumOfSelectInsts());
653 updateJCH((
uint64_t)ValueSites[IPVK_IndirectCallTarget].
size());
656 updateJCH(BCI->getInstrumentedBlocksHash());
667 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
670 LLVM_DEBUG(
dbgs() <<
"Function Hash Computation for " <<
F.getName() <<
":\n"
671 <<
" CRC = " << JC.
getCRC()
672 <<
", Selects = " << SIVisitor.getNumOfSelectInsts()
673 <<
", Edges = " << MST.
AllEdges.size() <<
", ICSites = "
674 << ValueSites[IPVK_IndirectCallTarget].size());
677 <<
", High32 CRC = " << JCH.
getCRC());
682 dbgs() <<
"Funcname=" <<
F.getName() <<
", Hash=" << FunctionHash
683 <<
" in building " <<
F.getParent()->getSourceFileName() <<
"\n";
689 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
701 for (
auto &&CM :
make_range(ComdatMembers.equal_range(
C))) {
702 assert(!isa<GlobalAlias>(CM.second));
703 Function *FM = dyn_cast<Function>(CM.second);
711template <
class Edge,
class BBInfo>
712void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
715 std::string OrigName =
F.getName().str();
716 std::string NewFuncName =
718 F.setName(
Twine(NewFuncName));
720 FuncName =
Twine(FuncName +
"." +
Twine(FunctionHash)).
str();
726 if (!
F.hasComdat()) {
728 NewComdat =
M->getOrInsertComdat(
StringRef(NewFuncName));
730 F.setComdat(NewComdat);
735 Comdat *OrigComdat =
F.getComdat();
736 std::string NewComdatName =
738 NewComdat =
M->getOrInsertComdat(
StringRef(NewComdatName));
741 for (
auto &&CM :
make_range(ComdatMembers.equal_range(OrigComdat))) {
743 cast<Function>(CM.second)->setComdat(NewComdat);
749template <
class Edge,
class BBInfo>
750void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(
751 std::vector<BasicBlock *> &InstrumentBBs) {
754 if (BCI->shouldInstrumentBlock(BB))
755 InstrumentBBs.push_back(&BB);
760 std::vector<Edge *> EdgeList;
761 EdgeList.reserve(MST.
AllEdges.size());
763 EdgeList.push_back(
E.get());
765 for (
auto &
E : EdgeList) {
768 InstrumentBBs.push_back(InstrBB);
774template <
class Edge,
class BBInfo>
775BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *
E) {
776 if (
E->InMST ||
E->Removed)
782 if (SrcBB ==
nullptr)
784 if (DestBB ==
nullptr)
799 return canInstrument(SrcBB);
801 return canInstrument(DestBB);
810 dbgs() <<
"Fail to split critical edge: not instrument this edge.\n");
815 IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;
817 <<
" --> " << getBBInfo(DestBB).
Index <<
"\n");
819 MST.
addEdge(SrcBB, InstrBB, 0);
821 Edge &NewEdge1 = MST.
addEdge(InstrBB, DestBB, 0);
822 NewEdge1.InMST =
true;
825 return canInstrument(InstrBB);
841 if (!isa<IntrinsicInst>(OrigCall)) {
844 std::optional<OperandBundleUse> ParentFunclet =
852 if (!BlockColors.
empty()) {
853 const ColorVector &CV = BlockColors.
find(OrigCall->getParent())->second;
854 assert(CV.
size() == 1 &&
"non-unique color for block!");
867 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
875 FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(
882 FuncInfo.FunctionHash);
884 auto &EntryBB =
F.getEntryBlock();
894 std::vector<BasicBlock *> InstrumentBBs;
895 FuncInfo.getInstrumentBBs(InstrumentBBs);
896 unsigned NumCounters =
897 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
902 auto &EntryBB =
F.getEntryBlock();
912 for (
auto *InstrBB : InstrumentBBs) {
915 "Cannot get the Instrumentation point");
920 ? Intrinsic::instrprof_cover
921 : Intrinsic::instrprof_increment),
926 FuncInfo.SIVisitor.instrumentSelects(&
I, NumCounters, FuncInfo.FuncNameVar,
927 FuncInfo.FunctionHash);
933 NumOfPGOICall += FuncInfo.ValueSites[IPVK_IndirectCallTarget].size();
940 if (
F.hasPersonalityFn() &&
945 for (
uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
946 unsigned SiteIndex = 0;
952 <<
" site: CallSite Index = " << SiteIndex <<
"\n");
955 assert(
Builder.GetInsertPoint() != Cand.InsertPt->getParent()->end() &&
956 "Cannot get the Instrumentation point");
958 Value *ToProfile =
nullptr;
959 if (Cand.V->getType()->isIntegerTy())
960 ToProfile =
Builder.CreateZExtOrTrunc(Cand.V,
Builder.getInt64Ty());
961 else if (Cand.V->getType()->isPointerTy())
963 assert(ToProfile &&
"value profiling Value is of unexpected type");
970 Builder.getInt64(FuncInfo.FunctionHash), ToProfile,
980struct PGOUseEdge :
public PGOEdge {
981 using PGOEdge::PGOEdge;
983 bool CountValid =
false;
993 std::string infoString()
const {
995 return PGOEdge::infoString();
996 return (
Twine(PGOEdge::infoString()) +
" Count=" +
Twine(CountValue))
1004struct PGOUseBBInfo :
public PGOBBInfo {
1007 int32_t UnknownCountInEdge = 0;
1008 int32_t UnknownCountOutEdge = 0;
1009 DirectEdges InEdges;
1010 DirectEdges OutEdges;
1012 PGOUseBBInfo(
unsigned IX) : PGOBBInfo(IX), CountValid(
false) {}
1021 std::string infoString()
const {
1023 return PGOBBInfo::infoString();
1024 return (
Twine(PGOBBInfo::infoString()) +
" Count=" +
Twine(CountValue))
1029 void addOutEdge(PGOUseEdge *
E) {
1030 OutEdges.push_back(
E);
1031 UnknownCountOutEdge++;
1035 void addInEdge(PGOUseEdge *
E) {
1036 InEdges.push_back(
E);
1037 UnknownCountInEdge++;
1046 for (
const auto &
E : Edges) {
1059 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
1062 bool HasSingleByteCoverage)
1063 :
F(
Func),
M(Modu),
BFI(BFIin), PSI(PSI),
1064 FuncInfo(
Func, TLI, ComdatMembers,
false, BPI, BFIin, IsCS,
1065 InstrumentFuncEntry, HasSingleByteCoverage),
1066 FreqAttr(FFA_Normal), IsCS(IsCS) {}
1068 void handleInstrProfError(
Error Err,
uint64_t MismatchedFuncSum);
1075 void populateCounters();
1084 void annotateValueSites();
1087 void annotateValueSites(
uint32_t Kind);
1090 void annotateIrrLoopHeaderWeights();
1093 enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };
1096 FuncFreqAttr getFuncFreqAttr()
const {
return FreqAttr; }
1105 PGOUseBBInfo &getBBInfo(
const BasicBlock *BB)
const {
1106 return FuncInfo.getBBInfo(BB);
1110 PGOUseBBInfo *findBBInfo(
const BasicBlock *BB)
const {
1111 return FuncInfo.findBBInfo(BB);
1116 void dumpInfo(
StringRef Str =
"")
const { FuncInfo.dumpInfo(Str); }
1118 uint64_t getProgramMaxCount()
const {
return ProgramMaxCount; }
1127 FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> FuncInfo;
1143 FuncFreqAttr FreqAttr;
1149 bool setInstrumentedCounts(
const std::vector<uint64_t> &CountFromProfile);
1162 FreqAttr = FFA_Cold;
1174 for (
auto &
E : FuncInfo.MST.AllEdges) {
1179 PGOUseBBInfo &SrcInfo = FuncInfo.getBBInfo(SrcBB);
1180 PGOUseBBInfo &DestInfo = FuncInfo.getBBInfo(DestBB);
1181 SrcInfo.addOutEdge(
E.get());
1182 DestInfo.addInEdge(
E.get());
1188bool PGOUseFunc::setInstrumentedCounts(
1189 const std::vector<uint64_t> &CountFromProfile) {
1191 std::vector<BasicBlock *> InstrumentBBs;
1192 FuncInfo.getInstrumentBBs(InstrumentBBs);
1196 unsigned NumCounters =
1197 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
1200 if (NumCounters != CountFromProfile.size()) {
1203 auto *FuncEntry = &*
F.begin();
1208 uint64_t CountValue = CountFromProfile[
I++];
1209 PGOUseBBInfo &
Info = getBBInfo(InstrBB);
1213 if (InstrBB == FuncEntry && CountValue == 0)
1215 Info.setBBInfoCount(CountValue);
1217 ProfileCountSize = CountFromProfile.size();
1221 auto setEdgeCount = [
this](PGOUseEdge *
E,
uint64_t Value) ->
void {
1223 this->getBBInfo(
E->SrcBB).UnknownCountOutEdge--;
1224 this->getBBInfo(
E->DestBB).UnknownCountInEdge--;
1230 for (
auto &
E : FuncInfo.MST.AllEdges) {
1231 if (
E->Removed ||
E->InMST)
1234 PGOUseBBInfo &SrcInfo = getBBInfo(SrcBB);
1238 if (SrcInfo.CountValid && SrcInfo.OutEdges.size() == 1)
1239 setEdgeCount(
E.get(), SrcInfo.CountValue);
1242 PGOUseBBInfo &DestInfo = getBBInfo(DestBB);
1245 if (DestInfo.CountValid && DestInfo.InEdges.size() == 1)
1246 setEdgeCount(
E.get(), DestInfo.CountValue);
1252 setEdgeCount(
E.get(), 0);
1259void PGOUseFunc::setEdgeCount(DirectEdges &Edges,
uint64_t Value) {
1260 for (
auto &
E : Edges) {
1265 getBBInfo(
E->SrcBB).UnknownCountOutEdge--;
1266 getBBInfo(
E->DestBB).UnknownCountInEdge--;
1274 const char MetadataName[] =
"instr_prof_hash_mismatch";
1277 auto *Existing =
F.getMetadata(LLVMContext::MD_annotation);
1279 MDTuple *Tuple = cast<MDTuple>(Existing);
1280 for (
const auto &
N : Tuple->
operands()) {
1281 if (
N.equalsStr(MetadataName))
1290 F.setMetadata(LLVMContext::MD_annotation, MD);
1293void PGOUseFunc::handleInstrProfError(
Error Err,
uint64_t MismatchedFuncSum) {
1295 auto &Ctx =
M->getContext();
1296 auto Err = IPE.
get();
1297 bool SkipWarning =
false;
1299 << FuncInfo.FuncName <<
": ");
1300 if (Err == instrprof_error::unknown_function) {
1301 IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++;
1304 }
else if (Err == instrprof_error::hash_mismatch ||
1305 Err == instrprof_error::malformed) {
1306 IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++;
1312 LLVM_DEBUG(
dbgs() <<
"hash mismatch (hash= " << FuncInfo.FunctionHash
1313 <<
" skip=" << SkipWarning <<
")");
1323 IPE.
message() + std::string(
" ") +
F.getName().str() +
1324 std::string(
" Hash = ") + std::to_string(FuncInfo.FunctionHash) +
1325 std::string(
" up to ") + std::to_string(MismatchedFuncSum) +
1326 std::string(
" count discarded");
1338 auto &Ctx =
M->getContext();
1341 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,
1342 &MismatchedFuncSum);
1344 handleInstrProfError(std::move(
E), MismatchedFuncSum);
1347 ProfileRecord = std::move(
Result.get());
1352 std::vector<uint64_t> &CountFromProfile = ProfileRecord.
Counts;
1354 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
1358 for (
unsigned I = 0, S = CountFromProfile.size();
I < S;
I++) {
1360 ValueSum += CountFromProfile[
I];
1362 AllZeros = (ValueSum == 0);
1366 getBBInfo(
nullptr).UnknownCountOutEdge = 2;
1367 getBBInfo(
nullptr).UnknownCountInEdge = 2;
1369 if (!setInstrumentedCounts(CountFromProfile)) {
1371 dbgs() <<
"Inconsistent number of counts, skipping this function");
1373 M->getName().data(),
1374 Twine(
"Inconsistent number of counts in ") +
F.getName().str() +
1375 Twine(
": the profile may be stale or there is a function name "
1387 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,
1388 &MismatchedFuncSum);
1389 if (
auto Err =
Result.takeError()) {
1390 handleInstrProfError(std::move(Err), MismatchedFuncSum);
1394 std::vector<uint64_t> &CountsFromProfile =
Result.get().Counts;
1398 if (FuncInfo.BCI->shouldInstrumentBlock(BB))
1404 InverseDependencies;
1405 for (
auto &BB :
F) {
1406 for (
auto *Dep : FuncInfo.BCI->getDependencies(BB)) {
1408 InverseDependencies[Dep].
insert(&BB);
1413 std::stack<const BasicBlock *> CoveredBlocksToProcess;
1414 for (
auto &[BB, IsCovered] : Coverage)
1416 CoveredBlocksToProcess.push(BB);
1418 while (!CoveredBlocksToProcess.empty()) {
1419 auto *CoveredBlock = CoveredBlocksToProcess.top();
1420 assert(Coverage[CoveredBlock]);
1421 CoveredBlocksToProcess.pop();
1422 for (
auto *BB : InverseDependencies[CoveredBlock]) {
1427 CoveredBlocksToProcess.push(BB);
1435 F.setEntryCount(Coverage[&
F.getEntryBlock()] ? 10000 : 0);
1436 for (
auto &BB :
F) {
1445 Weights.
push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0);
1446 if (Weights.
size() >= 2)
1447 BB.getTerminator()->setMetadata(LLVMContext::MD_prof,
1448 MDB.createBranchWeights(Weights));
1451 unsigned NumCorruptCoverage = 0;
1456 auto IsBlockDead = [&](
const BasicBlock &BB) -> std::optional<bool> {
1457 if (
auto C =
BFI.getBlockProfileCount(&BB))
1461 LLVM_DEBUG(
dbgs() <<
"Block Coverage: (Instrumented=*, Covered=X)\n");
1462 for (
auto &BB :
F) {
1463 LLVM_DEBUG(
dbgs() << (FuncInfo.BCI->shouldInstrumentBlock(BB) ?
"* " :
" ")
1464 << (Coverage[&BB] ?
"X " :
" ") <<
" " << BB.getName()
1470 if (Coverage[&BB] == IsBlockDead(BB).value_or(
false)) {
1472 dbgs() <<
"Found inconsistent block covearge for " << BB.getName()
1473 <<
": BCI=" << (Coverage[&BB] ?
"Covered" :
"Dead") <<
" BFI="
1474 << (IsBlockDead(BB).
value() ?
"Dead" :
"Covered") <<
"\n");
1475 ++NumCorruptCoverage;
1481 auto &Ctx =
M->getContext();
1483 M->getName().data(),
1484 Twine(
"Found inconsistent block coverage for function ") +
F.getName() +
1485 " in " +
Twine(NumCorruptCoverage) +
" blocks.",
1489 FuncInfo.BCI->viewBlockCoverageGraph(&Coverage);
1494void PGOUseFunc::populateCounters() {
1495 bool Changes =
true;
1496 unsigned NumPasses = 0;
1504 PGOUseBBInfo *Count = findBBInfo(&BB);
1505 if (Count ==
nullptr)
1507 if (!Count->CountValid) {
1508 if (Count->UnknownCountOutEdge == 0) {
1510 Count->CountValid =
true;
1512 }
else if (Count->UnknownCountInEdge == 0) {
1514 Count->CountValid =
true;
1518 if (Count->CountValid) {
1519 if (Count->UnknownCountOutEdge == 1) {
1525 if (Count->CountValue > OutSum)
1526 Total = Count->CountValue - OutSum;
1527 setEdgeCount(Count->OutEdges,
Total);
1530 if (Count->UnknownCountInEdge == 1) {
1533 if (Count->CountValue > InSum)
1534 Total = Count->CountValue - InSum;
1535 setEdgeCount(Count->InEdges,
Total);
1542 LLVM_DEBUG(
dbgs() <<
"Populate counts in " << NumPasses <<
" passes.\n");
1546 for (
auto &BB :
F) {
1547 auto BI = findBBInfo(&BB);
1550 assert(BI->CountValid &&
"BB count is not valid");
1553 uint64_t FuncEntryCount = getBBInfo(&*
F.begin()).CountValue;
1554 uint64_t FuncMaxCount = FuncEntryCount;
1555 for (
auto &BB :
F) {
1556 auto BI = findBBInfo(&BB);
1559 FuncMaxCount = std::max(FuncMaxCount, BI->CountValue);
1563 if (FuncMaxCount > 0 && FuncEntryCount == 0)
1566 markFunctionAttributes(FuncEntryCount, FuncMaxCount);
1569 FuncInfo.SIVisitor.annotateSelects(
this, &CountPosition);
1570 assert(CountPosition == ProfileCountSize);
1572 LLVM_DEBUG(FuncInfo.dumpInfo(
"after reading profile."));
1576void PGOUseFunc::setBranchWeights() {
1578 LLVM_DEBUG(
dbgs() <<
"\nSetting branch weights for func " <<
F.getName()
1579 <<
" IsCS=" << IsCS <<
"\n");
1580 for (
auto &BB :
F) {
1584 if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
1585 isa<IndirectBrInst>(TI) || isa<InvokeInst>(TI) ||
1586 isa<CallBrInst>(TI)))
1589 if (getBBInfo(&BB).CountValue == 0)
1593 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);
1594 unsigned Size = BBCountInfo.OutEdges.size();
1597 for (
unsigned s = 0; s <
Size; s++) {
1598 const PGOUseEdge *
E = BBCountInfo.OutEdges[s];
1601 if (DestBB ==
nullptr)
1605 if (EdgeCount > MaxCount)
1606 MaxCount = EdgeCount;
1607 EdgeCounts[SuccNum] = EdgeCount;
1616 auto &Ctx =
M->getContext();
1618 M->getName().data(),
1619 Twine(
"Profile in ") +
F.getName().str() +
1620 Twine(
" partially ignored") +
1621 Twine(
", possibly due to the lack of a return path."),
1629 if (isa<IndirectBrInst>(Pred->getTerminator()))
1635void PGOUseFunc::annotateIrrLoopHeaderWeights() {
1636 LLVM_DEBUG(
dbgs() <<
"\nAnnotating irreducible loop header weights.\n");
1638 for (
auto &BB :
F) {
1644 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);
1650void SelectInstVisitor::instrumentOneSelectInst(
SelectInst &SI) {
1655 auto *Step =
Builder.CreateZExt(
SI.getCondition(), Int64Ty);
1660 Builder.getInt32(*CurCtrIdx), Step});
1664void SelectInstVisitor::annotateOneSelectInst(
SelectInst &SI) {
1665 std::vector<uint64_t> &CountFromProfile = UseFunc->getProfileRecord().Counts;
1666 assert(*CurCtrIdx < CountFromProfile.size() &&
1667 "Out of bound access of counters");
1669 SCounts[0] = CountFromProfile[*CurCtrIdx];
1672 auto BI = UseFunc->findBBInfo(
SI.getParent());
1674 TotalCount = BI->CountValue;
1676 SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);
1677 uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);
1682void SelectInstVisitor::visitSelectInst(
SelectInst &SI) {
1686 if (
SI.getCondition()->getType()->isVectorTy())
1694 instrumentOneSelectInst(SI);
1697 annotateOneSelectInst(SI);
1705void PGOUseFunc::annotateValueSites() {
1713 annotateValueSites(Kind);
1717void PGOUseFunc::annotateValueSites(
uint32_t Kind) {
1718 assert(Kind <= IPVK_Last);
1719 unsigned ValueSiteIndex = 0;
1720 auto &ValueSites = FuncInfo.ValueSites[
Kind];
1722 if (NumValueSites != ValueSites.size()) {
1723 auto &Ctx =
M->getContext();
1725 M->getName().data(),
1726 Twine(
"Inconsistent number of value sites for ") +
1729 Twine(
"\", possibly due to the use of a stale profile."),
1735 LLVM_DEBUG(
dbgs() <<
"Read one value site profile (kind = " << Kind
1736 <<
"): Index = " << ValueSiteIndex <<
" out of "
1737 << NumValueSites <<
"\n");
1750 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
1755 ComdatMembers.insert(std::make_pair(
C, &
F));
1757 if (
Comdat *
C = GV.getComdat())
1758 ComdatMembers.insert(std::make_pair(
C, &GV));
1760 if (
Comdat *
C = GA.getComdat())
1761 ComdatMembers.insert(std::make_pair(
C, &GA));
1766 if (
F.isDeclaration())
1768 if (
F.hasFnAttribute(llvm::Attribute::NoProfile))
1770 if (
F.hasFnAttribute(llvm::Attribute::SkipProfile))
1778 unsigned NumCriticalEdges = 0;
1779 for (
auto &BB :
F) {
1788 <<
", NumCriticalEdges=" << NumCriticalEdges
1789 <<
" exceed the threshold. Skip PGO.\n");
1804 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1810 auto &TLI = LookupTLI(
F);
1811 auto *BPI = LookupBPI(
F);
1812 auto *BFI = LookupBFI(
F);
1856 auto BFIEntryCount =
F.getEntryCount();
1857 assert(BFIEntryCount && (BFIEntryCount->getCount() > 0) &&
1858 "Invalid BFI Entrycount");
1862 for (
auto &BBI :
F) {
1865 if (!Func.findBBInfo(&BBI))
1868 CountValue = Func.getBBInfo(&BBI).CountValue;
1869 BFICountValue = *BFICount;
1873 if (SumCount.isZero())
1877 "Incorrect sum of BFI counts");
1880 double Scale = (SumCount / SumBFICount).convertToDouble();
1881 if (Scale < 1.001 && Scale > 0.999)
1884 uint64_t FuncEntryCount = Func.getBBInfo(&*
F.begin()).CountValue;
1885 uint64_t NewEntryCount = 0.5 + FuncEntryCount * Scale;
1886 if (NewEntryCount == 0)
1888 if (NewEntryCount != FuncEntryCount) {
1891 <<
", entry_count " << FuncEntryCount <<
" --> "
1892 << NewEntryCount <<
"\n");
1909 unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;
1910 for (
auto &BBI :
F) {
1914 if (Func.getBBInfo(&BBI).CountValid)
1915 CountValue = Func.getBBInfo(&BBI).CountValue;
1922 BFICountValue = *BFICount;
1925 bool rawIsHot = CountValue >= HotCountThreshold;
1926 bool BFIIsHot = BFICountValue >= HotCountThreshold;
1928 bool ShowCount =
false;
1929 if (rawIsHot && !BFIIsHot) {
1930 Msg =
"raw-Hot to BFI-nonHot";
1932 }
else if (rawIsCold && BFIIsHot) {
1933 Msg =
"raw-Cold to BFI-Hot";
1942 uint64_t Diff = (BFICountValue >= CountValue)
1943 ? BFICountValue - CountValue
1944 : CountValue - BFICountValue;
1952 F.getSubprogram(), &BBI);
1954 <<
" Count=" <<
ore::NV(
"Count", CountValue)
1955 <<
" BFI_Count=" <<
ore::NV(
"Count", BFICountValue);
1957 Remark <<
" (" << Msg <<
")";
1964 F.getSubprogram(), &
F.getEntryBlock())
1965 <<
"In Func " <<
ore::NV(
"Function",
F.getName())
1966 <<
": Num_of_BB=" <<
ore::NV(
"Count", BBNum)
1967 <<
", Num_of_non_zerovalue_BB=" <<
ore::NV(
"Count", NonZeroBBNum)
1968 <<
", Num_of_mis_matching_BB=" <<
ore::NV(
"Count", BBMisMatchNum);
1980 auto &Ctx = M.getContext();
1983 ProfileRemappingFileName);
1984 if (
Error E = ReaderOrErr.takeError()) {
1992 std::unique_ptr<IndexedInstrProfReader> PGOReader =
1993 std::move(ReaderOrErr.get());
1999 if (!PGOReader->hasCSIRLevelProfile() && IsCS)
2003 if (!PGOReader->isIRLevelProfile()) {
2005 ProfileFileName.
data(),
"Not an IR level instrumentation profile"));
2008 if (PGOReader->functionEntryOnly()) {
2010 ProfileFileName.
data(),
2011 "Function entry profiles are not yet supported for optimization"));
2018 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
2023 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
2025 std::vector<Function *> HotFunctions;
2026 std::vector<Function *> ColdFunctions;
2030 bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
2033 bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();
2037 auto &TLI = LookupTLI(
F);
2038 auto *BPI = LookupBPI(
F);
2039 auto *BFI = LookupBFI(
F);
2040 if (!HasSingleByteCoverage) {
2046 PGOUseFunc Func(
F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS,
2047 InstrumentFuncEntry, HasSingleByteCoverage);
2048 if (HasSingleByteCoverage) {
2049 Func.populateCoverage(PGOReader.get());
2057 bool AllZeros =
false;
2058 if (!Func.readCounters(PGOReader.get(), AllZeros, PseudoKind))
2062 if (Func.getProgramMaxCount() != 0)
2063 ColdFunctions.push_back(&
F);
2068 if (
F.hasFnAttribute(Attribute::Cold))
2069 F.removeFnAttr(Attribute::Cold);
2072 F.addFnAttr(Attribute::Hot);
2075 Func.populateCounters();
2076 Func.setBranchWeights();
2077 Func.annotateValueSites();
2078 Func.annotateIrrLoopHeaderWeights();
2079 PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();
2080 if (FreqAttr == PGOUseFunc::FFA_Cold)
2081 ColdFunctions.push_back(&
F);
2082 else if (FreqAttr == PGOUseFunc::FFA_Hot)
2083 HotFunctions.push_back(&
F);
2088 std::unique_ptr<BranchProbabilityInfo> NewBPI =
2089 std::make_unique<BranchProbabilityInfo>(
F, LI);
2090 std::unique_ptr<BlockFrequencyInfo> NewBFI =
2091 std::make_unique<BlockFrequencyInfo>(
F, *NewBPI, LI);
2095 dbgs() <<
"pgo-view-counts: " << Func.getFunc().getName() <<
"\n";
2096 NewBFI->print(
dbgs());
2106 ViewGraph(&Func,
Twine(
"PGORawCounts_") + Func.getFunc().getName());
2108 dbgs() <<
"pgo-view-raw-counts: " << Func.getFunc().getName() <<
"\n";
2135 for (
auto &
F : HotFunctions) {
2136 F->addFnAttr(Attribute::InlineHint);
2137 LLVM_DEBUG(
dbgs() <<
"Set inline attribute to function: " <<
F->getName()
2140 for (
auto &
F : ColdFunctions) {
2143 if (
F->hasFnAttribute(Attribute::Hot)) {
2144 auto &Ctx = M.getContext();
2145 std::string Msg = std::string(
"Function ") +
F->getName().str() +
2146 std::string(
" is annotated as a hot function but"
2147 " the profile is cold");
2152 F->addFnAttr(Attribute::Cold);
2153 LLVM_DEBUG(
dbgs() <<
"Set cold attribute to function: " <<
F->getName()
2160 std::string Filename, std::string RemappingFilename,
bool IsCS,
2162 : ProfileFileName(
std::
move(Filename)),
2163 ProfileRemappingFileName(
std::
move(RemappingFilename)), IsCS(IsCS),
2190 LookupTLI, LookupBPI, LookupBFI, PSI, IsCS))
2197 if (!
Node->getName().empty())
2198 return Node->getName().str();
2200 std::string SimpleNodeName;
2209 assert(MaxCount > 0 &&
"Bad max count");
2212 for (
const auto &ECI : EdgeCounts)
2225 if (BrCondStr.empty())
2237 std::string BranchProbStr;
2240 OS <<
" (total count : " << TotalCount <<
")";
2246 << BrCondStr <<
" is true with probability : " << BranchProbStr;
2265 return &
G->getFunc().front();
2288 return std::string(
G->getFunc().getName());
2296 PGOUseBBInfo *BI = Graph->findBBInfo(Node);
2298 if (BI && BI->CountValid)
2299 OS << BI->CountValue <<
"\\l";
2307 if (!isa<SelectInst>(&
I))
2310 OS <<
"SELECT : { T = ";
2314 OS <<
"Unknown, F = Unknown }\\l";
2316 OS << TC <<
", F = " << FC <<
" }\\l";
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
This file finds the minimum set of blocks on a CFG that must be instrumented to infer execution cover...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Given that RA is a live value
static BasicBlock * getInstrBB(CFGMST< Edge, BBInfo > &MST, Edge &E, const DenseSet< const BasicBlock * > &ExecBlocks)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)
Module.h This file contains the declarations for the Module class.
static cl::opt< bool > PGOInstrumentEntry("pgo-instrument-entry", cl::init(false), cl::Hidden, cl::desc("Force to instrument function entry basicblock."))
static cl::opt< std::string > PGOTestProfileRemappingFile("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))
static cl::opt< bool > PGOFixEntryCount("pgo-fix-entry-count", cl::init(true), cl::Hidden, cl::desc("Fix function entry count in profile use."))
static void fixFuncEntryCount(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI)
static cl::opt< bool > PGOVerifyHotBFI("pgo-verify-hot-bfi", cl::init(false), cl::Hidden, cl::desc("Print out the non-match BFI count if a hot raw profile count " "becomes non-hot, or a cold raw profile count becomes hot. " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remakrs-analysis=pgo."))
static void annotateFunctionWithHashMismatch(Function &F, LLVMContext &ctx)
static cl::opt< bool > PGOTemporalInstrumentation("pgo-temporal-instrumentation", cl::desc("Use this option to enable temporal instrumentation"))
static bool skipPGO(const Function &F)
static cl::opt< unsigned > PGOFunctionSizeThreshold("pgo-function-size-threshold", cl::Hidden, cl::desc("Do not instrument functions smaller than this threshold."))
static cl::opt< unsigned > MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::desc("Max number of annotations for a single indirect " "call callsite"))
static void collectComdatMembers(Module &M, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
static cl::opt< unsigned > PGOVerifyBFICutoff("pgo-verify-bfi-cutoff", cl::init(5), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose " "profile count value is below."))
static cl::opt< std::string > PGOTraceFuncHash("pgo-trace-func-hash", cl::init("-"), cl::Hidden, cl::value_desc("function name"), cl::desc("Trace the hash of the function with this name."))
static void instrumentOneFunc(Function &F, Module *M, TargetLibraryInfo &TLI, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers, bool IsCS)
static void populateEHOperandBundle(VPCandidateInfo &Cand, DenseMap< BasicBlock *, ColorVector > &BlockColors, SmallVectorImpl< OperandBundleDef > &OpBundles)
static cl::opt< bool > PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))
static cl::opt< bool > PGOFunctionEntryCoverage("pgo-function-entry-coverage", cl::Hidden, cl::desc("Use this option to enable function entry coverage instrumentation."))
static bool InstrumentAllFunctions(Module &M, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS)
static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI, uint64_t HotCountThreshold, uint64_t ColdCountThreshold)
static cl::opt< unsigned > PGOVerifyBFIRatio("pgo-verify-bfi-ratio", cl::init(2), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: only print out " "mismatched BFI if the difference percentage is greater than " "this value (in percentage)."))
static cl::opt< bool > DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))
static cl::opt< unsigned > PGOFunctionCriticalEdgeThreshold("pgo-critical-edge-threshold", cl::init(20000), cl::Hidden, cl::desc("Do not instrument functions with the number of critical edges " " greater than this threshold."))
static cl::opt< std::string > PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose."))
static bool canRenameComdat(Function &F, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)
static cl::opt< bool > PGOVerifyBFI("pgo-verify-bfi", cl::init(false), cl::Hidden, cl::desc("Print out mismatched BFI counts after setting profile metadata " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remakrs-analysis=pgo."))
static cl::opt< bool > PGOBlockCoverage("pgo-block-coverage", cl::desc("Use this option to enable basic block coverage instrumentation"))
static cl::opt< bool > PGOOldCFGHashing("pgo-instr-old-cfg-hashing", cl::init(false), cl::Hidden, cl::desc("Use the old CFG function hashing"))
static uint64_t sumEdgeCount(const ArrayRef< PGOUseEdge * > Edges)
static cl::opt< bool > PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))
Function::ProfileCount ProfileCount
static cl::opt< bool > EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))
static cl::opt< unsigned > MaxNumMemOPAnnotations("memop-max-annotations", cl::init(4), cl::Hidden, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic"))
static cl::opt< bool > DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
static std::string getSimpleNodeName(const BasicBlock *Node)
static cl::opt< bool > PGOViewBlockCoverageGraph("pgo-view-block-coverage-graph", cl::desc("Create a dot file of CFGs with block " "coverage inference information"))
static GlobalVariable * createIRLevelProfileFlagVar(Module &M, bool IsCS)
static void setupBBInfoEdges(FuncPGOInstrumentation< PGOUseEdge, PGOUseBBInfo > &FuncInfo)
Set up InEdges/OutEdges for all BBs in the MST.
static bool isIndirectBrTarget(BasicBlock *BB)
static std::string getBranchCondString(Instruction *TI)
static bool annotateAllFunctions(Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, vfs::FileSystem &FS, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, ProfileSummaryInfo *PSI, bool IsCS)
static cl::opt< PGOViewCountsType > PGOViewRawCounts("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
static const char * ValueProfKindDescr[]
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
static void setBranchWeights(SwitchInst *SI, ArrayRef< uint32_t > Weights)
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Defines the virtual file system interface vfs::FileSystem.
void printAsOperand(OutputBuffer &OB, Prec P=Prec::Default, bool StrictlyWorse=false) const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Class for arbitrary precision integers.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
std::optional< uint64_t > getBlockProfileCount(const BasicBlock *BB, bool AllowSynthetic=false) const
Returns the estimated profile count of BB.
Conditional or Unconditional Branch instruction.
bool isConditional() const
Value * getCondition() const
Analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
An union-find based Minimum Spanning Tree for CFG.
Edge & addEdge(BasicBlock *Src, BasicBlock *Dest, uint64_t W)
std::vector< std::unique_ptr< Edge > > AllEdges
BBInfo * findBBInfo(const BasicBlock *BB) const
BBInfo & getBBInfo(const BasicBlock *BB) const
DenseMap< const BasicBlock *, std::unique_ptr< BBInfo > > BBInfos
void dumpEdges(raw_ostream &OS, const Twine &Message) const
Predicate getPredicate() const
Return the predicate for this instruction.
StringRef getName() const
void setSelectionKind(SelectionKind Val)
SelectionKind getSelectionKind() const
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Base class for error info classes.
virtual std::string message() const
Return the error message as a string.
Lightweight error class with error context and mandatory checking.
Tagged union holding either a T or a Error.
Class to represent profile counts.
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
@ HiddenVisibility
The GV is hidden.
@ ExternalLinkage
Externally visible function.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
This instruction compares its operands according to the predicate given to the constructor.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Reader for the indexed binary instrprof format.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)
Return the NamedInstrProfRecord associated with FuncName and FuncHash.
uint64_t getMaximumFunctionCount(bool UseCS)
Return the maximum of all known function counts.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
RetTy visitSelectInst(SelectInst &I)
instrprof_error get() const
std::string message() const override
Return the error message as a string.
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
const BasicBlock * getParent() const
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
void update(ArrayRef< uint8_t > Data)
This is an important class for using LLVM in a threaded context.
MDString * createString(StringRef Str)
Return the given string as metadata.
MDNode * createIrrLoopHeaderWeight(uint64_t Weight)
Return metadata containing an irreducible loop header weight.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
PGOInstrumentationUse(std::string Filename="", std::string RemappingFilename="", bool IsCS=false, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
uint64_t getOrCompColdCountThreshold() const
Returns ColdCountThreshold if set.
void refresh()
If no summary is present, attempt to refresh.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
uint64_t getOrCompHotCountThreshold() const
Returns HotCountThreshold if set.
This class represents the LLVM 'select' instruction.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
TinyPtrVector - This class is specialized for cases where there are normally 0 or 1 element in a vect...
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
The instances of the Type class are immutable: once they are created, they are never changed.
void print(raw_ostream &O, bool IsForDebug=false, bool NoDetails=false) const
Print the current type.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
Utility analysis that determines what values are worth profiling.
std::vector< CandidateInfo > get(InstrProfValueKind Kind) const
returns a list of value profiling candidates of the given kind
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to an std::string.
The virtual file system interface.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
This file contains the declaration of the Comdat class, which represents a single COMDAT in LLVM.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const CustomOperand< const MCSubtargetInfo & > Msg[]
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
uint64_t getFuncHash(const FuncRecordTy *Record)
Return the structural hash associated with the function.
void checkExpectAnnotations(Instruction &I, const ArrayRef< uint32_t > ExistingWeights, bool IsFrontend)
checkExpectAnnotations - compares PGO counters to the thresholds used for llvm.expect and warns if th...
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< FuncNode * > Func
void write64le(void *P, uint64_t V)
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Interval::succ_iterator succ_end(Interval *I)
void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
Function::ProfileCount ProfileCount
auto successors(const MachineBasicBlock *BB)
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
DenseMap< BasicBlock *, ColorVector > colorEHFunclets(Function &F)
If an EH funclet personality is in use (see isFuncletEHPersonality), this will recompute which blocks...
cl::opt< bool > PGOWarnMissing
raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")
bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, BranchProbabilityInfo *BPI=nullptr, BlockFrequencyInfo *BFI=nullptr)
OperandBundleDefT< Value * > OperandBundleDef
cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))
GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
cl::opt< bool > NoPGOWarnMismatch
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
cl::opt< PGOViewCountsType > PGOViewCounts("pgo-view-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text with " "block profile counts and branch probabilities " "right after PGO profile annotation step. The " "profile counts are computed using branch " "probabilities from the runtime profile data and " "block frequency propagation algorithm. To view " "the raw counts from the profile, use option " "-pgo-view-raw-counts instead. To limit graph " "display to only one function, use filtering option " "-view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If this edge is a critical edge, insert a new node to split the critical edge.
void ViewGraph(const GraphType &G, const Twine &Name, bool ShortNames=false, const Twine &Title="", GraphProgram::Name Program=GraphProgram::DOT)
ViewGraph - Emit a dot graph, run 'dot', run gv on the postscript file, then cleanup.
bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges=false)
Return true if the specified edge is a critical edge.
static uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
auto predecessors(const MachineBasicBlock *BB)
void setProfMetadata(Module *M, Instruction *TI, ArrayRef< uint64_t > EdgeCounts, uint64_t MaxCount)
cl::opt< bool > DebugInfoCorrelate("debug-info-correlate", cl::desc("Use debug info to correlate profiles."), cl::init(false))
SuccIterator< const Instruction, const BasicBlock > const_succ_iterator
cl::opt< bool > NoPGOWarnMismatchComdatWeak
Implement std::hash so that hash_code can be used in STL containers.
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
DOTGraphTraits(bool isSimple=false)
static std::string getGraphName(const PGOUseFunc *G)
std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph)
DOTGraphTraits - Template class that can be specialized to customize how graphs are converted to 'dot...
DefaultDOTGraphTraits - This class provides the default implementations of all of the DOTGraphTraits ...
static ChildIteratorType child_end(const NodeRef N)
static NodeRef getEntryNode(const PGOUseFunc *G)
static ChildIteratorType child_begin(const NodeRef N)
static nodes_iterator nodes_end(const PGOUseFunc *G)
static nodes_iterator nodes_begin(const PGOUseFunc *G)
Profiling information for a single function.
std::vector< uint64_t > Counts
CountPseudoKind getCountPseudoKind() const
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
static void setCSFlagInHash(uint64_t &FuncHash)
Instruction * AnnotatedInst