LLVM  10.0.0svn
Classes | Namespaces | Macros | Typedefs | Enumerations | Functions | Variables
PGOInstrumentation.cpp File Reference
#include "CFGMST.h"
#include "ValueProfileCollector.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Comdat.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ProfileSummary.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Pass.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/CRC.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/MisExpect.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <memory>
#include <numeric>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include "llvm/ProfileData/InstrProfData.inc"

Go to the source code of this file.

Classes

struct  llvm::GraphTraits< PGOUseFunc * >
 
struct  llvm::DOTGraphTraits< PGOUseFunc * >
 

Namespaces

 llvm
 This class represents lattice values for constants.
 

Macros

#define DEBUG_TYPE   "pgo-instrumentation"
 
#define VALUE_PROF_KIND(Enumerator, Value, Descr)   Descr,
 

Typedefs

using ProfileCount = Function::ProfileCount
 
using VPCandidateInfo = ValueProfileCollector::CandidateInfo
 

Enumerations

enum  VisitMode
 The select instruction visitor plays three roles specified by the mode. More...
 

Functions

 STATISTIC (NumOfPGOInstrument, "Number of edges instrumented.")
 
 STATISTIC (NumOfPGOSelectInsts, "Number of select instruction instrumented.")
 
 STATISTIC (NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented.")
 
 STATISTIC (NumOfPGOEdge, "Number of edges.")
 
 STATISTIC (NumOfPGOBB, "Number of basic-blocks.")
 
 STATISTIC (NumOfPGOSplit, "Number of critical edge splits.")
 
 STATISTIC (NumOfPGOFunc, "Number of functions having valid profile counts.")
 
 STATISTIC (NumOfPGOMismatch, "Number of functions having mismatch profile.")
 
 STATISTIC (NumOfPGOMissing, "Number of functions without profile.")
 
 STATISTIC (NumOfPGOICall, "Number of indirect call value instrumentations.")
 
 STATISTIC (NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO.")
 
 STATISTIC (NumOfCSPGOSelectInsts, "Number of select instruction instrumented in CSPGO.")
 
 STATISTIC (NumOfCSPGOMemIntrinsics, "Number of mem intrinsics instrumented in CSPGO.")
 
 STATISTIC (NumOfCSPGOEdge, "Number of edges in CSPGO.")
 
 STATISTIC (NumOfCSPGOBB, "Number of basic-blocks in CSPGO.")
 
 STATISTIC (NumOfCSPGOSplit, "Number of critical edge splits in CSPGO.")
 
 STATISTIC (NumOfCSPGOFunc, "Number of functions having valid profile counts in CSPGO.")
 
 STATISTIC (NumOfCSPGOMismatch, "Number of functions having mismatch profile in CSPGO.")
 
 STATISTIC (NumOfCSPGOMissing, "Number of functions without profile in CSPGO.")
 
static std::string getBranchCondString (Instruction *TI)
 
 INITIALIZE_PASS_BEGIN (PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass
 
 INITIALIZE_PASS_BEGIN (PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass
 
 INITIALIZE_PASS (PGOInstrumentationGenCreateVarLegacyPass, "pgo-instr-gen-create-var", "Create PGO instrumentation version variable for CSPGO.", false, false) ModulePass *llvm
 
static bool canRenameComdat (Function &F, std::unordered_multimap< Comdat *, GlobalValue *> &ComdatMembers)
 
static void instrumentOneFunc (Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, std::unordered_multimap< Comdat *, GlobalValue *> &ComdatMembers, bool IsCS)
 
static uint64_t sumEdgeCount (const ArrayRef< PGOUseEdge *> Edges)
 
static bool isIndirectBrTarget (BasicBlock *BB)
 
static void collectComdatMembers (Module &M, std::unordered_multimap< Comdat *, GlobalValue *> &ComdatMembers)
 
static bool InstrumentAllFunctions (Module &M, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS)
 
static bool annotateAllFunctions (Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, ProfileSummaryInfo *PSI, bool IsCS)
 
static std::string getSimpleNodeName (const BasicBlock *Node)
 
void llvm::setIrrLoopHeaderMetadata (Module *M, Instruction *TI, uint64_t Count)
 

Variables

static cl::opt< std::string > PGOTestProfileFile ("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose."))
 
static cl::opt< std::string > PGOTestProfileRemappingFile ("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))
 
static cl::opt< boolDisableValueProfiling ("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
 
static cl::opt< unsignedMaxNumAnnotations ("icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of annotations for a single indirect " "call callsite"))
 
static cl::opt< unsignedMaxNumMemOPAnnotations ("memop-max-annotations", cl::init(4), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic"))
 
static cl::opt< boolDoComdatRenaming ("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))
 
static cl::opt< boolPGOWarnMissing ("pgo-warn-missing-function", cl::init(false), cl::Hidden, cl::desc("Use this option to turn on/off " "warnings about missing profile data for " "functions."))
 
static cl::opt< boolNoPGOWarnMismatch ("no-pgo-warn-mismatch", cl::init(false), cl::Hidden, cl::desc("Use this option to turn off/on " "warnings about profile cfg mismatch."))
 
static cl::opt< boolNoPGOWarnMismatchComdat ("no-pgo-warn-mismatch-comdat", cl::init(true), cl::Hidden, cl::desc("The option is used to turn on/off " "warnings about hash mismatch for comdat " "functions."))
 
static cl::opt< boolPGOInstrSelect ("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))
 
static cl::opt< PGOViewCountsTypePGOViewRawCounts ("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
 
static cl::opt< boolPGOInstrMemOP ("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))
 
static cl::opt< boolEmitBranchProbability ("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))
 
cl::opt< PGOViewCountsType > PGOViewCounts
 
cl::opt< std::string > ViewBlockFreqFuncName
 
static const charValueProfKindDescr []
 
pgo instr gen
 
pgo instr PGO instrumentation
 
pgo instr PGO false
 
pgo instr use
 
pgo instr Read PGO instrumentation profile
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "pgo-instrumentation"

Definition at line 126 of file PGOInstrumentation.cpp.

Referenced by llvm::setProfMetadata().

◆ VALUE_PROF_KIND

#define VALUE_PROF_KIND (   Enumerator,
  Value,
  Descr 
)    Descr,

Typedef Documentation

◆ ProfileCount

Definition at line 123 of file PGOInstrumentation.cpp.

◆ VPCandidateInfo

Definition at line 124 of file PGOInstrumentation.cpp.

Enumeration Type Documentation

◆ VisitMode

enum VisitMode

The select instruction visitor plays three roles specified by the mode.

In VM_counting mode, it simply counts the number of select instructions. In VM_instrument mode, it inserts code to count the number times TrueValue of select is taken. In VM_annotate mode, it reads the profile data and annotate the select instruction with metadata.

Definition at line 303 of file PGOInstrumentation.cpp.

Function Documentation

◆ annotateAllFunctions()

static bool annotateAllFunctions ( Module M,
StringRef  ProfileFileName,
StringRef  ProfileRemappingFileName,
function_ref< BranchProbabilityInfo *(Function &)>  LookupBPI,
function_ref< BlockFrequencyInfo *(Function &)>  LookupBFI,
ProfileSummaryInfo PSI,
bool  IsCS 
)
static

◆ canRenameComdat()

static bool canRenameComdat ( Function F,
std::unordered_multimap< Comdat *, GlobalValue *> &  ComdatMembers 
)
static

◆ collectComdatMembers()

static void collectComdatMembers ( Module M,
std::unordered_multimap< Comdat *, GlobalValue *> &  ComdatMembers 
)
static

Definition at line 1445 of file PGOInstrumentation.cpp.

References C, DoComdatRenaming, and F().

Referenced by annotateAllFunctions(), and InstrumentAllFunctions().

◆ getBranchCondString()

static std::string getBranchCondString ( Instruction TI)
static

Definition at line 260 of file PGOInstrumentation.cpp.

References llvm::dyn_cast(), and llvm::BranchInst::isConditional().

Referenced by llvm::setProfMetadata().

◆ getSimpleNodeName()

static std::string getSimpleNodeName ( const BasicBlock Node)
static

Definition at line 1686 of file PGOInstrumentation.cpp.

◆ INITIALIZE_PASS()

INITIALIZE_PASS ( PGOInstrumentationGenCreateVarLegacyPass  ,
"pgo-instr-gen-create-var"  ,
"Create PGO instrumentation version variable for CSPGO."  ,
false  ,
false   
)

◆ INITIALIZE_PASS_BEGIN() [1/2]

INITIALIZE_PASS_BEGIN ( PGOInstrumentationGenLegacyPass  ,
"pgo-instr-gen ,
"PGO instrumentation."  ,
false  ,
false   
)

◆ INITIALIZE_PASS_BEGIN() [2/2]

INITIALIZE_PASS_BEGIN ( PGOInstrumentationUseLegacyPass  ,
"pgo-instr-use ,
"Read PGO instrumentation profile."  ,
false  ,
false   
)

◆ InstrumentAllFunctions()

static bool InstrumentAllFunctions ( Module M,
function_ref< BranchProbabilityInfo *(Function &)>  LookupBPI,
function_ref< BlockFrequencyInfo *(Function &)>  LookupBFI,
bool  IsCS 
)
static

◆ instrumentOneFunc()

static void instrumentOneFunc ( Function F,
Module M,
BranchProbabilityInfo BPI,
BlockFrequencyInfo BFI,
std::unordered_multimap< Comdat *, GlobalValue *> &  ComdatMembers,
bool  IsCS 
)
static

◆ isIndirectBrTarget()

static bool isIndirectBrTarget ( BasicBlock BB)
static

◆ STATISTIC() [1/19]

STATISTIC ( NumOfPGOInstrument  ,
"Number of edges instrumented."   
)

◆ STATISTIC() [2/19]

STATISTIC ( NumOfPGOSelectInsts  ,
"Number of select instruction instrumented."   
)

◆ STATISTIC() [3/19]

STATISTIC ( NumOfPGOMemIntrinsics  ,
"Number of mem intrinsics instrumented."   
)

◆ STATISTIC() [4/19]

STATISTIC ( NumOfPGOEdge  ,
"Number of edges."   
)

◆ STATISTIC() [5/19]

STATISTIC ( NumOfPGOBB  ,
"Number of basic-blocks."   
)

◆ STATISTIC() [6/19]

STATISTIC ( NumOfPGOSplit  ,
"Number of critical edge splits."   
)

◆ STATISTIC() [7/19]

STATISTIC ( NumOfPGOFunc  ,
"Number of functions having valid profile counts."   
)

◆ STATISTIC() [8/19]

STATISTIC ( NumOfPGOMismatch  ,
"Number of functions having mismatch profile."   
)

◆ STATISTIC() [9/19]

STATISTIC ( NumOfPGOMissing  ,
"Number of functions without profile."   
)

◆ STATISTIC() [10/19]

STATISTIC ( NumOfPGOICall  ,
"Number of indirect call value instrumentations."   
)

◆ STATISTIC() [11/19]

STATISTIC ( NumOfCSPGOInstrument  ,
"Number of edges instrumented in CSPGO."   
)

◆ STATISTIC() [12/19]

STATISTIC ( NumOfCSPGOSelectInsts  ,
"Number of select instruction instrumented in CSPGO."   
)

◆ STATISTIC() [13/19]

STATISTIC ( NumOfCSPGOMemIntrinsics  ,
"Number of mem intrinsics instrumented in CSPGO."   
)

◆ STATISTIC() [14/19]

STATISTIC ( NumOfCSPGOEdge  ,
"Number of edges in CSPGO."   
)

◆ STATISTIC() [15/19]

STATISTIC ( NumOfCSPGOBB  ,
"Number of basic-blocks in CSPGO."   
)

◆ STATISTIC() [16/19]

STATISTIC ( NumOfCSPGOSplit  ,
"Number of critical edge splits in CSPGO."   
)

◆ STATISTIC() [17/19]

STATISTIC ( NumOfCSPGOFunc  ,
"Number of functions having valid profile counts in CSPGO."   
)

◆ STATISTIC() [18/19]

STATISTIC ( NumOfCSPGOMismatch  ,
"Number of functions having mismatch profile in CSPGO."   
)

◆ STATISTIC() [19/19]

STATISTIC ( NumOfCSPGOMissing  ,
"Number of functions without profile in CSPGO."   
)

◆ sumEdgeCount()

static uint64_t sumEdgeCount ( const ArrayRef< PGOUseEdge *>  Edges)
static

Variable Documentation

◆ DisableValueProfiling

cl::opt<bool> DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
static

◆ DoComdatRenaming

cl::opt<bool> DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))
static

◆ EmitBranchProbability

cl::opt<bool> EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))
static

Referenced by llvm::setProfMetadata().

◆ false

pgo instr Read PGO instrumentation false

Definition at line 438 of file PGOInstrumentation.cpp.

◆ gen

pgo instr gen

Definition at line 438 of file PGOInstrumentation.cpp.

◆ instrumentation

pgo instr PGO instrumentation

Definition at line 438 of file PGOInstrumentation.cpp.

◆ MaxNumAnnotations

cl::opt<unsigned> MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of annotations for a single indirect " "call callsite"))
static

Referenced by isIndirectBrTarget().

◆ MaxNumMemOPAnnotations

cl::opt<unsigned> MaxNumMemOPAnnotations("memop-max-annotations", cl::init(4), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic"))
static

Referenced by isIndirectBrTarget().

◆ NoPGOWarnMismatch

cl::opt<bool> NoPGOWarnMismatch("no-pgo-warn-mismatch", cl::init(false), cl::Hidden, cl::desc("Use this option to turn off/on " "warnings about profile cfg mismatch."))
static

◆ NoPGOWarnMismatchComdat

cl::opt<bool> NoPGOWarnMismatchComdat("no-pgo-warn-mismatch-comdat", cl::init(true), cl::Hidden, cl::desc("The option is used to turn on/off " "warnings about hash mismatch for comdat " "functions."))
static

◆ PGOInstrMemOP

cl::opt<bool> PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))
static

Referenced by instrumentOneFunc().

◆ PGOInstrSelect

cl::opt<bool> PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))
static

Referenced by isIndirectBrTarget().

◆ PGOTestProfileFile

cl::opt<std::string> PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose."))
static

◆ PGOTestProfileRemappingFile

cl::opt<std::string> PGOTestProfileRemappingFile("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))
static

◆ PGOViewCounts

cl::opt<PGOViewCountsType> PGOViewCounts

◆ PGOViewRawCounts

cl::opt<PGOViewCountsType> PGOViewRawCounts("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
static

Referenced by annotateAllFunctions().

◆ PGOWarnMissing

cl::opt<bool> PGOWarnMissing("pgo-warn-missing-function", cl::init(false), cl::Hidden, cl::desc("Use this option to turn on/off " "warnings about missing profile data for " "functions."))
static

◆ profile

pgo instr Read PGO instrumentation profile

Definition at line 452 of file PGOInstrumentation.cpp.

◆ use

pgo instr use

Definition at line 452 of file PGOInstrumentation.cpp.

◆ ValueProfKindDescr

const char* ValueProfKindDescr[]
static
Initial value:
= {
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
}

Definition at line 291 of file PGOInstrumentation.cpp.

Referenced by instrumentOneFunc(), and isIndirectBrTarget().

◆ ViewBlockFreqFuncName

cl::opt<std::string> ViewBlockFreqFuncName