34#define DEBUG_TYPE "lower-expect-intrinsic"
37 "Number of 'expect' intrinsic instructions handled");
53 cl::desc(
"Weight of the branch likely to be taken (default = 2000)"));
56 cl::desc(
"Weight of the branch unlikely to be taken (default = 1)"));
58static std::tuple<uint32_t, uint32_t>
60 if (IntrinsicID == Intrinsic::expect) {
67 "expect with probability must have 3 arguments");
69 double TrueProb = Confidence->getValueAPF().convertToDouble();
70 assert((TrueProb >= 0.0 && TrueProb <= 1.0) &&
71 "probability value must be in the range [0.0, 1.0]");
72 double FalseProb = (1.0 - TrueProb) / (BranchCount - 1);
73 uint32_t LikelyBW = ceil((TrueProb * (
double)(INT32_MAX - 1)) + 1.0);
74 uint32_t UnlikelyBW = ceil((FalseProb * (
double)(INT32_MAX - 1)) + 1.0);
75 return std::make_tuple(LikelyBW, UnlikelyBW);
80 CallInst *CI = dyn_cast<CallInst>(
SI.getCondition());
95 unsigned n =
SI.getNumCases();
96 uint32_t LikelyBranchWeightVal, UnlikelyBranchWeightVal;
97 std::tie(LikelyBranchWeightVal, UnlikelyBranchWeightVal) =
103 Weights[
Index] = LikelyBranchWeightVal;
107 SI.setCondition(ArgValue);
109 SI.setMetadata(LLVMContext::MD_prof,
128 bool ExpectedValueIsLikely =
true;
135 auto *Confidence = cast<ConstantFP>(Expect->
getArgOperand(2));
136 double TrueProb = Confidence->getValueAPF().convertToDouble();
137 ExpectedValueIsLikely = (TrueProb > 0.5);
152 while (!isa<PHINode>(V)) {
153 if (
ZExtInst *ZExt = dyn_cast<ZExtInst>(V)) {
154 V = ZExt->getOperand(0);
159 if (
SExtInst *SExt = dyn_cast<SExtInst>(V)) {
160 V = SExt->getOperand(0);
166 if (!BinOp || BinOp->
getOpcode() != Instruction::Xor)
178 auto ApplyOperations = [&](
const APInt &
Value) {
181 switch (Op->getOpcode()) {
182 case Instruction::Xor:
183 Result ^= cast<ConstantInt>(Op->getOperand(1))->getValue();
185 case Instruction::ZExt:
186 Result = Result.zext(Op->getType()->getIntegerBitWidth());
188 case Instruction::SExt:
189 Result = Result.sext(Op->getType()->getIntegerBitWidth());
198 auto *PhiDef = cast<PHINode>(V);
202 auto GetDomConditional = [&](
unsigned i) ->
BranchInst * {
219 for (
unsigned i = 0, e = PhiDef->getNumIncomingValues(); i != e; ++i) {
221 Value *PhiOpnd = PhiDef->getIncomingValue(i);
231 const APInt &CurrentPhiValue = ApplyOperations(CI->
getValue());
232 if (ExpectedValueIsLikely == (ExpectedPhiValue == CurrentPhiValue))
249 auto *OpndIncomingBB = PhiDef->getIncomingBlock(i);
250 auto IsOpndComingFromSuccessor = [&](
BasicBlock *Succ) {
251 if (OpndIncomingBB == Succ)
262 uint32_t LikelyBranchWeightVal, UnlikelyBranchWeightVal;
263 std::tie(LikelyBranchWeightVal, UnlikelyBranchWeightVal) =
getBranchWeight(
265 if (!ExpectedValueIsLikely)
266 std::swap(LikelyBranchWeightVal, UnlikelyBranchWeightVal);
271 UnlikelyBranchWeightVal));
272 else if (IsOpndComingFromSuccessor(BI->
getSuccessor(0)))
275 LikelyBranchWeightVal));
293 ICmpInst *CmpI = dyn_cast<ICmpInst>(BSI.getCondition());
297 CI = dyn_cast<CallInst>(BSI.getCondition());
304 CmpConstOperand = dyn_cast<ConstantInt>(CmpI->
getOperand(1));
305 if (!CmpConstOperand)
314 if (CmpConstOperand) {
333 uint32_t LikelyBranchWeightVal, UnlikelyBranchWeightVal;
334 std::tie(LikelyBranchWeightVal, UnlikelyBranchWeightVal) =
338 if ((ExpectedValue->
getZExtValue() == ValueComparedTo) ==
342 ExpectedWeights = {LikelyBranchWeightVal, UnlikelyBranchWeightVal};
346 ExpectedWeights = {UnlikelyBranchWeightVal, LikelyBranchWeightVal};
352 BSI.setCondition(ArgValue);
356 BSI.setMetadata(LLVMContext::MD_prof,
Node);
365 return handleBrSelExpect<BranchInst>(BI);
369 bool Changed =
false;
373 if (
BranchInst *BI = dyn_cast<BranchInst>(BB.getTerminator())) {
375 ExpectIntrinsicsHandled++;
376 }
else if (
SwitchInst *
SI = dyn_cast<SwitchInst>(BB.getTerminator())) {
378 ExpectIntrinsicsHandled++;
385 CallInst *CI = dyn_cast<CallInst>(&Inst);
389 ExpectIntrinsicsHandled++;
438char LowerExpectIntrinsic::ID = 0;
440 "Lower 'expect' Intrinsics",
false,
false)
443 return new LowerExpectIntrinsic();
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool runOnFunction(Function &F, bool PostInlining)
static cl::opt< uint32_t > UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(1), cl::desc("Weight of the branch unlikely to be taken (default = 1)"))
static bool handleBranchExpect(BranchInst &BI)
static std::tuple< uint32_t, uint32_t > getBranchWeight(Intrinsic::ID IntrinsicID, CallInst *CI, int BranchCount)
static bool handleBrSelExpect(BrSelInst &BSI)
static cl::opt< uint32_t > LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(2000), cl::desc("Weight of the branch likely to be taken (default = 2000)"))
static bool handleSwitchExpect(SwitchInst &SI)
static bool lowerExpectIntrinsic(Function &F)
static void handlePhiDef(CallInst *Expect)
Handler for PHINodes that define the value argument to an @llvm.expect call.
The header file for the LowerExpectIntrinsic pass as used by the new pass manager.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Class for arbitrary precision integers.
A container for analyses that lazily runs them and caches their results.
LLVM Basic Block Representation.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
BinaryOps getOpcode() const
Conditional or Unconditional Branch instruction.
bool isConditional() const
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Predicate getPredicate() const
Return the predicate for this instruction.
This is the shared class of boolean and integer constants.
unsigned getBitWidth() const
getBitWidth - Return the bitwidth of this constant.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
FunctionPass class - This class is used to implement most global optimizations.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
This instruction compares its operands according to the predicate given to the constructor.
const BasicBlock * getParent() const
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
unsigned getCaseIndex() const
Returns number of current case.
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
This class represents zero extension of integer types.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
initializer< Ty > init(const Ty &Val)
void checkFrontendInstrumentation(Instruction &I, const ArrayRef< uint32_t > ExpectedWeights)
checkFrontendInstrumentation - compares PGO counters to the thresholds used for llvm....
void checkExpectAnnotations(Instruction &I, const ArrayRef< uint32_t > ExistingWeights, bool IsFrontend)
checkExpectAnnotations - compares PGO counters to the thresholds used for llvm.expect and warns if th...
This is an optimization pass for GlobalISel generic memory operations.
FunctionPass * createLowerExpectIntrinsicPass()
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
auto reverse(ContainerTy &&C)
void initializeLowerExpectIntrinsicPass(PassRegistry &)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &)
Run the pass over the function.