LLVM 22.0.0git
ProfDataUtils.h
Go to the documentation of this file.
1//===- llvm/IR/ProfDataUtils.h - Profiling Metadata Utilities ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// @file
10/// This file contains the declarations for profiling metadata utility
11/// functions.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_PROFDATAUTILS_H
16#define LLVM_IR_PROFDATAUTILS_H
17
19#include "llvm/IR/Metadata.h"
21
22namespace llvm {
24 LLVM_ABI static const char *BranchWeights;
25 LLVM_ABI static const char *ValueProfile;
26 LLVM_ABI static const char *FunctionEntryCount;
28 LLVM_ABI static const char *ExpectedBranchWeights;
30};
31
32/// Profile-based loop metadata that should be accessed only by using
33/// \c llvm::getLoopEstimatedTripCount and \c llvm::setLoopEstimatedTripCount.
34LLVM_ABI extern const char *LLVMLoopEstimatedTripCount;
35
36/// Checks if an Instruction has MD_prof Metadata
37LLVM_ABI bool hasProfMD(const Instruction &I);
38
39/// Checks if an MDNode contains Branch Weight Metadata
40LLVM_ABI bool isBranchWeightMD(const MDNode *ProfileData);
41
42/// Checks if an MDNode contains value profiling Metadata
43LLVM_ABI bool isValueProfileMD(const MDNode *ProfileData);
44
45/// Checks if an instructions has Branch Weight Metadata
46///
47/// \param I The instruction to check
48/// \returns True if I has an MD_prof node containing Branch Weights. False
49/// otherwise.
51
52/// Checks if an instructions has valid Branch Weight Metadata
53///
54/// \param I The instruction to check
55/// \returns True if I has an MD_prof node containing valid Branch Weights,
56/// i.e., one weight for each successor. False otherwise.
58
59/// Get the branch weights metadata node
60///
61/// \param I The Instruction to get the weights from.
62/// \returns A pointer to I's branch weights metadata node, if it exists.
63/// Nullptr otherwise.
65
66/// Get the valid branch weights metadata node
67///
68/// \param I The Instruction to get the weights from.
69/// \returns A pointer to I's valid branch weights metadata node, if it exists.
70/// Nullptr otherwise.
72
73/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
74/// intrinsic
76
77/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
78/// intrinsic
79LLVM_ABI bool hasBranchWeightOrigin(const MDNode *ProfileData);
80
81/// Return the offset to the first branch weight data
82LLVM_ABI unsigned getBranchWeightOffset(const MDNode *ProfileData);
83
84LLVM_ABI unsigned getNumBranchWeights(const MDNode &ProfileData);
85
86/// Extract branch weights from MD_prof metadata
87///
88/// \param ProfileData A pointer to an MDNode.
89/// \param [out] Weights An output vector to fill with branch weights
90/// \returns True if weights were extracted, False otherwise. When false Weights
91/// will be cleared.
92LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData,
94
95/// Faster version of extractBranchWeights() that skips checks and must only
96/// be called with "branch_weights" metadata nodes. Supports uint32_t.
97LLVM_ABI void extractFromBranchWeightMD32(const MDNode *ProfileData,
99
100/// Faster version of extractBranchWeights() that skips checks and must only
101/// be called with "branch_weights" metadata nodes. Supports uint64_t.
102LLVM_ABI void extractFromBranchWeightMD64(const MDNode *ProfileData,
104
105/// Extract branch weights attatched to an Instruction
106///
107/// \param I The Instruction to extract weights from.
108/// \param [out] Weights An output vector to fill with branch weights
109/// \returns True if weights were extracted, False otherwise. When false Weights
110/// will be cleared.
113
114/// Extract branch weights from a conditional branch or select Instruction.
115///
116/// \param I The instruction to extract branch weights from.
117/// \param [out] TrueVal will contain the branch weight for the True branch
118/// \param [out] FalseVal will contain the branch weight for the False branch
119/// \returns True on success with profile weights filled in. False if no
120/// metadata or invalid metadata was found.
122 uint64_t &FalseVal);
123
124/// Retrieve the total of all weights from MD_prof data.
125///
126/// \param ProfileData The profile data to extract the total weight from
127/// \param [out] TotalWeights input variable to fill with total weights
128/// \returns True on success with profile total weights filled in. False if no
129/// metadata was found.
130LLVM_ABI bool extractProfTotalWeight(const MDNode *ProfileData,
131 uint64_t &TotalWeights);
132
133/// Retrieve the total of all weights from an instruction.
134///
135/// \param I The instruction to extract the total weight from
136/// \param [out] TotalWeights input variable to fill with total weights
137/// \returns True on success with profile total weights filled in. False if no
138/// metadata was found.
140 uint64_t &TotalWeights);
141
142/// Create a new `branch_weights` metadata node and add or overwrite
143/// a `prof` metadata reference to instruction `I`.
144/// \param I the Instruction to set branch weights on.
145/// \param Weights an array of weights to set on instruction I.
146/// \param IsExpected were these weights added from an llvm.expect* intrinsic.
148 bool IsExpected);
149
150/// downscale the given weights preserving the ratio. If the maximum value is
151/// not already known and not provided via \param KnownMaxCount , it will be
152/// obtained from \param Weights.
155 std::optional<uint64_t> KnownMaxCount = std::nullopt);
156
157/// Calculate what to divide by to scale counts.
158///
159/// Given the maximum count, calculate a divisor that will scale all the
160/// weights to strictly less than std::numeric_limits<uint32_t>::max().
162 return MaxCount < std::numeric_limits<uint32_t>::max()
163 ? 1
164 : MaxCount / std::numeric_limits<uint32_t>::max() + 1;
165}
166
167/// Scale an individual branch count.
168///
169/// Scale a 64-bit weight down to 32-bits using \c Scale.
170///
172 uint64_t Scaled = Count / Scale;
173 assert(Scaled <= std::numeric_limits<uint32_t>::max() && "overflow 32-bits");
174 return Scaled;
175}
176
177/// Specify that the branch weights for this terminator cannot be known at
178/// compile time. This should only be called by passes, and never as a default
179/// behavior in e.g. MDBuilder. The goal is to use this info to validate passes
180/// do not accidentally drop profile info, and this API is called in cases where
181/// the pass explicitly cannot provide that info. Defaulting it in would hide
182/// bugs where the pass forgets to transfer over or otherwise specify profile
183/// info. Use `PassName` to capture the pass name (i.e. DEBUG_TYPE) for
184/// debuggability.
186 StringRef PassName);
187
188/// Analogous to setExplicitlyUnknownBranchWeights, but for functions and their
189/// entry counts.
191 StringRef PassName);
192
193LLVM_ABI bool isExplicitlyUnknownProfileMetadata(const MDNode &MD);
194LLVM_ABI bool hasExplicitlyUnknownBranchWeights(const Instruction &I);
195
196/// Scaling the profile data attached to 'I' using the ratio of S/T.
197LLVM_ABI void scaleProfData(Instruction &I, uint64_t S, uint64_t T);
198
199/// Get the branch weights of a branch conditioned on b1 || b2, where b1 and b2
200/// are 2 booleans that are the conditions of 2 branches for which we have the
201/// branch weights B1 and B2, respectively. In both B1 and B2, the first
202/// position (index 0) is for the 'true' branch, and the second position (index
203/// 1) is for the 'false' branch.
206 const SmallVector<uint32_t, 2> &B2) {
207 // For the first conditional branch, the probability the "true" case is taken
208 // is p(b1) = B1[0] / (B1[0] + B1[1]). The "false" case's probability is
209 // p(not b1) = B1[1] / (B1[0] + B1[1]).
210 // Similarly for the second conditional branch and B2.
211 //
212 // The probability of the new branch NOT being taken is:
213 // not P = p((not b1) and (not b2)) =
214 // = B1[1] / (B1[0]+B1[1]) * B2[1] / (B2[0]+B2[1]) =
215 // = B1[1] * B2[1] / (B1[0] + B1[1]) * (B2[0] + B2[1])
216 // Then the probability of it being taken is: P = 1 - (not P).
217 // The denominator will be the same as above, and the numerator of P will be:
218 // (B1[0] + B1[1]) * (B2[0] + B2[1]) - B1[1]*B2[1]
219 // Which then reduces to what's shown below (out of the 4 terms coming out of
220 // the product of sums, the subtracted one cancels out).
221 assert(B1.size() == 2);
222 assert(B2.size() == 2);
223 auto FalseWeight = B1[1] * B2[1];
224 auto TrueWeight = B1[0] * B2[0] + B1[0] * B2[1] + B1[1] * B2[0];
225 return {TrueWeight, FalseWeight};
226}
227} // namespace llvm
228#endif
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
@ Scaled
#define LLVM_ABI
Definition Compiler.h:213
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
This file contains the declarations for metadata subclasses.
#define T
This file defines the SmallVector class.
static const char PassName[]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
Metadata node.
Definition Metadata.h:1077
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalWeights)
Retrieve the total of all weights from MD_prof data.
LLVM_ABI unsigned getBranchWeightOffset(const MDNode *ProfileData)
Return the offset to the first branch weight data.
LLVM_ABI bool isBranchWeightMD(const MDNode *ProfileData)
Checks if an MDNode contains Branch Weight Metadata.
LLVM_ABI bool isExplicitlyUnknownProfileMetadata(const MDNode &MD)
LLVM_ABI MDNode * getBranchWeightMDNode(const Instruction &I)
Get the branch weights metadata node.
LLVM_ABI void setExplicitlyUnknownBranchWeights(Instruction &I, StringRef PassName)
Specify that the branch weights for this terminator cannot be known at compile time.
LLVM_ABI bool hasBranchWeightOrigin(const Instruction &I)
Check if Branch Weight Metadata has an "expected" field from an llvm.expect* intrinsic.
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
LLVM_ABI MDNode * getValidBranchWeightMDNode(const Instruction &I)
Get the valid branch weights metadata node.
SmallVector< uint64_t, 2 > getDisjunctionWeights(const SmallVector< uint32_t, 2 > &B1, const SmallVector< uint32_t, 2 > &B2)
Get the branch weights of a branch conditioned on b1 || b2, where b1 and b2 are 2 booleans that are t...
LLVM_ABI bool hasValidBranchWeightMD(const Instruction &I)
Checks if an instructions has valid Branch Weight Metadata.
FunctionAddr VTableAddr Count
Definition InstrProf.h:139
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI void setExplicitlyUnknownFunctionEntryCount(Function &F, StringRef PassName)
Analogous to setExplicitlyUnknownBranchWeights, but for functions and their entry counts.
LLVM_ABI bool isValueProfileMD(const MDNode *ProfileData)
Checks if an MDNode contains value profiling Metadata.
LLVM_ABI unsigned getNumBranchWeights(const MDNode &ProfileData)
LLVM_ABI void extractFromBranchWeightMD32(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...
LLVM_ABI bool hasExplicitlyUnknownBranchWeights(const Instruction &I)
LLVM_ABI bool hasProfMD(const Instruction &I)
Checks if an Instruction has MD_prof Metadata.
LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
LLVM_ABI const char * LLVMLoopEstimatedTripCount
Profile-based loop metadata that should be accessed only by using llvm::getLoopEstimatedTripCount and...
uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)
Scale an individual branch count.
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
uint64_t calculateCountScale(uint64_t MaxCount)
Calculate what to divide by to scale counts.
LLVM_ABI SmallVector< uint32_t > downscaleWeights(ArrayRef< uint64_t > Weights, std::optional< uint64_t > KnownMaxCount=std::nullopt)
downscale the given weights preserving the ratio.
LLVM_ABI void scaleProfData(Instruction &I, uint64_t S, uint64_t T)
Scaling the profile data attached to 'I' using the ratio of S/T.
LLVM_ABI void extractFromBranchWeightMD64(const MDNode *ProfileData, SmallVectorImpl< uint64_t > &Weights)
Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...
static LLVM_ABI const char * ExpectedBranchWeights
static LLVM_ABI const char * SyntheticFunctionEntryCount
static LLVM_ABI const char * BranchWeights
static LLVM_ABI const char * FunctionEntryCount
static LLVM_ABI const char * UnknownBranchWeightsMarker
static LLVM_ABI const char * ValueProfile