LLVM 17.0.0git
MIRSampleProfile.cpp
Go to the documentation of this file.
1//===-------- MIRSampleProfile.cpp: MIRSampleFDO (For FSAFDO) -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides the implementation of the MIRSampleProfile loader, mainly
10// for flow sensitive SampleFDO.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/DenseSet.h"
24#include "llvm/CodeGen/Passes.h"
25#include "llvm/IR/Function.h"
28#include "llvm/Support/Debug.h"
33
34using namespace llvm;
35using namespace sampleprof;
36using namespace llvm::sampleprofutil;
38
39#define DEBUG_TYPE "fs-profile-loader"
40
42 "show-fs-branchprob", cl::Hidden, cl::init(false),
43 cl::desc("Print setting flow sensitive branch probabilities"));
45 "fs-profile-debug-prob-diff-threshold", cl::init(10),
46 cl::desc("Only show debug message if the branch probility is greater than "
47 "this value (in percentage)."));
48
50 "fs-profile-debug-bw-threshold", cl::init(10000),
51 cl::desc("Only show debug message if the source branch weight is greater "
52 " than this value."));
53
54static cl::opt<bool> ViewBFIBefore("fs-viewbfi-before", cl::Hidden,
55 cl::init(false),
56 cl::desc("View BFI before MIR loader"));
57static cl::opt<bool> ViewBFIAfter("fs-viewbfi-after", cl::Hidden,
58 cl::init(false),
59 cl::desc("View BFI after MIR loader"));
60
63
65 "Load MIR Sample Profile",
66 /* cfg = */ false, /* is_analysis = */ false)
73 /* cfg = */ false, /* is_analysis = */ false)
74
76
78llvm::createMIRProfileLoaderPass(std::string File, std::string RemappingFile,
80 IntrusiveRefCntPtr<vfs::FileSystem> FS) {
81 return new MIRProfileLoaderPass(File, RemappingFile, P, std::move(FS));
82}
83
84namespace llvm {
85
86// Internal option used to control BFI display only after MBP pass.
87// Defined in CodeGen/MachineBlockFrequencyInfo.cpp:
88// -view-block-layout-with-bfi={none | fraction | integer | count}
90
91// Command line option to specify the name of the function for CFG dump
92// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
94
95namespace afdo_detail {
96template <> struct IRTraits<MachineBasicBlock> {
110 static Function &getFunction(MachineFunction &F) { return F.getFunction(); }
113 }
115 return BB->predecessors();
116 }
118 return BB->successors();
119 }
120};
121} // namespace afdo_detail
122
124 : public SampleProfileLoaderBaseImpl<MachineBasicBlock> {
125public:
129 DT = MDT;
130 PDT = MPDT;
131 LI = MLI;
132 BFI = MBFI;
133 ORE = MORE;
134 }
136 P = Pass;
139 assert(LowBit < HighBit && "HighBit needs to be greater than Lowbit");
140 }
141
145 std::move(FS)) {}
146
149 bool doInitialization(Module &M);
150 bool isValid() const { return ProfileIsValid; }
151
152protected:
154
155 /// Hold the information of the basic block frequency.
157
158 /// PassNum is the sequence number this pass is called, start from 1.
160
161 // LowBit in the FS discriminator used by this instance. Note the number is
162 // 0-based. Base discrimnator use bit 0 to bit 11.
163 unsigned LowBit;
164 // HighwBit in the FS discriminator used by this instance. Note the number
165 // is 0-based.
166 unsigned HighBit;
167
168 bool ProfileIsValid = true;
170 if (ImprovedFSDiscriminator && MI.isMetaInstruction())
171 return std::error_code();
172 return getInstWeightImpl(MI);
173 }
174};
175
176template <>
178 MachineBasicBlock>::computeDominanceAndLoopInfo(MachineFunction &F) {}
179
181 LLVM_DEBUG(dbgs() << "\nPropagation complete. Setting branch probs\n");
182 for (auto &BI : F) {
183 MachineBasicBlock *BB = &BI;
184 if (BB->succ_size() < 2)
185 continue;
186 const MachineBasicBlock *EC = EquivalenceClass[BB];
187 uint64_t BBWeight = BlockWeights[EC];
188 uint64_t SumEdgeWeight = 0;
189 for (MachineBasicBlock *Succ : BB->successors()) {
190 Edge E = std::make_pair(BB, Succ);
191 SumEdgeWeight += EdgeWeights[E];
192 }
193
194 if (BBWeight != SumEdgeWeight) {
195 LLVM_DEBUG(dbgs() << "BBweight is not equal to SumEdgeWeight: BBWWeight="
196 << BBWeight << " SumEdgeWeight= " << SumEdgeWeight
197 << "\n");
198 BBWeight = SumEdgeWeight;
199 }
200 if (BBWeight == 0) {
201 LLVM_DEBUG(dbgs() << "SKIPPED. All branch weights are zero.\n");
202 continue;
203 }
204
205#ifndef NDEBUG
206 uint64_t BBWeightOrig = BBWeight;
207#endif
208 uint32_t MaxWeight = std::numeric_limits<uint32_t>::max();
209 uint32_t Factor = 1;
210 if (BBWeight > MaxWeight) {
211 Factor = BBWeight / MaxWeight + 1;
212 BBWeight /= Factor;
213 LLVM_DEBUG(dbgs() << "Scaling weights by " << Factor << "\n");
214 }
215
217 SE = BB->succ_end();
218 SI != SE; ++SI) {
219 MachineBasicBlock *Succ = *SI;
220 Edge E = std::make_pair(BB, Succ);
221 uint64_t EdgeWeight = EdgeWeights[E];
222 EdgeWeight /= Factor;
223
224 assert(BBWeight >= EdgeWeight &&
225 "BBweight is larger than EdgeWeight -- should not happen.\n");
226
228 BranchProbability NewProb(EdgeWeight, BBWeight);
229 if (OldProb == NewProb)
230 continue;
231 BB->setSuccProbability(SI, NewProb);
232#ifndef NDEBUG
233 if (!ShowFSBranchProb)
234 continue;
235 bool Show = false;
237 if (OldProb > NewProb)
238 Diff = OldProb - NewProb;
239 else
240 Diff = NewProb - OldProb;
242 Show &= (BBWeightOrig >= FSProfileDebugBWThreshold);
243
244 auto DIL = BB->findBranchDebugLoc();
245 auto SuccDIL = Succ->findBranchDebugLoc();
246 if (Show) {
247 dbgs() << "Set branch fs prob: MBB (" << BB->getNumber() << " -> "
248 << Succ->getNumber() << "): ";
249 if (DIL)
250 dbgs() << DIL->getFilename() << ":" << DIL->getLine() << ":"
251 << DIL->getColumn();
252 if (SuccDIL)
253 dbgs() << "-->" << SuccDIL->getFilename() << ":" << SuccDIL->getLine()
254 << ":" << SuccDIL->getColumn();
255 dbgs() << " W=" << BBWeightOrig << " " << OldProb << " --> " << NewProb
256 << "\n";
257 }
258#endif
259 }
260 }
261}
262
264 auto &Ctx = M.getContext();
265
267 Filename, Ctx, *FS, P, RemappingFilename);
268 if (std::error_code EC = ReaderOrErr.getError()) {
269 std::string Msg = "Could not open profile: " + EC.message();
270 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
271 return false;
272 }
273
274 Reader = std::move(ReaderOrErr.get());
275 Reader->setModule(&M);
277 Reader->getSummary();
278
279 return true;
280}
281
283 Function &Func = MF.getFunction();
284 clearFunctionData(false);
285 Samples = Reader->getSamplesFor(Func);
286 if (!Samples || Samples->empty())
287 return false;
288
289 if (getFunctionLoc(MF) == 0)
290 return false;
291
292 DenseSet<GlobalValue::GUID> InlinedGUIDs;
293 bool Changed = computeAndPropagateWeights(MF, InlinedGUIDs);
294
295 // Set the new BPI, BFI.
296 setBranchProbs(MF);
297
298 return Changed;
299}
300
301} // namespace llvm
302
304 std::string FileName, std::string RemappingFileName, FSDiscriminatorPass P,
306 : MachineFunctionPass(ID), ProfileFileName(FileName), P(P) {
307 LowBit = getFSPassBitBegin(P);
308 HighBit = getFSPassBitEnd(P);
309
310 auto VFS = FS ? std::move(FS) : vfs::getRealFileSystem();
311 MIRSampleLoader = std::make_unique<MIRProfileLoader>(
312 FileName, RemappingFileName, std::move(VFS));
313 assert(LowBit < HighBit && "HighBit needs to be greater than Lowbit");
314}
315
316bool MIRProfileLoaderPass::runOnMachineFunction(MachineFunction &MF) {
317 if (!MIRSampleLoader->isValid())
318 return false;
319
320 LLVM_DEBUG(dbgs() << "MIRProfileLoader pass working on Func: "
321 << MF.getFunction().getName() << "\n");
322 MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
323 MIRSampleLoader->setInitVals(
324 &getAnalysis<MachineDominatorTree>(),
325 &getAnalysis<MachinePostDominatorTree>(), &getAnalysis<MachineLoopInfo>(),
326 MBFI, &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE());
327
328 MF.RenumberBlocks();
330 (ViewBlockFreqFuncName.empty() ||
332 MBFI->view("MIR_Prof_loader_b." + MF.getName(), false);
333 }
334
335 bool Changed = MIRSampleLoader->runOnFunction(MF);
336 if (Changed)
337 MBFI->calculate(MF, *MBFI->getMBPI(), *&getAnalysis<MachineLoopInfo>());
338
340 (ViewBlockFreqFuncName.empty() ||
342 MBFI->view("MIR_prof_loader_a." + MF.getName(), false);
343 }
344
345 return Changed;
346}
347
348bool MIRProfileLoaderPass::doInitialization(Module &M) {
349 LLVM_DEBUG(dbgs() << "MIRProfileLoader pass working on Module " << M.getName()
350 << "\n");
351
352 MIRSampleLoader->setFSPass(P);
353 return MIRSampleLoader->doInitialization(M);
354}
355
356void MIRProfileLoaderPass::getAnalysisUsage(AnalysisUsage &AU) const {
357 AU.setPreservesAll();
364}
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
std::string Name
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
cl::opt< bool > ImprovedFSDiscriminator("improved-fs-discriminator", cl::Hidden, cl::init(false), cl::desc("New FS discriminators encoding (incompatible with the original " "encoding)"))
Load MIR Sample Profile
static cl::opt< bool > ShowFSBranchProb("show-fs-branchprob", cl::Hidden, cl::init(false), cl::desc("Print setting flow sensitive branch probabilities"))
static cl::opt< bool > ViewBFIAfter("fs-viewbfi-after", cl::Hidden, cl::init(false), cl::desc("View BFI after MIR loader"))
static cl::opt< unsigned > FSProfileDebugBWThreshold("fs-profile-debug-bw-threshold", cl::init(10000), cl::desc("Only show debug message if the source branch weight is greater " " than this value."))
cl::opt< bool > ImprovedFSDiscriminator
#define DEBUG_TYPE
static cl::opt< bool > ViewBFIBefore("fs-viewbfi-before", cl::Hidden, cl::init(false), cl::desc("View BFI before MIR loader"))
static cl::opt< unsigned > FSProfileDebugProbDiffThreshold("fs-profile-debug-prob-diff-threshold", cl::init(10), cl::desc("Only show debug message if the branch probility is greater than " "this value (in percentage)."))
===- MachineOptimizationRemarkEmitter.h - Opt Diagnostics -*- C++ -*-—===//
#define P(N)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
@ SI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file provides the interface for the sampled PGO profile loader base implementation.
This file provides the utility functions for the sampled PGO loader base implementation.
Defines the virtual file system interface vfs::FileSystem.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
AnalysisUsage & addRequiredTransitive()
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Diagnostic information for the sample profiler.
Represents either an error or a value T.
Definition: ErrorOr.h:56
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
Class to represent profile counts.
Definition: Function.h:252
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
MIRProfileLoaderPass(std::string FileName="", std::string RemappingFileName="", FSDiscriminatorPass P=FSDiscriminatorPass::Pass1, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
FS bits will only use the '1' bits in the Mask.
MIRProfileLoader(StringRef Name, StringRef RemapName, IntrusiveRefCntPtr< vfs::FileSystem > FS)
void setBranchProbs(MachineFunction &F)
ErrorOr< uint64_t > getInstWeight(const MachineInstr &MI) override
bool runOnFunction(MachineFunction &F)
MachineBlockFrequencyInfo * BFI
Hold the information of the basic block frequency.
FSDiscriminatorPass P
PassNum is the sequence number this pass is called, start from 1.
bool doInitialization(Module &M)
void setInitVals(MachineDominatorTree *MDT, MachinePostDominatorTree *MPDT, MachineLoopInfo *MLI, MachineBlockFrequencyInfo *MBFI, MachineOptimizationRemarkEmitter *MORE)
void setFSPass(FSDiscriminatorPass Pass)
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
unsigned succ_size() const
std::vector< MachineBasicBlock * >::iterator succ_iterator
DebugLoc findBranchDebugLoc()
Find and return the merged DebugLoc of the branch instructions of the block.
iterator_range< succ_iterator > successors()
iterator_range< pred_iterator > predecessors()
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
void view(const Twine &Name, bool isSimple=true) const
Pop up a ghostview window with the current block frequency propagation rendered using dot.
const MachineBranchProbabilityInfo * getMBPI() const
void calculate(const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI, const MachineLoopInfo &MLI)
calculate - compute block frequency info for the given function.
BranchProbability getEdgeProbability(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
void RenumberBlocks(MachineBasicBlock *MBBFrom=nullptr)
RenumberBlocks - This discards all of the MachineBasicBlock numbers and recomputes them.
Representation of each machine instruction.
Definition: MachineInstr.h:68
Diagnostic information for optimization analysis remarks.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:91
bool computeAndPropagateWeights(FunctionT &F, const DenseSet< GlobalValue::GUID > &InlinedGUIDs)
Generate branch weight metadata for all branches in F.
std::string Filename
Name of the profile file to load.
EquivalenceClassMap EquivalenceClass
Equivalence classes for block weights.
unsigned getFunctionLoc(FunctionT &Func)
Get the line number for the function header.
ErrorOr< uint64_t > getInstWeightImpl(const InstructionT &Inst)
IntrusiveRefCntPtr< vfs::FileSystem > FS
VirtualFileSystem to load profile files from.
std::string RemappingFilename
Name of the profile remapping file to load.
EdgeWeightMap EdgeWeights
Map edges to their computed weights.
FunctionSamples * Samples
Samples collected for the body of this function.
BlockWeightMap BlockWeights
Map basic blocks to their computed weights.
OptRemarkEmitterT * ORE
Optimization Remark Emitter used to emit diagnostic remarks.
void clearFunctionData(bool ResetDT=true)
Clear all the per-function data used to load samples and propagate weights.
DominatorTreePtrT DT
Dominance, post-dominance and loop information.
std::pair< const BasicBlockT *, const BasicBlockT * > Edge
std::unique_ptr< SampleProfileReader > Reader
Profile reader object.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:164
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
A range adaptor for a pair of iterators.
static ErrorOr< std::unique_ptr< SampleProfileReader > > create(const std::string Filename, LLVMContext &C, vfs::FileSystem &FS, FSDiscriminatorPass P=FSDiscriminatorPass::Base, const std::string RemapFilename="")
Create a sample profile reader appropriate to the file format.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
char & MIRProfileLoaderPassID
This pass reads flow sensitive profile.
static unsigned getFSPassBitEnd(FSDiscriminatorPass P)
Definition: Discriminator.h:89
cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
cl::opt< GVDAGType > ViewBlockLayoutWithBFI("view-block-layout-with-bfi", cl::Hidden, cl::desc("Pop up a window to show a dag displaying MBP layout and associated " "block frequencies of the CFG."), cl::values(clEnumValN(GVDT_None, "none", "do not display graphs."), clEnumValN(GVDT_Fraction, "fraction", "display a graph using the " "fractional block frequency representation."), clEnumValN(GVDT_Integer, "integer", "display a graph using the raw " "integer fractional block frequency representation."), clEnumValN(GVDT_Count, "count", "display a graph using the real " "profile count if available.")))
FunctionPass * createMIRProfileLoaderPass(std::string File, std::string RemappingFile, sampleprof::FSDiscriminatorPass P, IntrusiveRefCntPtr< vfs::FileSystem > FS)
Read Flow Sensitive Profile.
static unsigned getFSPassBitBegin(FSDiscriminatorPass P)
Definition: Discriminator.h:96
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1946
Definition: BitVector.h:858
#define MORE()
Definition: regcomp.c:252
static PredRangeT getPredecessors(MachineBasicBlock *BB)
static SuccRangeT getSuccessors(MachineBasicBlock *BB)
static const MachineBasicBlock * getEntryBB(const MachineFunction *F)
static Function & getFunction(MachineFunction &F)