LLVM 19.0.0git
MIRSampleProfile.cpp
Go to the documentation of this file.
1//===-------- MIRSampleProfile.cpp: MIRSampleFDO (For FSAFDO) -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides the implementation of the MIRSampleProfile loader, mainly
10// for flow sensitive SampleFDO.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/DenseSet.h"
25#include "llvm/CodeGen/Passes.h"
26#include "llvm/IR/Function.h"
27#include "llvm/IR/PseudoProbe.h"
30#include "llvm/Support/Debug.h"
35#include <optional>
36
37using namespace llvm;
38using namespace sampleprof;
39using namespace llvm::sampleprofutil;
41
42#define DEBUG_TYPE "fs-profile-loader"
43
45 "show-fs-branchprob", cl::Hidden, cl::init(false),
46 cl::desc("Print setting flow sensitive branch probabilities"));
48 "fs-profile-debug-prob-diff-threshold", cl::init(10),
49 cl::desc("Only show debug message if the branch probility is greater than "
50 "this value (in percentage)."));
51
53 "fs-profile-debug-bw-threshold", cl::init(10000),
54 cl::desc("Only show debug message if the source branch weight is greater "
55 " than this value."));
56
57static cl::opt<bool> ViewBFIBefore("fs-viewbfi-before", cl::Hidden,
58 cl::init(false),
59 cl::desc("View BFI before MIR loader"));
60static cl::opt<bool> ViewBFIAfter("fs-viewbfi-after", cl::Hidden,
61 cl::init(false),
62 cl::desc("View BFI after MIR loader"));
63
64namespace llvm {
66}
68
70 "Load MIR Sample Profile",
71 /* cfg = */ false, /* is_analysis = */ false)
78 /* cfg = */ false, /* is_analysis = */ false)
79
81
83llvm::createMIRProfileLoaderPass(std::string File, std::string RemappingFile,
85 IntrusiveRefCntPtr<vfs::FileSystem> FS) {
86 return new MIRProfileLoaderPass(File, RemappingFile, P, std::move(FS));
87}
88
89namespace llvm {
90
91// Internal option used to control BFI display only after MBP pass.
92// Defined in CodeGen/MachineBlockFrequencyInfo.cpp:
93// -view-block-layout-with-bfi={none | fraction | integer | count}
95
96// Command line option to specify the name of the function for CFG dump
97// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
99
100std::optional<PseudoProbe> extractProbe(const MachineInstr &MI) {
101 if (MI.isPseudoProbe()) {
102 PseudoProbe Probe;
103 Probe.Id = MI.getOperand(1).getImm();
104 Probe.Type = MI.getOperand(2).getImm();
105 Probe.Attr = MI.getOperand(3).getImm();
106 Probe.Factor = 1;
107 DILocation *DebugLoc = MI.getDebugLoc();
108 Probe.Discriminator = DebugLoc ? DebugLoc->getDiscriminator() : 0;
109 return Probe;
110 }
111
112 // Ignore callsite probes since they do not have FS discriminators.
113 return std::nullopt;
114}
115
116namespace afdo_detail {
117template <> struct IRTraits<MachineBasicBlock> {
131 static Function &getFunction(MachineFunction &F) { return F.getFunction(); }
134 }
136 return BB->predecessors();
137 }
139 return BB->successors();
140 }
141};
142} // namespace afdo_detail
143
145 : public SampleProfileLoaderBaseImpl<MachineFunction> {
146public:
150 DT = MDT;
151 PDT = MPDT;
152 LI = MLI;
153 BFI = MBFI;
154 ORE = MORE;
155 }
157 P = Pass;
160 assert(LowBit < HighBit && "HighBit needs to be greater than Lowbit");
161 }
162
166 std::move(FS)) {}
167
170 bool doInitialization(Module &M);
171 bool isValid() const { return ProfileIsValid; }
172
173protected:
175
176 /// Hold the information of the basic block frequency.
178
179 /// PassNum is the sequence number this pass is called, start from 1.
181
182 // LowBit in the FS discriminator used by this instance. Note the number is
183 // 0-based. Base discrimnator use bit 0 to bit 11.
184 unsigned LowBit;
185 // HighwBit in the FS discriminator used by this instance. Note the number
186 // is 0-based.
187 unsigned HighBit;
188
189 bool ProfileIsValid = true;
192 return getProbeWeight(MI);
193 if (ImprovedFSDiscriminator && MI.isMetaInstruction())
194 return std::error_code();
195 return getInstWeightImpl(MI);
196 }
197};
198
199template <>
201 MachineFunction &F) {}
202
204 LLVM_DEBUG(dbgs() << "\nPropagation complete. Setting branch probs\n");
205 for (auto &BI : F) {
206 MachineBasicBlock *BB = &BI;
207 if (BB->succ_size() < 2)
208 continue;
209 const MachineBasicBlock *EC = EquivalenceClass[BB];
210 uint64_t BBWeight = BlockWeights[EC];
211 uint64_t SumEdgeWeight = 0;
212 for (MachineBasicBlock *Succ : BB->successors()) {
213 Edge E = std::make_pair(BB, Succ);
214 SumEdgeWeight += EdgeWeights[E];
215 }
216
217 if (BBWeight != SumEdgeWeight) {
218 LLVM_DEBUG(dbgs() << "BBweight is not equal to SumEdgeWeight: BBWWeight="
219 << BBWeight << " SumEdgeWeight= " << SumEdgeWeight
220 << "\n");
221 BBWeight = SumEdgeWeight;
222 }
223 if (BBWeight == 0) {
224 LLVM_DEBUG(dbgs() << "SKIPPED. All branch weights are zero.\n");
225 continue;
226 }
227
228#ifndef NDEBUG
229 uint64_t BBWeightOrig = BBWeight;
230#endif
231 uint32_t MaxWeight = std::numeric_limits<uint32_t>::max();
232 uint32_t Factor = 1;
233 if (BBWeight > MaxWeight) {
234 Factor = BBWeight / MaxWeight + 1;
235 BBWeight /= Factor;
236 LLVM_DEBUG(dbgs() << "Scaling weights by " << Factor << "\n");
237 }
238
240 SE = BB->succ_end();
241 SI != SE; ++SI) {
242 MachineBasicBlock *Succ = *SI;
243 Edge E = std::make_pair(BB, Succ);
244 uint64_t EdgeWeight = EdgeWeights[E];
245 EdgeWeight /= Factor;
246
247 assert(BBWeight >= EdgeWeight &&
248 "BBweight is larger than EdgeWeight -- should not happen.\n");
249
250 BranchProbability OldProb = BFI->getMBPI()->getEdgeProbability(BB, SI);
251 BranchProbability NewProb(EdgeWeight, BBWeight);
252 if (OldProb == NewProb)
253 continue;
254 BB->setSuccProbability(SI, NewProb);
255#ifndef NDEBUG
256 if (!ShowFSBranchProb)
257 continue;
258 bool Show = false;
260 if (OldProb > NewProb)
261 Diff = OldProb - NewProb;
262 else
263 Diff = NewProb - OldProb;
265 Show &= (BBWeightOrig >= FSProfileDebugBWThreshold);
266
267 auto DIL = BB->findBranchDebugLoc();
268 auto SuccDIL = Succ->findBranchDebugLoc();
269 if (Show) {
270 dbgs() << "Set branch fs prob: MBB (" << BB->getNumber() << " -> "
271 << Succ->getNumber() << "): ";
272 if (DIL)
273 dbgs() << DIL->getFilename() << ":" << DIL->getLine() << ":"
274 << DIL->getColumn();
275 if (SuccDIL)
276 dbgs() << "-->" << SuccDIL->getFilename() << ":" << SuccDIL->getLine()
277 << ":" << SuccDIL->getColumn();
278 dbgs() << " W=" << BBWeightOrig << " " << OldProb << " --> " << NewProb
279 << "\n";
280 }
281#endif
282 }
283 }
284}
285
287 auto &Ctx = M.getContext();
288
290 Filename, Ctx, *FS, P, RemappingFilename);
291 if (std::error_code EC = ReaderOrErr.getError()) {
292 std::string Msg = "Could not open profile: " + EC.message();
293 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
294 return false;
295 }
296
297 Reader = std::move(ReaderOrErr.get());
298 Reader->setModule(&M);
300
301 // Load pseudo probe descriptors for probe-based function samples.
302 if (Reader->profileIsProbeBased()) {
303 ProbeManager = std::make_unique<PseudoProbeManager>(M);
304 if (!ProbeManager->moduleIsProbed(M)) {
305 return false;
306 }
307 }
308
309 return true;
310}
311
313 // Do not load non-FS profiles. A line or probe can get a zero-valued
314 // discriminator at certain pass which could result in accidentally loading
315 // the corresponding base counter in the non-FS profile, while a non-zero
316 // discriminator would end up getting zero samples. This could in turn undo
317 // the sample distribution effort done by previous BFI maintenance and the
318 // probe distribution factor work for pseudo probes.
319 if (!Reader->profileIsFS())
320 return false;
321
322 Function &Func = MF.getFunction();
323 clearFunctionData(false);
324 Samples = Reader->getSamplesFor(Func);
325 if (!Samples || Samples->empty())
326 return false;
327
329 if (!ProbeManager->profileIsValid(MF.getFunction(), *Samples))
330 return false;
331 } else {
332 if (getFunctionLoc(MF) == 0)
333 return false;
334 }
335
336 DenseSet<GlobalValue::GUID> InlinedGUIDs;
337 bool Changed = computeAndPropagateWeights(MF, InlinedGUIDs);
338
339 // Set the new BPI, BFI.
340 setBranchProbs(MF);
341
342 return Changed;
343}
344
345} // namespace llvm
346
348 std::string FileName, std::string RemappingFileName, FSDiscriminatorPass P,
350 : MachineFunctionPass(ID), ProfileFileName(FileName), P(P) {
351 LowBit = getFSPassBitBegin(P);
352 HighBit = getFSPassBitEnd(P);
353
354 auto VFS = FS ? std::move(FS) : vfs::getRealFileSystem();
355 MIRSampleLoader = std::make_unique<MIRProfileLoader>(
356 FileName, RemappingFileName, std::move(VFS));
357 assert(LowBit < HighBit && "HighBit needs to be greater than Lowbit");
358}
359
360bool MIRProfileLoaderPass::runOnMachineFunction(MachineFunction &MF) {
361 if (!MIRSampleLoader->isValid())
362 return false;
363
364 LLVM_DEBUG(dbgs() << "MIRProfileLoader pass working on Func: "
365 << MF.getFunction().getName() << "\n");
366 MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
367 MIRSampleLoader->setInitVals(
368 &getAnalysis<MachineDominatorTree>(),
369 &getAnalysis<MachinePostDominatorTree>(), &getAnalysis<MachineLoopInfo>(),
370 MBFI, &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE());
371
372 MF.RenumberBlocks();
374 (ViewBlockFreqFuncName.empty() ||
376 MBFI->view("MIR_Prof_loader_b." + MF.getName(), false);
377 }
378
379 bool Changed = MIRSampleLoader->runOnFunction(MF);
380 if (Changed)
381 MBFI->calculate(MF, *MBFI->getMBPI(), *&getAnalysis<MachineLoopInfo>());
382
384 (ViewBlockFreqFuncName.empty() ||
386 MBFI->view("MIR_prof_loader_a." + MF.getName(), false);
387 }
388
389 return Changed;
390}
391
392bool MIRProfileLoaderPass::doInitialization(Module &M) {
393 LLVM_DEBUG(dbgs() << "MIRProfileLoader pass working on Module " << M.getName()
394 << "\n");
395
396 MIRSampleLoader->setFSPass(P);
397 return MIRSampleLoader->doInitialization(M);
398}
399
400void MIRProfileLoaderPass::getAnalysisUsage(AnalysisUsage &AU) const {
401 AU.setPreservesAll();
408}
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
std::string Name
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
Load MIR Sample Profile
static cl::opt< bool > ShowFSBranchProb("show-fs-branchprob", cl::Hidden, cl::init(false), cl::desc("Print setting flow sensitive branch probabilities"))
static cl::opt< bool > ViewBFIAfter("fs-viewbfi-after", cl::Hidden, cl::init(false), cl::desc("View BFI after MIR loader"))
static cl::opt< unsigned > FSProfileDebugBWThreshold("fs-profile-debug-bw-threshold", cl::init(10000), cl::desc("Only show debug message if the source branch weight is greater " " than this value."))
#define DEBUG_TYPE
static cl::opt< bool > ViewBFIBefore("fs-viewbfi-before", cl::Hidden, cl::init(false), cl::desc("View BFI before MIR loader"))
static cl::opt< unsigned > FSProfileDebugProbDiffThreshold("fs-profile-debug-prob-diff-threshold", cl::init(10), cl::desc("Only show debug message if the branch probility is greater than " "this value (in percentage)."))
===- MachineOptimizationRemarkEmitter.h - Opt Diagnostics -*- C++ -*-—===//
#define P(N)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file provides the interface for the sampled PGO profile loader base implementation.
This file provides the utility functions for the sampled PGO loader base implementation.
Defines the virtual file system interface vfs::FileSystem.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
AnalysisUsage & addRequiredTransitive()
Debug location.
A debug info location.
Definition: DebugLoc.h:33
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Diagnostic information for the sample profiler.
Represents either an error or a value T.
Definition: ErrorOr.h:56
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
Class to represent profile counts.
Definition: Function.h:277
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
MIRProfileLoaderPass(std::string FileName="", std::string RemappingFileName="", FSDiscriminatorPass P=FSDiscriminatorPass::Pass1, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
FS bits will only use the '1' bits in the Mask.
MIRProfileLoader(StringRef Name, StringRef RemapName, IntrusiveRefCntPtr< vfs::FileSystem > FS)
void setBranchProbs(MachineFunction &F)
ErrorOr< uint64_t > getInstWeight(const MachineInstr &MI) override
bool runOnFunction(MachineFunction &F)
MachineBlockFrequencyInfo * BFI
Hold the information of the basic block frequency.
FSDiscriminatorPass P
PassNum is the sequence number this pass is called, start from 1.
bool doInitialization(Module &M)
void setInitVals(MachineDominatorTree *MDT, MachinePostDominatorTree *MPDT, MachineLoopInfo *MLI, MachineBlockFrequencyInfo *MBFI, MachineOptimizationRemarkEmitter *MORE)
void setFSPass(FSDiscriminatorPass Pass)
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
unsigned succ_size() const
std::vector< MachineBasicBlock * >::iterator succ_iterator
DebugLoc findBranchDebugLoc()
Find and return the merged DebugLoc of the branch instructions of the block.
iterator_range< succ_iterator > successors()
iterator_range< pred_iterator > predecessors()
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
void view(const Twine &Name, bool isSimple=true) const
Pop up a ghostview window with the current block frequency propagation rendered using dot.
const MachineBranchProbabilityInfo * getMBPI() const
void calculate(const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI, const MachineLoopInfo &MLI)
calculate - compute block frequency info for the given function.
BranchProbability getEdgeProbability(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
void RenumberBlocks(MachineBasicBlock *MBBFrom=nullptr)
RenumberBlocks - This discards all of the MachineBasicBlock numbers and recomputes them.
Representation of each machine instruction.
Definition: MachineInstr.h:68
Diagnostic information for optimization analysis remarks.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
bool computeAndPropagateWeights(FunctionT &F, const DenseSet< GlobalValue::GUID > &InlinedGUIDs)
Generate branch weight metadata for all branches in F.
void computeDominanceAndLoopInfo(FunctionT &F)
IntrusiveRefCntPtr< vfs::FileSystem > FS
VirtualFileSystem to load profile files from.
EdgeWeightMap EdgeWeights
Map edges to their computed weights.
OptRemarkEmitterT * ORE
Optimization Remark Emitter used to emit diagnostic remarks.
unsigned getFunctionLoc(FunctionT &Func)
Get the line number for the function header.
ErrorOr< uint64_t > getInstWeightImpl(const InstructionT &Inst)
EquivalenceClassMap EquivalenceClass
Equivalence classes for block weights.
std::unique_ptr< SampleProfileReader > Reader
Profile reader object.
DominatorTreePtrT DT
Dominance, post-dominance and loop information.
std::string Filename
Name of the profile file to load.
virtual ErrorOr< uint64_t > getProbeWeight(const InstructionT &Inst)
std::string RemappingFilename
Name of the profile remapping file to load.
FunctionSamples * Samples
Samples collected for the body of this function.
std::pair< const BasicBlockT *, const BasicBlockT * > Edge
void clearFunctionData(bool ResetDT=true)
Clear all the per-function data used to load samples and propagate weights.
BlockWeightMap BlockWeights
Map basic blocks to their computed weights.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:164
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
A range adaptor for a pair of iterators.
static ErrorOr< std::unique_ptr< SampleProfileReader > > create(const std::string Filename, LLVMContext &C, vfs::FileSystem &FS, FSDiscriminatorPass P=FSDiscriminatorPass::Base, const std::string RemapFilename="")
Create a sample profile reader appropriate to the file format.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
static unsigned getFSPassBitBegin(sampleprof::FSDiscriminatorPass P)
Definition: Discriminator.h:94
char & MIRProfileLoaderPassID
This pass reads flow sensitive profile.
static unsigned getFSPassBitEnd(sampleprof::FSDiscriminatorPass P)
Definition: Discriminator.h:87
cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))
std::optional< PseudoProbe > extractProbe(const Instruction &Inst)
Definition: PseudoProbe.cpp:56
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
cl::opt< GVDAGType > ViewBlockLayoutWithBFI("view-block-layout-with-bfi", cl::Hidden, cl::desc("Pop up a window to show a dag displaying MBP layout and associated " "block frequencies of the CFG."), cl::values(clEnumValN(GVDT_None, "none", "do not display graphs."), clEnumValN(GVDT_Fraction, "fraction", "display a graph using the " "fractional block frequency representation."), clEnumValN(GVDT_Integer, "integer", "display a graph using the raw " "integer fractional block frequency representation."), clEnumValN(GVDT_Count, "count", "display a graph using the real " "profile count if available.")))
cl::opt< bool > ImprovedFSDiscriminator("improved-fs-discriminator", cl::Hidden, cl::init(false), cl::desc("New FS discriminators encoding (incompatible with the original " "encoding)"))
FunctionPass * createMIRProfileLoaderPass(std::string File, std::string RemappingFile, sampleprof::FSDiscriminatorPass P, IntrusiveRefCntPtr< vfs::FileSystem > FS)
Read Flow Sensitive Profile.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1858
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
#define MORE()
Definition: regcomp.c:252
uint32_t Discriminator
Definition: PseudoProbe.h:96
static PredRangeT getPredecessors(MachineBasicBlock *BB)
static SuccRangeT getSuccessors(MachineBasicBlock *BB)
static const MachineBasicBlock * getEntryBB(const MachineFunction *F)
static Function & getFunction(MachineFunction &F)