LLVM 18.0.0git
MIRSampleProfile.cpp
Go to the documentation of this file.
1//===-------- MIRSampleProfile.cpp: MIRSampleFDO (For FSAFDO) -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides the implementation of the MIRSampleProfile loader, mainly
10// for flow sensitive SampleFDO.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/DenseSet.h"
25#include "llvm/CodeGen/Passes.h"
26#include "llvm/IR/Function.h"
27#include "llvm/IR/PseudoProbe.h"
30#include "llvm/Support/Debug.h"
35#include <optional>
36
37using namespace llvm;
38using namespace sampleprof;
39using namespace llvm::sampleprofutil;
41
42#define DEBUG_TYPE "fs-profile-loader"
43
45 "show-fs-branchprob", cl::Hidden, cl::init(false),
46 cl::desc("Print setting flow sensitive branch probabilities"));
48 "fs-profile-debug-prob-diff-threshold", cl::init(10),
49 cl::desc("Only show debug message if the branch probility is greater than "
50 "this value (in percentage)."));
51
53 "fs-profile-debug-bw-threshold", cl::init(10000),
54 cl::desc("Only show debug message if the source branch weight is greater "
55 " than this value."));
56
57static cl::opt<bool> ViewBFIBefore("fs-viewbfi-before", cl::Hidden,
58 cl::init(false),
59 cl::desc("View BFI before MIR loader"));
60static cl::opt<bool> ViewBFIAfter("fs-viewbfi-after", cl::Hidden,
61 cl::init(false),
62 cl::desc("View BFI after MIR loader"));
63
66
68 "Load MIR Sample Profile",
69 /* cfg = */ false, /* is_analysis = */ false)
76 /* cfg = */ false, /* is_analysis = */ false)
77
79
81llvm::createMIRProfileLoaderPass(std::string File, std::string RemappingFile,
83 IntrusiveRefCntPtr<vfs::FileSystem> FS) {
84 return new MIRProfileLoaderPass(File, RemappingFile, P, std::move(FS));
85}
86
87namespace llvm {
88
89// Internal option used to control BFI display only after MBP pass.
90// Defined in CodeGen/MachineBlockFrequencyInfo.cpp:
91// -view-block-layout-with-bfi={none | fraction | integer | count}
93
94// Command line option to specify the name of the function for CFG dump
95// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
97
98std::optional<PseudoProbe> extractProbe(const MachineInstr &MI) {
99 if (MI.isPseudoProbe()) {
100 PseudoProbe Probe;
101 Probe.Id = MI.getOperand(1).getImm();
102 Probe.Type = MI.getOperand(2).getImm();
103 Probe.Attr = MI.getOperand(3).getImm();
104 Probe.Factor = 1;
105 DILocation *DebugLoc = MI.getDebugLoc();
106 Probe.Discriminator = DebugLoc ? DebugLoc->getDiscriminator() : 0;
107 return Probe;
108 }
109
110 // Ignore callsite probes since they do not have FS discriminators.
111 return std::nullopt;
112}
113
114namespace afdo_detail {
115template <> struct IRTraits<MachineBasicBlock> {
129 static Function &getFunction(MachineFunction &F) { return F.getFunction(); }
132 }
134 return BB->predecessors();
135 }
137 return BB->successors();
138 }
139};
140} // namespace afdo_detail
141
143 : public SampleProfileLoaderBaseImpl<MachineFunction> {
144public:
148 DT = MDT;
149 PDT = MPDT;
150 LI = MLI;
151 BFI = MBFI;
152 ORE = MORE;
153 }
155 P = Pass;
158 assert(LowBit < HighBit && "HighBit needs to be greater than Lowbit");
159 }
160
164 std::move(FS)) {}
165
168 bool doInitialization(Module &M);
169 bool isValid() const { return ProfileIsValid; }
170
171protected:
173
174 /// Hold the information of the basic block frequency.
176
177 /// PassNum is the sequence number this pass is called, start from 1.
179
180 // LowBit in the FS discriminator used by this instance. Note the number is
181 // 0-based. Base discrimnator use bit 0 to bit 11.
182 unsigned LowBit;
183 // HighwBit in the FS discriminator used by this instance. Note the number
184 // is 0-based.
185 unsigned HighBit;
186
187 bool ProfileIsValid = true;
190 return getProbeWeight(MI);
191 if (ImprovedFSDiscriminator && MI.isMetaInstruction())
192 return std::error_code();
193 return getInstWeightImpl(MI);
194 }
195};
196
197template <>
199 MachineFunction &F) {}
200
202 LLVM_DEBUG(dbgs() << "\nPropagation complete. Setting branch probs\n");
203 for (auto &BI : F) {
204 MachineBasicBlock *BB = &BI;
205 if (BB->succ_size() < 2)
206 continue;
207 const MachineBasicBlock *EC = EquivalenceClass[BB];
208 uint64_t BBWeight = BlockWeights[EC];
209 uint64_t SumEdgeWeight = 0;
210 for (MachineBasicBlock *Succ : BB->successors()) {
211 Edge E = std::make_pair(BB, Succ);
212 SumEdgeWeight += EdgeWeights[E];
213 }
214
215 if (BBWeight != SumEdgeWeight) {
216 LLVM_DEBUG(dbgs() << "BBweight is not equal to SumEdgeWeight: BBWWeight="
217 << BBWeight << " SumEdgeWeight= " << SumEdgeWeight
218 << "\n");
219 BBWeight = SumEdgeWeight;
220 }
221 if (BBWeight == 0) {
222 LLVM_DEBUG(dbgs() << "SKIPPED. All branch weights are zero.\n");
223 continue;
224 }
225
226#ifndef NDEBUG
227 uint64_t BBWeightOrig = BBWeight;
228#endif
229 uint32_t MaxWeight = std::numeric_limits<uint32_t>::max();
230 uint32_t Factor = 1;
231 if (BBWeight > MaxWeight) {
232 Factor = BBWeight / MaxWeight + 1;
233 BBWeight /= Factor;
234 LLVM_DEBUG(dbgs() << "Scaling weights by " << Factor << "\n");
235 }
236
238 SE = BB->succ_end();
239 SI != SE; ++SI) {
240 MachineBasicBlock *Succ = *SI;
241 Edge E = std::make_pair(BB, Succ);
242 uint64_t EdgeWeight = EdgeWeights[E];
243 EdgeWeight /= Factor;
244
245 assert(BBWeight >= EdgeWeight &&
246 "BBweight is larger than EdgeWeight -- should not happen.\n");
247
248 BranchProbability OldProb = BFI->getMBPI()->getEdgeProbability(BB, SI);
249 BranchProbability NewProb(EdgeWeight, BBWeight);
250 if (OldProb == NewProb)
251 continue;
252 BB->setSuccProbability(SI, NewProb);
253#ifndef NDEBUG
254 if (!ShowFSBranchProb)
255 continue;
256 bool Show = false;
258 if (OldProb > NewProb)
259 Diff = OldProb - NewProb;
260 else
261 Diff = NewProb - OldProb;
263 Show &= (BBWeightOrig >= FSProfileDebugBWThreshold);
264
265 auto DIL = BB->findBranchDebugLoc();
266 auto SuccDIL = Succ->findBranchDebugLoc();
267 if (Show) {
268 dbgs() << "Set branch fs prob: MBB (" << BB->getNumber() << " -> "
269 << Succ->getNumber() << "): ";
270 if (DIL)
271 dbgs() << DIL->getFilename() << ":" << DIL->getLine() << ":"
272 << DIL->getColumn();
273 if (SuccDIL)
274 dbgs() << "-->" << SuccDIL->getFilename() << ":" << SuccDIL->getLine()
275 << ":" << SuccDIL->getColumn();
276 dbgs() << " W=" << BBWeightOrig << " " << OldProb << " --> " << NewProb
277 << "\n";
278 }
279#endif
280 }
281 }
282}
283
285 auto &Ctx = M.getContext();
286
288 Filename, Ctx, *FS, P, RemappingFilename);
289 if (std::error_code EC = ReaderOrErr.getError()) {
290 std::string Msg = "Could not open profile: " + EC.message();
291 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
292 return false;
293 }
294
295 Reader = std::move(ReaderOrErr.get());
296 Reader->setModule(&M);
298
299 // Load pseudo probe descriptors for probe-based function samples.
300 if (Reader->profileIsProbeBased()) {
301 ProbeManager = std::make_unique<PseudoProbeManager>(M);
302 if (!ProbeManager->moduleIsProbed(M)) {
303 return false;
304 }
305 }
306
307 return true;
308}
309
311 // Do not load non-FS profiles. A line or probe can get a zero-valued
312 // discriminator at certain pass which could result in accidentally loading
313 // the corresponding base counter in the non-FS profile, while a non-zero
314 // discriminator would end up getting zero samples. This could in turn undo
315 // the sample distribution effort done by previous BFI maintenance and the
316 // probe distribution factor work for pseudo probes.
317 if (!Reader->profileIsFS())
318 return false;
319
320 Function &Func = MF.getFunction();
321 clearFunctionData(false);
322 Samples = Reader->getSamplesFor(Func);
323 if (!Samples || Samples->empty())
324 return false;
325
327 if (!ProbeManager->profileIsValid(MF.getFunction(), *Samples))
328 return false;
329 } else {
330 if (getFunctionLoc(MF) == 0)
331 return false;
332 }
333
334 DenseSet<GlobalValue::GUID> InlinedGUIDs;
335 bool Changed = computeAndPropagateWeights(MF, InlinedGUIDs);
336
337 // Set the new BPI, BFI.
338 setBranchProbs(MF);
339
340 return Changed;
341}
342
343} // namespace llvm
344
346 std::string FileName, std::string RemappingFileName, FSDiscriminatorPass P,
348 : MachineFunctionPass(ID), ProfileFileName(FileName), P(P) {
349 LowBit = getFSPassBitBegin(P);
350 HighBit = getFSPassBitEnd(P);
351
352 auto VFS = FS ? std::move(FS) : vfs::getRealFileSystem();
353 MIRSampleLoader = std::make_unique<MIRProfileLoader>(
354 FileName, RemappingFileName, std::move(VFS));
355 assert(LowBit < HighBit && "HighBit needs to be greater than Lowbit");
356}
357
358bool MIRProfileLoaderPass::runOnMachineFunction(MachineFunction &MF) {
359 if (!MIRSampleLoader->isValid())
360 return false;
361
362 LLVM_DEBUG(dbgs() << "MIRProfileLoader pass working on Func: "
363 << MF.getFunction().getName() << "\n");
364 MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
365 MIRSampleLoader->setInitVals(
366 &getAnalysis<MachineDominatorTree>(),
367 &getAnalysis<MachinePostDominatorTree>(), &getAnalysis<MachineLoopInfo>(),
368 MBFI, &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE());
369
370 MF.RenumberBlocks();
372 (ViewBlockFreqFuncName.empty() ||
374 MBFI->view("MIR_Prof_loader_b." + MF.getName(), false);
375 }
376
377 bool Changed = MIRSampleLoader->runOnFunction(MF);
378 if (Changed)
379 MBFI->calculate(MF, *MBFI->getMBPI(), *&getAnalysis<MachineLoopInfo>());
380
382 (ViewBlockFreqFuncName.empty() ||
384 MBFI->view("MIR_prof_loader_a." + MF.getName(), false);
385 }
386
387 return Changed;
388}
389
390bool MIRProfileLoaderPass::doInitialization(Module &M) {
391 LLVM_DEBUG(dbgs() << "MIRProfileLoader pass working on Module " << M.getName()
392 << "\n");
393
394 MIRSampleLoader->setFSPass(P);
395 return MIRSampleLoader->doInitialization(M);
396}
397
398void MIRProfileLoaderPass::getAnalysisUsage(AnalysisUsage &AU) const {
399 AU.setPreservesAll();
406}
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
std::string Name
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
cl::opt< bool > ImprovedFSDiscriminator("improved-fs-discriminator", cl::Hidden, cl::init(false), cl::desc("New FS discriminators encoding (incompatible with the original " "encoding)"))
Load MIR Sample Profile
static cl::opt< bool > ShowFSBranchProb("show-fs-branchprob", cl::Hidden, cl::init(false), cl::desc("Print setting flow sensitive branch probabilities"))
static cl::opt< bool > ViewBFIAfter("fs-viewbfi-after", cl::Hidden, cl::init(false), cl::desc("View BFI after MIR loader"))
static cl::opt< unsigned > FSProfileDebugBWThreshold("fs-profile-debug-bw-threshold", cl::init(10000), cl::desc("Only show debug message if the source branch weight is greater " " than this value."))
cl::opt< bool > ImprovedFSDiscriminator
#define DEBUG_TYPE
static cl::opt< bool > ViewBFIBefore("fs-viewbfi-before", cl::Hidden, cl::init(false), cl::desc("View BFI before MIR loader"))
static cl::opt< unsigned > FSProfileDebugProbDiffThreshold("fs-profile-debug-prob-diff-threshold", cl::init(10), cl::desc("Only show debug message if the branch probility is greater than " "this value (in percentage)."))
===- MachineOptimizationRemarkEmitter.h - Opt Diagnostics -*- C++ -*-—===//
#define P(N)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file provides the interface for the sampled PGO profile loader base implementation.
This file provides the utility functions for the sampled PGO loader base implementation.
Defines the virtual file system interface vfs::FileSystem.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
AnalysisUsage & addRequiredTransitive()
Debug location.
A debug info location.
Definition: DebugLoc.h:33
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Diagnostic information for the sample profiler.
Represents either an error or a value T.
Definition: ErrorOr.h:56
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
Class to represent profile counts.
Definition: Function.h:254
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
MIRProfileLoaderPass(std::string FileName="", std::string RemappingFileName="", FSDiscriminatorPass P=FSDiscriminatorPass::Pass1, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
FS bits will only use the '1' bits in the Mask.
MIRProfileLoader(StringRef Name, StringRef RemapName, IntrusiveRefCntPtr< vfs::FileSystem > FS)
void setBranchProbs(MachineFunction &F)
ErrorOr< uint64_t > getInstWeight(const MachineInstr &MI) override
bool runOnFunction(MachineFunction &F)
MachineBlockFrequencyInfo * BFI
Hold the information of the basic block frequency.
FSDiscriminatorPass P
PassNum is the sequence number this pass is called, start from 1.
bool doInitialization(Module &M)
void setInitVals(MachineDominatorTree *MDT, MachinePostDominatorTree *MPDT, MachineLoopInfo *MLI, MachineBlockFrequencyInfo *MBFI, MachineOptimizationRemarkEmitter *MORE)
void setFSPass(FSDiscriminatorPass Pass)
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
unsigned succ_size() const
std::vector< MachineBasicBlock * >::iterator succ_iterator
DebugLoc findBranchDebugLoc()
Find and return the merged DebugLoc of the branch instructions of the block.
iterator_range< succ_iterator > successors()
iterator_range< pred_iterator > predecessors()
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
void view(const Twine &Name, bool isSimple=true) const
Pop up a ghostview window with the current block frequency propagation rendered using dot.
const MachineBranchProbabilityInfo * getMBPI() const
void calculate(const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI, const MachineLoopInfo &MLI)
calculate - compute block frequency info for the given function.
BranchProbability getEdgeProbability(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
void RenumberBlocks(MachineBasicBlock *MBBFrom=nullptr)
RenumberBlocks - This discards all of the MachineBasicBlock numbers and recomputes them.
Representation of each machine instruction.
Definition: MachineInstr.h:68
Diagnostic information for optimization analysis remarks.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
bool computeAndPropagateWeights(FunctionT &F, const DenseSet< GlobalValue::GUID > &InlinedGUIDs)
Generate branch weight metadata for all branches in F.
void computeDominanceAndLoopInfo(FunctionT &F)
IntrusiveRefCntPtr< vfs::FileSystem > FS
VirtualFileSystem to load profile files from.
EdgeWeightMap EdgeWeights
Map edges to their computed weights.
OptRemarkEmitterT * ORE
Optimization Remark Emitter used to emit diagnostic remarks.
unsigned getFunctionLoc(FunctionT &Func)
Get the line number for the function header.
ErrorOr< uint64_t > getInstWeightImpl(const InstructionT &Inst)
EquivalenceClassMap EquivalenceClass
Equivalence classes for block weights.
std::unique_ptr< SampleProfileReader > Reader
Profile reader object.
DominatorTreePtrT DT
Dominance, post-dominance and loop information.
std::string Filename
Name of the profile file to load.
virtual ErrorOr< uint64_t > getProbeWeight(const InstructionT &Inst)
std::string RemappingFilename
Name of the profile remapping file to load.
FunctionSamples * Samples
Samples collected for the body of this function.
std::pair< const BasicBlockT *, const BasicBlockT * > Edge
void clearFunctionData(bool ResetDT=true)
Clear all the per-function data used to load samples and propagate weights.
BlockWeightMap BlockWeights
Map basic blocks to their computed weights.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:164
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
A range adaptor for a pair of iterators.
static ErrorOr< std::unique_ptr< SampleProfileReader > > create(const std::string Filename, LLVMContext &C, vfs::FileSystem &FS, FSDiscriminatorPass P=FSDiscriminatorPass::Base, const std::string RemapFilename="")
Create a sample profile reader appropriate to the file format.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
static unsigned getFSPassBitBegin(sampleprof::FSDiscriminatorPass P)
Definition: Discriminator.h:94
char & MIRProfileLoaderPassID
This pass reads flow sensitive profile.
static unsigned getFSPassBitEnd(sampleprof::FSDiscriminatorPass P)
Definition: Discriminator.h:87
cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))
std::optional< PseudoProbe > extractProbe(const Instruction &Inst)
Definition: PseudoProbe.cpp:56
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
cl::opt< GVDAGType > ViewBlockLayoutWithBFI("view-block-layout-with-bfi", cl::Hidden, cl::desc("Pop up a window to show a dag displaying MBP layout and associated " "block frequencies of the CFG."), cl::values(clEnumValN(GVDT_None, "none", "do not display graphs."), clEnumValN(GVDT_Fraction, "fraction", "display a graph using the " "fractional block frequency representation."), clEnumValN(GVDT_Integer, "integer", "display a graph using the raw " "integer fractional block frequency representation."), clEnumValN(GVDT_Count, "count", "display a graph using the real " "profile count if available.")))
FunctionPass * createMIRProfileLoaderPass(std::string File, std::string RemappingFile, sampleprof::FSDiscriminatorPass P, IntrusiveRefCntPtr< vfs::FileSystem > FS)
Read Flow Sensitive Profile.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1854
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
#define MORE()
Definition: regcomp.c:252
uint32_t Discriminator
Definition: PseudoProbe.h:96
static PredRangeT getPredecessors(MachineBasicBlock *BB)
static SuccRangeT getSuccessors(MachineBasicBlock *BB)
static const MachineBasicBlock * getEntryBB(const MachineFunction *F)
static Function & getFunction(MachineFunction &F)