Bug Summary

File:include/llvm/Support/Error.h
Warning:line 200, column 5
Potential leak of memory pointed to by 'Payload._M_t._M_head_impl'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name PGOInstrumentation.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-9/lib/clang/9.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/include -I /build/llvm-toolchain-snapshot-9~svn362543/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/9.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-9/lib/clang/9.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/lib/Transforms/Instrumentation -fdebug-prefix-map=/build/llvm-toolchain-snapshot-9~svn362543=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2019-06-05-060531-1271-1 -x c++ /build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp -faddrsig

/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp

→
1//===- PGOInstrumentation.cpp - MST-based PGO Instrumentation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements PGO instrumentation using a minimum spanning tree based
10// on the following paper:
11// [1] Donald E. Knuth, Francis R. Stevenson. Optimal measurement of points
12// for program frequency counts. BIT Numerical Mathematics 1973, Volume 13,
13// Issue 3, pp 313-322
14// The idea of the algorithm based on the fact that for each node (except for
15// the entry and exit), the sum of incoming edge counts equals the sum of
16// outgoing edge counts. The count of edge on spanning tree can be derived from
17// those edges not on the spanning tree. Knuth proves this method instruments
18// the minimum number of edges.
19//
20// The minimal spanning tree here is actually a maximum weight tree -- on-tree
21// edges have higher frequencies (more likely to execute). The idea is to
22// instrument those less frequently executed edges to reduce the runtime
23// overhead of instrumented binaries.
24//
25// This file contains two passes:
26// (1) Pass PGOInstrumentationGen which instruments the IR to generate edge
27// count profile, and generates the instrumentation for indirect call
28// profiling.
29// (2) Pass PGOInstrumentationUse which reads the edge count profile and
30// annotates the branch weights. It also reads the indirect call value
31// profiling records and annotate the indirect call instructions.
32//
33// To get the precise counter information, These two passes need to invoke at
34// the same compilation point (so they see the same IR). For pass
35// PGOInstrumentationGen, the real work is done in instrumentOneFunc(). For
36// pass PGOInstrumentationUse, the real work in done in class PGOUseFunc and
37// the profile is opened in module level and passed to each PGOUseFunc instance.
38// The shared code for PGOInstrumentationGen and PGOInstrumentationUse is put
39// in class FuncPGOInstrumentation.
40//
41// Class PGOEdge represents a CFG edge and some auxiliary information. Class
42// BBInfo contains auxiliary information for each BB. These two classes are used
43// in pass PGOInstrumentationGen. Class PGOUseEdge and UseBBInfo are the derived
44// class of PGOEdge and BBInfo, respectively. They contains extra data structure
45// used in populating profile counters.
46// The MST implementation is in Class CFGMST (CFGMST.h).
47//
48//===----------------------------------------------------------------------===//
49
50#include "CFGMST.h"
51#include "llvm/ADT/APInt.h"
52#include "llvm/ADT/ArrayRef.h"
53#include "llvm/ADT/STLExtras.h"
54#include "llvm/ADT/SmallVector.h"
55#include "llvm/ADT/Statistic.h"
56#include "llvm/ADT/StringRef.h"
57#include "llvm/ADT/Triple.h"
58#include "llvm/ADT/Twine.h"
59#include "llvm/ADT/iterator.h"
60#include "llvm/ADT/iterator_range.h"
61#include "llvm/Analysis/BlockFrequencyInfo.h"
62#include "llvm/Analysis/BranchProbabilityInfo.h"
63#include "llvm/Analysis/CFG.h"
64#include "llvm/Analysis/IndirectCallVisitor.h"
65#include "llvm/Analysis/LoopInfo.h"
66#include "llvm/Analysis/OptimizationRemarkEmitter.h"
67#include "llvm/Analysis/ProfileSummaryInfo.h"
68#include "llvm/IR/Attributes.h"
69#include "llvm/IR/BasicBlock.h"
70#include "llvm/IR/CFG.h"
71#include "llvm/IR/CallSite.h"
72#include "llvm/IR/Comdat.h"
73#include "llvm/IR/Constant.h"
74#include "llvm/IR/Constants.h"
75#include "llvm/IR/DiagnosticInfo.h"
76#include "llvm/IR/Dominators.h"
77#include "llvm/IR/Function.h"
78#include "llvm/IR/GlobalAlias.h"
79#include "llvm/IR/GlobalValue.h"
80#include "llvm/IR/GlobalVariable.h"
81#include "llvm/IR/IRBuilder.h"
82#include "llvm/IR/InstVisitor.h"
83#include "llvm/IR/InstrTypes.h"
84#include "llvm/IR/Instruction.h"
85#include "llvm/IR/Instructions.h"
86#include "llvm/IR/IntrinsicInst.h"
87#include "llvm/IR/Intrinsics.h"
88#include "llvm/IR/LLVMContext.h"
89#include "llvm/IR/MDBuilder.h"
90#include "llvm/IR/Module.h"
91#include "llvm/IR/PassManager.h"
92#include "llvm/IR/ProfileSummary.h"
93#include "llvm/IR/Type.h"
94#include "llvm/IR/Value.h"
95#include "llvm/Pass.h"
96#include "llvm/ProfileData/InstrProf.h"
97#include "llvm/ProfileData/InstrProfReader.h"
98#include "llvm/Support/BranchProbability.h"
99#include "llvm/Support/Casting.h"
100#include "llvm/Support/CommandLine.h"
101#include "llvm/Support/DOTGraphTraits.h"
102#include "llvm/Support/Debug.h"
103#include "llvm/Support/Error.h"
104#include "llvm/Support/ErrorHandling.h"
105#include "llvm/Support/GraphWriter.h"
106#include "llvm/Support/JamCRC.h"
107#include "llvm/Support/raw_ostream.h"
108#include "llvm/Transforms/Instrumentation.h"
109#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
110#include "llvm/Transforms/Utils/BasicBlockUtils.h"
111#include <algorithm>
112#include <cassert>
113#include <cstdint>
114#include <memory>
115#include <numeric>
116#include <string>
117#include <unordered_map>
118#include <utility>
119#include <vector>
120
121using namespace llvm;
122using ProfileCount = Function::ProfileCount;
123
124#define DEBUG_TYPE"pgo-instrumentation" "pgo-instrumentation"
125
126STATISTIC(NumOfPGOInstrument, "Number of edges instrumented.")static llvm::Statistic NumOfPGOInstrument = {"pgo-instrumentation"
, "NumOfPGOInstrument", "Number of edges instrumented.", {0},
{false}};
127STATISTIC(NumOfPGOSelectInsts, "Number of select instruction instrumented.")static llvm::Statistic NumOfPGOSelectInsts = {"pgo-instrumentation"
, "NumOfPGOSelectInsts", "Number of select instruction instrumented."
, {0}, {false}};
128STATISTIC(NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented.")static llvm::Statistic NumOfPGOMemIntrinsics = {"pgo-instrumentation"
, "NumOfPGOMemIntrinsics", "Number of mem intrinsics instrumented."
, {0}, {false}};
129STATISTIC(NumOfPGOEdge, "Number of edges.")static llvm::Statistic NumOfPGOEdge = {"pgo-instrumentation",
"NumOfPGOEdge", "Number of edges.", {0}, {false}};
130STATISTIC(NumOfPGOBB, "Number of basic-blocks.")static llvm::Statistic NumOfPGOBB = {"pgo-instrumentation", "NumOfPGOBB"
, "Number of basic-blocks.", {0}, {false}};
131STATISTIC(NumOfPGOSplit, "Number of critical edge splits.")static llvm::Statistic NumOfPGOSplit = {"pgo-instrumentation"
, "NumOfPGOSplit", "Number of critical edge splits.", {0}, {false
}};
132STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts.")static llvm::Statistic NumOfPGOFunc = {"pgo-instrumentation",
"NumOfPGOFunc", "Number of functions having valid profile counts."
, {0}, {false}};
133STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile.")static llvm::Statistic NumOfPGOMismatch = {"pgo-instrumentation"
, "NumOfPGOMismatch", "Number of functions having mismatch profile."
, {0}, {false}};
134STATISTIC(NumOfPGOMissing, "Number of functions without profile.")static llvm::Statistic NumOfPGOMissing = {"pgo-instrumentation"
, "NumOfPGOMissing", "Number of functions without profile.", {
0}, {false}};
135STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations.")static llvm::Statistic NumOfPGOICall = {"pgo-instrumentation"
, "NumOfPGOICall", "Number of indirect call value instrumentations."
, {0}, {false}};
136STATISTIC(NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO.")static llvm::Statistic NumOfCSPGOInstrument = {"pgo-instrumentation"
, "NumOfCSPGOInstrument", "Number of edges instrumented in CSPGO."
, {0}, {false}};
137STATISTIC(NumOfCSPGOSelectInsts,static llvm::Statistic NumOfCSPGOSelectInsts = {"pgo-instrumentation"
, "NumOfCSPGOSelectInsts", "Number of select instruction instrumented in CSPGO."
, {0}, {false}}
138 "Number of select instruction instrumented in CSPGO.")static llvm::Statistic NumOfCSPGOSelectInsts = {"pgo-instrumentation"
, "NumOfCSPGOSelectInsts", "Number of select instruction instrumented in CSPGO."
, {0}, {false}};
139STATISTIC(NumOfCSPGOMemIntrinsics,static llvm::Statistic NumOfCSPGOMemIntrinsics = {"pgo-instrumentation"
, "NumOfCSPGOMemIntrinsics", "Number of mem intrinsics instrumented in CSPGO."
, {0}, {false}}
140 "Number of mem intrinsics instrumented in CSPGO.")static llvm::Statistic NumOfCSPGOMemIntrinsics = {"pgo-instrumentation"
, "NumOfCSPGOMemIntrinsics", "Number of mem intrinsics instrumented in CSPGO."
, {0}, {false}};
141STATISTIC(NumOfCSPGOEdge, "Number of edges in CSPGO.")static llvm::Statistic NumOfCSPGOEdge = {"pgo-instrumentation"
, "NumOfCSPGOEdge", "Number of edges in CSPGO.", {0}, {false}
};
142STATISTIC(NumOfCSPGOBB, "Number of basic-blocks in CSPGO.")static llvm::Statistic NumOfCSPGOBB = {"pgo-instrumentation",
"NumOfCSPGOBB", "Number of basic-blocks in CSPGO.", {0}, {false
}};
143STATISTIC(NumOfCSPGOSplit, "Number of critical edge splits in CSPGO.")static llvm::Statistic NumOfCSPGOSplit = {"pgo-instrumentation"
, "NumOfCSPGOSplit", "Number of critical edge splits in CSPGO."
, {0}, {false}};
144STATISTIC(NumOfCSPGOFunc,static llvm::Statistic NumOfCSPGOFunc = {"pgo-instrumentation"
, "NumOfCSPGOFunc", "Number of functions having valid profile counts in CSPGO."
, {0}, {false}}
145 "Number of functions having valid profile counts in CSPGO.")static llvm::Statistic NumOfCSPGOFunc = {"pgo-instrumentation"
, "NumOfCSPGOFunc", "Number of functions having valid profile counts in CSPGO."
, {0}, {false}};
146STATISTIC(NumOfCSPGOMismatch,static llvm::Statistic NumOfCSPGOMismatch = {"pgo-instrumentation"
, "NumOfCSPGOMismatch", "Number of functions having mismatch profile in CSPGO."
, {0}, {false}}
147 "Number of functions having mismatch profile in CSPGO.")static llvm::Statistic NumOfCSPGOMismatch = {"pgo-instrumentation"
, "NumOfCSPGOMismatch", "Number of functions having mismatch profile in CSPGO."
, {0}, {false}};
148STATISTIC(NumOfCSPGOMissing, "Number of functions without profile in CSPGO.")static llvm::Statistic NumOfCSPGOMissing = {"pgo-instrumentation"
, "NumOfCSPGOMissing", "Number of functions without profile in CSPGO."
, {0}, {false}};
149
150// Command line option to specify the file to read profile from. This is
151// mainly used for testing.
152static cl::opt<std::string>
153 PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden,
154 cl::value_desc("filename"),
155 cl::desc("Specify the path of profile data file. This is"
156 "mainly for test purpose."));
157static cl::opt<std::string> PGOTestProfileRemappingFile(
158 "pgo-test-profile-remapping-file", cl::init(""), cl::Hidden,
159 cl::value_desc("filename"),
160 cl::desc("Specify the path of profile remapping file. This is mainly for "
161 "test purpose."));
162
163// Command line option to disable value profiling. The default is false:
164// i.e. value profiling is enabled by default. This is for debug purpose.
165static cl::opt<bool> DisableValueProfiling("disable-vp", cl::init(false),
166 cl::Hidden,
167 cl::desc("Disable Value Profiling"));
168
169// Command line option to set the maximum number of VP annotations to write to
170// the metadata for a single indirect call callsite.
171static cl::opt<unsigned> MaxNumAnnotations(
172 "icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore,
173 cl::desc("Max number of annotations for a single indirect "
174 "call callsite"));
175
176// Command line option to set the maximum number of value annotations
177// to write to the metadata for a single memop intrinsic.
178static cl::opt<unsigned> MaxNumMemOPAnnotations(
179 "memop-max-annotations", cl::init(4), cl::Hidden, cl::ZeroOrMore,
180 cl::desc("Max number of preicise value annotations for a single memop"
181 "intrinsic"));
182
183// Command line option to control appending FunctionHash to the name of a COMDAT
184// function. This is to avoid the hash mismatch caused by the preinliner.
185static cl::opt<bool> DoComdatRenaming(
186 "do-comdat-renaming", cl::init(false), cl::Hidden,
187 cl::desc("Append function hash to the name of COMDAT function to avoid "
188 "function hash mismatch due to the preinliner"));
189
190// Command line option to enable/disable the warning about missing profile
191// information.
192static cl::opt<bool>
193 PGOWarnMissing("pgo-warn-missing-function", cl::init(false), cl::Hidden,
194 cl::desc("Use this option to turn on/off "
195 "warnings about missing profile data for "
196 "functions."));
197
198// Command line option to enable/disable the warning about a hash mismatch in
199// the profile data.
200static cl::opt<bool>
201 NoPGOWarnMismatch("no-pgo-warn-mismatch", cl::init(false), cl::Hidden,
202 cl::desc("Use this option to turn off/on "
203 "warnings about profile cfg mismatch."));
204
205// Command line option to enable/disable the warning about a hash mismatch in
206// the profile data for Comdat functions, which often turns out to be false
207// positive due to the pre-instrumentation inline.
208static cl::opt<bool>
209 NoPGOWarnMismatchComdat("no-pgo-warn-mismatch-comdat", cl::init(true),
210 cl::Hidden,
211 cl::desc("The option is used to turn on/off "
212 "warnings about hash mismatch for comdat "
213 "functions."));
214
215// Command line option to enable/disable select instruction instrumentation.
216static cl::opt<bool>
217 PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden,
218 cl::desc("Use this option to turn on/off SELECT "
219 "instruction instrumentation. "));
220
221// Command line option to turn on CFG dot or text dump of raw profile counts
222static cl::opt<PGOViewCountsType> PGOViewRawCounts(
223 "pgo-view-raw-counts", cl::Hidden,
224 cl::desc("A boolean option to show CFG dag or text "
225 "with raw profile counts from "
226 "profile data. See also option "
227 "-pgo-view-counts. To limit graph "
228 "display to only one function, use "
229 "filtering option -view-bfi-func-name."),
230 cl::values(clEnumValN(PGOVCT_None, "none", "do not show.")llvm::cl::OptionEnumValue { "none", int(PGOVCT_None), "do not show."
},
231 clEnumValN(PGOVCT_Graph, "graph", "show a graph.")llvm::cl::OptionEnumValue { "graph", int(PGOVCT_Graph), "show a graph."
},
232 clEnumValN(PGOVCT_Text, "text", "show in text.")llvm::cl::OptionEnumValue { "text", int(PGOVCT_Text), "show in text."
}));
233
234// Command line option to enable/disable memop intrinsic call.size profiling.
235static cl::opt<bool>
236 PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden,
237 cl::desc("Use this option to turn on/off "
238 "memory intrinsic size profiling."));
239
240// Emit branch probability as optimization remarks.
241static cl::opt<bool>
242 EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden,
243 cl::desc("When this option is on, the annotated "
244 "branch probability will be emitted as "
245 "optimization remarks: -{Rpass|"
246 "pass-remarks}=pgo-instrumentation"));
247
248// Command line option to turn on CFG dot dump after profile annotation.
249// Defined in Analysis/BlockFrequencyInfo.cpp: -pgo-view-counts
250extern cl::opt<PGOViewCountsType> PGOViewCounts;
251
252// Command line option to specify the name of the function for CFG dump
253// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
254extern cl::opt<std::string> ViewBlockFreqFuncName;
255
256// Return a string describing the branch condition that can be
257// used in static branch probability heuristics:
258static std::string getBranchCondString(Instruction *TI) {
259 BranchInst *BI = dyn_cast<BranchInst>(TI);
260 if (!BI || !BI->isConditional())
261 return std::string();
262
263 Value *Cond = BI->getCondition();
264 ICmpInst *CI = dyn_cast<ICmpInst>(Cond);
265 if (!CI)
266 return std::string();
267
268 std::string result;
269 raw_string_ostream OS(result);
270 OS << CmpInst::getPredicateName(CI->getPredicate()) << "_";
271 CI->getOperand(0)->getType()->print(OS, true);
272
273 Value *RHS = CI->getOperand(1);
274 ConstantInt *CV = dyn_cast<ConstantInt>(RHS);
275 if (CV) {
276 if (CV->isZero())
277 OS << "_Zero";
278 else if (CV->isOne())
279 OS << "_One";
280 else if (CV->isMinusOne())
281 OS << "_MinusOne";
282 else
283 OS << "_Const";
284 }
285 OS.flush();
286 return result;
287}
288
289namespace {
290
291/// The select instruction visitor plays three roles specified
292/// by the mode. In \c VM_counting mode, it simply counts the number of
293/// select instructions. In \c VM_instrument mode, it inserts code to count
294/// the number times TrueValue of select is taken. In \c VM_annotate mode,
295/// it reads the profile data and annotate the select instruction with metadata.
296enum VisitMode { VM_counting, VM_instrument, VM_annotate };
297class PGOUseFunc;
298
299/// Instruction Visitor class to visit select instructions.
300struct SelectInstVisitor : public InstVisitor<SelectInstVisitor> {
301 Function &F;
302 unsigned NSIs = 0; // Number of select instructions instrumented.
303 VisitMode Mode = VM_counting; // Visiting mode.
304 unsigned *CurCtrIdx = nullptr; // Pointer to current counter index.
305 unsigned TotalNumCtrs = 0; // Total number of counters
306 GlobalVariable *FuncNameVar = nullptr;
307 uint64_t FuncHash = 0;
308 PGOUseFunc *UseFunc = nullptr;
309
310 SelectInstVisitor(Function &Func) : F(Func) {}
311
312 void countSelects(Function &Func) {
313 NSIs = 0;
314 Mode = VM_counting;
315 visit(Func);
316 }
317
318 // Visit the IR stream and instrument all select instructions. \p
319 // Ind is a pointer to the counter index variable; \p TotalNC
320 // is the total number of counters; \p FNV is the pointer to the
321 // PGO function name var; \p FHash is the function hash.
322 void instrumentSelects(Function &Func, unsigned *Ind, unsigned TotalNC,
323 GlobalVariable *FNV, uint64_t FHash) {
324 Mode = VM_instrument;
325 CurCtrIdx = Ind;
326 TotalNumCtrs = TotalNC;
327 FuncHash = FHash;
328 FuncNameVar = FNV;
329 visit(Func);
330 }
331
332 // Visit the IR stream and annotate all select instructions.
333 void annotateSelects(Function &Func, PGOUseFunc *UF, unsigned *Ind) {
334 Mode = VM_annotate;
335 UseFunc = UF;
336 CurCtrIdx = Ind;
337 visit(Func);
338 }
339
340 void instrumentOneSelectInst(SelectInst &SI);
341 void annotateOneSelectInst(SelectInst &SI);
342
343 // Visit \p SI instruction and perform tasks according to visit mode.
344 void visitSelectInst(SelectInst &SI);
345
346 // Return the number of select instructions. This needs be called after
347 // countSelects().
348 unsigned getNumOfSelectInsts() const { return NSIs; }
349};
350
351/// Instruction Visitor class to visit memory intrinsic calls.
352struct MemIntrinsicVisitor : public InstVisitor<MemIntrinsicVisitor> {
353 Function &F;
354 unsigned NMemIs = 0; // Number of memIntrinsics instrumented.
355 VisitMode Mode = VM_counting; // Visiting mode.
356 unsigned CurCtrId = 0; // Current counter index.
357 unsigned TotalNumCtrs = 0; // Total number of counters
358 GlobalVariable *FuncNameVar = nullptr;
359 uint64_t FuncHash = 0;
360 PGOUseFunc *UseFunc = nullptr;
361 std::vector<Instruction *> Candidates;
362
363 MemIntrinsicVisitor(Function &Func) : F(Func) {}
364
365 void countMemIntrinsics(Function &Func) {
366 NMemIs = 0;
367 Mode = VM_counting;
368 visit(Func);
369 }
370
371 void instrumentMemIntrinsics(Function &Func, unsigned TotalNC,
372 GlobalVariable *FNV, uint64_t FHash) {
373 Mode = VM_instrument;
374 TotalNumCtrs = TotalNC;
375 FuncHash = FHash;
376 FuncNameVar = FNV;
377 visit(Func);
378 }
379
380 std::vector<Instruction *> findMemIntrinsics(Function &Func) {
381 Candidates.clear();
382 Mode = VM_annotate;
383 visit(Func);
384 return Candidates;
385 }
386
387 // Visit the IR stream and annotate all mem intrinsic call instructions.
388 void instrumentOneMemIntrinsic(MemIntrinsic &MI);
389
390 // Visit \p MI instruction and perform tasks according to visit mode.
391 void visitMemIntrinsic(MemIntrinsic &SI);
392
393 unsigned getNumOfMemIntrinsics() const { return NMemIs; }
394};
395
396class PGOInstrumentationGenLegacyPass : public ModulePass {
397public:
398 static char ID;
399
400 PGOInstrumentationGenLegacyPass(bool IsCS = false)
401 : ModulePass(ID), IsCS(IsCS) {
402 initializePGOInstrumentationGenLegacyPassPass(
403 *PassRegistry::getPassRegistry());
404 }
405
406 StringRef getPassName() const override { return "PGOInstrumentationGenPass"; }
407
408private:
409 // Is this is context-sensitive instrumentation.
410 bool IsCS;
411 bool runOnModule(Module &M) override;
412
413 void getAnalysisUsage(AnalysisUsage &AU) const override {
414 AU.addRequired<BlockFrequencyInfoWrapperPass>();
415 }
416};
417
418class PGOInstrumentationUseLegacyPass : public ModulePass {
419public:
420 static char ID;
421
422 // Provide the profile filename as the parameter.
423 PGOInstrumentationUseLegacyPass(std::string Filename = "", bool IsCS = false)
424 : ModulePass(ID), ProfileFileName(std::move(Filename)), IsCS(IsCS) {
425 if (!PGOTestProfileFile.empty())
426 ProfileFileName = PGOTestProfileFile;
427 initializePGOInstrumentationUseLegacyPassPass(
428 *PassRegistry::getPassRegistry());
429 }
430
431 StringRef getPassName() const override { return "PGOInstrumentationUsePass"; }
432
433private:
434 std::string ProfileFileName;
435 // Is this is context-sensitive instrumentation use.
436 bool IsCS;
437
438 bool runOnModule(Module &M) override;
439
440 void getAnalysisUsage(AnalysisUsage &AU) const override {
441 AU.addRequired<ProfileSummaryInfoWrapperPass>();
442 AU.addRequired<BlockFrequencyInfoWrapperPass>();
443 }
444};
445
446class PGOInstrumentationGenCreateVarLegacyPass : public ModulePass {
447public:
448 static char ID;
449 StringRef getPassName() const override {
450 return "PGOInstrumentationGenCreateVarPass";
451 }
452 PGOInstrumentationGenCreateVarLegacyPass(std::string CSInstrName = "")
453 : ModulePass(ID), InstrProfileOutput(CSInstrName) {
454 initializePGOInstrumentationGenCreateVarLegacyPassPass(
455 *PassRegistry::getPassRegistry());
456 }
457
458private:
459 bool runOnModule(Module &M) override {
460 createProfileFileNameVar(M, InstrProfileOutput);
461 createIRLevelProfileFlagVar(M, true);
462 return false;
463 }
464 std::string InstrProfileOutput;
465};
466
467} // end anonymous namespace
468
469char PGOInstrumentationGenLegacyPass::ID = 0;
470
471INITIALIZE_PASS_BEGIN(PGOInstrumentationGenLegacyPass, "pgo-instr-gen",static void *initializePGOInstrumentationGenLegacyPassPassOnce
(PassRegistry &Registry) {
472 "PGO instrumentation.", false, false)static void *initializePGOInstrumentationGenLegacyPassPassOnce
(PassRegistry &Registry) {
473INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)initializeBlockFrequencyInfoWrapperPassPass(Registry);
474INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)initializeBranchProbabilityInfoWrapperPassPass(Registry);
475INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass, "pgo-instr-gen",PassInfo *PI = new PassInfo( "PGO instrumentation.", "pgo-instr-gen"
, &PGOInstrumentationGenLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<PGOInstrumentationGenLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializePGOInstrumentationGenLegacyPassPassFlag
; void llvm::initializePGOInstrumentationGenLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializePGOInstrumentationGenLegacyPassPassFlag
, initializePGOInstrumentationGenLegacyPassPassOnce, std::ref
(Registry)); }
476 "PGO instrumentation.", false, false)PassInfo *PI = new PassInfo( "PGO instrumentation.", "pgo-instr-gen"
, &PGOInstrumentationGenLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<PGOInstrumentationGenLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializePGOInstrumentationGenLegacyPassPassFlag
; void llvm::initializePGOInstrumentationGenLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializePGOInstrumentationGenLegacyPassPassFlag
, initializePGOInstrumentationGenLegacyPassPassOnce, std::ref
(Registry)); }
477
478ModulePass *llvm::createPGOInstrumentationGenLegacyPass(bool IsCS) {
479 return new PGOInstrumentationGenLegacyPass(IsCS);
480}
481
482char PGOInstrumentationUseLegacyPass::ID = 0;
483
484INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use",static void *initializePGOInstrumentationUseLegacyPassPassOnce
(PassRegistry &Registry) {
485 "Read PGO instrumentation profile.", false, false)static void *initializePGOInstrumentationUseLegacyPassPassOnce
(PassRegistry &Registry) {
486INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)initializeBlockFrequencyInfoWrapperPassPass(Registry);
487INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)initializeBranchProbabilityInfoWrapperPassPass(Registry);
488INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)initializeProfileSummaryInfoWrapperPassPass(Registry);
489INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass, "pgo-instr-use",PassInfo *PI = new PassInfo( "Read PGO instrumentation profile."
, "pgo-instr-use", &PGOInstrumentationUseLegacyPass::ID, PassInfo
::NormalCtor_t(callDefaultCtor<PGOInstrumentationUseLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializePGOInstrumentationUseLegacyPassPassFlag
; void llvm::initializePGOInstrumentationUseLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializePGOInstrumentationUseLegacyPassPassFlag
, initializePGOInstrumentationUseLegacyPassPassOnce, std::ref
(Registry)); }
490 "Read PGO instrumentation profile.", false, false)PassInfo *PI = new PassInfo( "Read PGO instrumentation profile."
, "pgo-instr-use", &PGOInstrumentationUseLegacyPass::ID, PassInfo
::NormalCtor_t(callDefaultCtor<PGOInstrumentationUseLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializePGOInstrumentationUseLegacyPassPassFlag
; void llvm::initializePGOInstrumentationUseLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializePGOInstrumentationUseLegacyPassPassFlag
, initializePGOInstrumentationUseLegacyPassPassOnce, std::ref
(Registry)); }
491
492ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename,
493 bool IsCS) {
494 return new PGOInstrumentationUseLegacyPass(Filename.str(), IsCS);
495}
496
497char PGOInstrumentationGenCreateVarLegacyPass::ID = 0;
498
499INITIALIZE_PASS(PGOInstrumentationGenCreateVarLegacyPass,static void *initializePGOInstrumentationGenCreateVarLegacyPassPassOnce
(PassRegistry &Registry) { PassInfo *PI = new PassInfo( "Create PGO instrumentation version variable for CSPGO."
, "pgo-instr-gen-create-var", &PGOInstrumentationGenCreateVarLegacyPass
::ID, PassInfo::NormalCtor_t(callDefaultCtor<PGOInstrumentationGenCreateVarLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializePGOInstrumentationGenCreateVarLegacyPassPassFlag
; void llvm::initializePGOInstrumentationGenCreateVarLegacyPassPass
(PassRegistry &Registry) { llvm::call_once(InitializePGOInstrumentationGenCreateVarLegacyPassPassFlag
, initializePGOInstrumentationGenCreateVarLegacyPassPassOnce,
std::ref(Registry)); }
500 "pgo-instr-gen-create-var",static void *initializePGOInstrumentationGenCreateVarLegacyPassPassOnce
(PassRegistry &Registry) { PassInfo *PI = new PassInfo( "Create PGO instrumentation version variable for CSPGO."
, "pgo-instr-gen-create-var", &PGOInstrumentationGenCreateVarLegacyPass
::ID, PassInfo::NormalCtor_t(callDefaultCtor<PGOInstrumentationGenCreateVarLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializePGOInstrumentationGenCreateVarLegacyPassPassFlag
; void llvm::initializePGOInstrumentationGenCreateVarLegacyPassPass
(PassRegistry &Registry) { llvm::call_once(InitializePGOInstrumentationGenCreateVarLegacyPassPassFlag
, initializePGOInstrumentationGenCreateVarLegacyPassPassOnce,
std::ref(Registry)); }
501 "Create PGO instrumentation version variable for CSPGO.", false,static void *initializePGOInstrumentationGenCreateVarLegacyPassPassOnce
(PassRegistry &Registry) { PassInfo *PI = new PassInfo( "Create PGO instrumentation version variable for CSPGO."
, "pgo-instr-gen-create-var", &PGOInstrumentationGenCreateVarLegacyPass
::ID, PassInfo::NormalCtor_t(callDefaultCtor<PGOInstrumentationGenCreateVarLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializePGOInstrumentationGenCreateVarLegacyPassPassFlag
; void llvm::initializePGOInstrumentationGenCreateVarLegacyPassPass
(PassRegistry &Registry) { llvm::call_once(InitializePGOInstrumentationGenCreateVarLegacyPassPassFlag
, initializePGOInstrumentationGenCreateVarLegacyPassPassOnce,
std::ref(Registry)); }
502 false)static void *initializePGOInstrumentationGenCreateVarLegacyPassPassOnce
(PassRegistry &Registry) { PassInfo *PI = new PassInfo( "Create PGO instrumentation version variable for CSPGO."
, "pgo-instr-gen-create-var", &PGOInstrumentationGenCreateVarLegacyPass
::ID, PassInfo::NormalCtor_t(callDefaultCtor<PGOInstrumentationGenCreateVarLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializePGOInstrumentationGenCreateVarLegacyPassPassFlag
; void llvm::initializePGOInstrumentationGenCreateVarLegacyPassPass
(PassRegistry &Registry) { llvm::call_once(InitializePGOInstrumentationGenCreateVarLegacyPassPassFlag
, initializePGOInstrumentationGenCreateVarLegacyPassPassOnce,
std::ref(Registry)); }
503
504ModulePass *
505llvm::createPGOInstrumentationGenCreateVarLegacyPass(StringRef CSInstrName) {
506 return new PGOInstrumentationGenCreateVarLegacyPass(CSInstrName);
507}
508
509namespace {
510
511/// An MST based instrumentation for PGO
512///
513/// Implements a Minimum Spanning Tree (MST) based instrumentation for PGO
514/// in the function level.
515struct PGOEdge {
516 // This class implements the CFG edges. Note the CFG can be a multi-graph.
517 // So there might be multiple edges with same SrcBB and DestBB.
518 const BasicBlock *SrcBB;
519 const BasicBlock *DestBB;
520 uint64_t Weight;
521 bool InMST = false;
522 bool Removed = false;
523 bool IsCritical = false;
524
525 PGOEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
526 : SrcBB(Src), DestBB(Dest), Weight(W) {}
527
528 // Return the information string of an edge.
529 const std::string infoString() const {
530 return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +
531 (IsCritical ? "c" : " ") + " W=" + Twine(Weight)).str();
532 }
533};
534
535// This class stores the auxiliary information for each BB.
536struct BBInfo {
537 BBInfo *Group;
538 uint32_t Index;
539 uint32_t Rank = 0;
540
541 BBInfo(unsigned IX) : Group(this), Index(IX) {}
542
543 // Return the information string of this object.
544 const std::string infoString() const {
545 return (Twine("Index=") + Twine(Index)).str();
546 }
547};
548
549// This class implements the CFG edges. Note the CFG can be a multi-graph.
550template <class Edge, class BBInfo> class FuncPGOInstrumentation {
551private:
552 Function &F;
553
554 // Is this is context-sensitive instrumentation.
555 bool IsCS;
556
557 // A map that stores the Comdat group in function F.
558 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
559
560 void computeCFGHash();
561 void renameComdatFunction();
562
563public:
564 std::vector<std::vector<Instruction *>> ValueSites;
565 SelectInstVisitor SIVisitor;
566 MemIntrinsicVisitor MIVisitor;
567 std::string FuncName;
568 GlobalVariable *FuncNameVar;
569
570 // CFG hash value for this function.
571 uint64_t FunctionHash = 0;
572
573 // The Minimum Spanning Tree of function CFG.
574 CFGMST<Edge, BBInfo> MST;
575
576 // Collect all the BBs that will be instrumented, and store them in
577 // InstrumentBBs.
578 void getInstrumentBBs(std::vector<BasicBlock *> &InstrumentBBs);
579
580 // Give an edge, find the BB that will be instrumented.
581 // Return nullptr if there is no BB to be instrumented.
582 BasicBlock *getInstrBB(Edge *E);
583
584 // Return the auxiliary BB information.
585 BBInfo &getBBInfo(const BasicBlock *BB) const { return MST.getBBInfo(BB); }
586
587 // Return the auxiliary BB information if available.
588 BBInfo *findBBInfo(const BasicBlock *BB) const { return MST.findBBInfo(BB); }
589
590 // Dump edges and BB information.
591 void dumpInfo(std::string Str = "") const {
592 MST.dumpEdges(dbgs(), Twine("Dump Function ") + FuncName + " Hash: " +
593 Twine(FunctionHash) + "\t" + Str);
594 }
595
596 FuncPGOInstrumentation(
597 Function &Func,
598 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
599 bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,
600 BlockFrequencyInfo *BFI = nullptr, bool IsCS = false)
601 : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers),
602 ValueSites(IPVK_Last + 1), SIVisitor(Func), MIVisitor(Func),
603 MST(F, BPI, BFI) {
604 // This should be done before CFG hash computation.
605 SIVisitor.countSelects(Func);
606 MIVisitor.countMemIntrinsics(Func);
607 if (!IsCS) {
608 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
609 NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
610 NumOfPGOBB += MST.BBInfos.size();
611 ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func);
612 } else {
613 NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
614 NumOfCSPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
615 NumOfCSPGOBB += MST.BBInfos.size();
616 }
617 ValueSites[IPVK_MemOPSize] = MIVisitor.findMemIntrinsics(Func);
618
619 FuncName = getPGOFuncName(F);
620 computeCFGHash();
621 if (!ComdatMembers.empty())
622 renameComdatFunction();
623 LLVM_DEBUG(dumpInfo("after CFGMST"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dumpInfo("after CFGMST"); } } while
(false);
624
625 for (auto &E : MST.AllEdges) {
626 if (E->Removed)
627 continue;
628 IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++;
629 if (!E->InMST)
630 IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++;
631 }
632
633 if (CreateGlobalVar)
634 FuncNameVar = createPGOFuncNameVar(F, FuncName);
635 }
636};
637
638} // end anonymous namespace
639
640// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
641// value of each BB in the CFG. The higher 32 bits record the number of edges.
642template <class Edge, class BBInfo>
643void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
644 std::vector<char> Indexes;
645 JamCRC JC;
646 for (auto &BB : F) {
647 const Instruction *TI = BB.getTerminator();
648 for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) {
649 BasicBlock *Succ = TI->getSuccessor(I);
650 auto BI = findBBInfo(Succ);
651 if (BI == nullptr)
652 continue;
653 uint32_t Index = BI->Index;
654 for (int J = 0; J < 4; J++)
655 Indexes.push_back((char)(Index >> (J * 8)));
656 }
657 }
658 JC.update(Indexes);
659
660 // Hash format for context sensitive profile. Reserve 4 bits for other
661 // information.
662 FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
663 (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
664 //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
665 (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
666 // Reserve bit 60-63 for other information purpose.
667 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
668 if (IsCS)
669 NamedInstrProfRecord::setCSFlagInHash(FunctionHash);
670 LLVM_DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
671 << " CRC = " << JC.getCRC()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
672 << ", Selects = " << SIVisitor.getNumOfSelectInsts()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
673 << ", Edges = " << MST.AllEdges.size() << ", ICSites = "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
674 << ValueSites[IPVK_IndirectCallTarget].size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
675 << ", Hash = " << FunctionHash << "\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false);
676}
677
678// Check if we can safely rename this Comdat function.
679static bool canRenameComdat(
680 Function &F,
681 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
682 if (!DoComdatRenaming || !canRenameComdatFunc(F, true))
683 return false;
684
685 // FIXME: Current only handle those Comdat groups that only containing one
686 // function and function aliases.
687 // (1) For a Comdat group containing multiple functions, we need to have a
688 // unique postfix based on the hashes for each function. There is a
689 // non-trivial code refactoring to do this efficiently.
690 // (2) Variables can not be renamed, so we can not rename Comdat function in a
691 // group including global vars.
692 Comdat *C = F.getComdat();
693 for (auto &&CM : make_range(ComdatMembers.equal_range(C))) {
694 if (dyn_cast<GlobalAlias>(CM.second))
695 continue;
696 Function *FM = dyn_cast<Function>(CM.second);
697 if (FM != &F)
698 return false;
699 }
700 return true;
701}
702
703// Append the CFGHash to the Comdat function name.
704template <class Edge, class BBInfo>
705void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
706 if (!canRenameComdat(F, ComdatMembers))
707 return;
708 std::string OrigName = F.getName().str();
709 std::string NewFuncName =
710 Twine(F.getName() + "." + Twine(FunctionHash)).str();
711 F.setName(Twine(NewFuncName));
712 GlobalAlias::create(GlobalValue::WeakAnyLinkage, OrigName, &F);
713 FuncName = Twine(FuncName + "." + Twine(FunctionHash)).str();
714 Comdat *NewComdat;
715 Module *M = F.getParent();
716 // For AvailableExternallyLinkage functions, change the linkage to
717 // LinkOnceODR and put them into comdat. This is because after renaming, there
718 // is no backup external copy available for the function.
719 if (!F.hasComdat()) {
720 assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage)((F.getLinkage() == GlobalValue::AvailableExternallyLinkage) ?
static_cast<void> (0) : __assert_fail ("F.getLinkage() == GlobalValue::AvailableExternallyLinkage"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 720, __PRETTY_FUNCTION__));
721 NewComdat = M->getOrInsertComdat(StringRef(NewFuncName));
722 F.setLinkage(GlobalValue::LinkOnceODRLinkage);
723 F.setComdat(NewComdat);
724 return;
725 }
726
727 // This function belongs to a single function Comdat group.
728 Comdat *OrigComdat = F.getComdat();
729 std::string NewComdatName =
730 Twine(OrigComdat->getName() + "." + Twine(FunctionHash)).str();
731 NewComdat = M->getOrInsertComdat(StringRef(NewComdatName));
732 NewComdat->setSelectionKind(OrigComdat->getSelectionKind());
733
734 for (auto &&CM : make_range(ComdatMembers.equal_range(OrigComdat))) {
735 if (GlobalAlias *GA = dyn_cast<GlobalAlias>(CM.second)) {
736 // For aliases, change the name directly.
737 assert(dyn_cast<Function>(GA->getAliasee()->stripPointerCasts()) == &F)((dyn_cast<Function>(GA->getAliasee()->stripPointerCasts
()) == &F) ? static_cast<void> (0) : __assert_fail (
"dyn_cast<Function>(GA->getAliasee()->stripPointerCasts()) == &F"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 737, __PRETTY_FUNCTION__));
738 std::string OrigGAName = GA->getName().str();
739 GA->setName(Twine(GA->getName() + "." + Twine(FunctionHash)));
740 GlobalAlias::create(GlobalValue::WeakAnyLinkage, OrigGAName, GA);
741 continue;
742 }
743 // Must be a function.
744 Function *CF = dyn_cast<Function>(CM.second);
745 assert(CF)((CF) ? static_cast<void> (0) : __assert_fail ("CF", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 745, __PRETTY_FUNCTION__));
746 CF->setComdat(NewComdat);
747 }
748}
749
750// Collect all the BBs that will be instruments and return them in
751// InstrumentBBs.
752template <class Edge, class BBInfo>
753void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(
754 std::vector<BasicBlock *> &InstrumentBBs) {
755 // Use a worklist as we will update the vector during the iteration.
756 std::vector<Edge *> EdgeList;
757 EdgeList.reserve(MST.AllEdges.size());
758 for (auto &E : MST.AllEdges)
759 EdgeList.push_back(E.get());
760
761 for (auto &E : EdgeList) {
762 BasicBlock *InstrBB = getInstrBB(E);
763 if (InstrBB)
764 InstrumentBBs.push_back(InstrBB);
765 }
766}
767
768// Given a CFG E to be instrumented, find which BB to place the instrumented
769// code. The function will split the critical edge if necessary.
770template <class Edge, class BBInfo>
771BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
772 if (E->InMST || E->Removed)
773 return nullptr;
774
775 BasicBlock *SrcBB = const_cast<BasicBlock *>(E->SrcBB);
776 BasicBlock *DestBB = const_cast<BasicBlock *>(E->DestBB);
777 // For a fake edge, instrument the real BB.
778 if (SrcBB == nullptr)
779 return DestBB;
780 if (DestBB == nullptr)
781 return SrcBB;
782
783 // Instrument the SrcBB if it has a single successor,
784 // otherwise, the DestBB if this is not a critical edge.
785 Instruction *TI = SrcBB->getTerminator();
786 if (TI->getNumSuccessors() <= 1)
787 return SrcBB;
788 if (!E->IsCritical)
789 return DestBB;
790
791 // For a critical edge, we have to split. Instrument the newly
792 // created BB.
793 IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;
794 LLVM_DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Indexdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Split critical edge: "
<< getBBInfo(SrcBB).Index << " --> " <<
getBBInfo(DestBB).Index << "\n"; } } while (false)
795 << " --> " << getBBInfo(DestBB).Index << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Split critical edge: "
<< getBBInfo(SrcBB).Index << " --> " <<
getBBInfo(DestBB).Index << "\n"; } } while (false);
796 unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
797 BasicBlock *InstrBB = SplitCriticalEdge(TI, SuccNum);
798 if (!InstrBB) {
799 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Fail to split critical edge: not instrument this edge.\n"
; } } while (false)
800 dbgs() << "Fail to split critical edge: not instrument this edge.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Fail to split critical edge: not instrument this edge.\n"
; } } while (false);
801 return nullptr;
802 }
803 // Need to add two new edges. First one: Add new edge of SrcBB->InstrBB.
804 MST.addEdge(SrcBB, InstrBB, 0);
805 // Second one: Add new edge of InstrBB->DestBB.
806 Edge &NewEdge1 = MST.addEdge(InstrBB, DestBB, 0);
807 NewEdge1.InMST = true;
808 E->Removed = true;
809
810 return InstrBB;
811}
812
813// Visit all edge and instrument the edges not in MST, and do value profiling.
814// Critical edges will be split.
815static void instrumentOneFunc(
816 Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI,
817 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
818 bool IsCS) {
819 // Split indirectbr critical edges here before computing the MST rather than
820 // later in getInstrBB() to avoid invalidating it.
821 SplitIndirectBrCriticalEdges(F, BPI, BFI);
822
823 FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, ComdatMembers, true, BPI,
824 BFI, IsCS);
825 std::vector<BasicBlock *> InstrumentBBs;
826 FuncInfo.getInstrumentBBs(InstrumentBBs);
827 unsigned NumCounters =
828 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
829
830 uint32_t I = 0;
831 Type *I8PtrTy = Type::getInt8PtrTy(M->getContext());
832 for (auto *InstrBB : InstrumentBBs) {
833 IRBuilder<> Builder(InstrBB, InstrBB->getFirstInsertionPt());
834 assert(Builder.GetInsertPoint() != InstrBB->end() &&((Builder.GetInsertPoint() != InstrBB->end() && "Cannot get the Instrumentation point"
) ? static_cast<void> (0) : __assert_fail ("Builder.GetInsertPoint() != InstrBB->end() && \"Cannot get the Instrumentation point\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 835, __PRETTY_FUNCTION__))
835 "Cannot get the Instrumentation point")((Builder.GetInsertPoint() != InstrBB->end() && "Cannot get the Instrumentation point"
) ? static_cast<void> (0) : __assert_fail ("Builder.GetInsertPoint() != InstrBB->end() && \"Cannot get the Instrumentation point\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 835, __PRETTY_FUNCTION__));
836 Builder.CreateCall(
837 Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment),
838 {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
839 Builder.getInt64(FuncInfo.FunctionHash), Builder.getInt32(NumCounters),
840 Builder.getInt32(I++)});
841 }
842
843 // Now instrument select instructions:
844 FuncInfo.SIVisitor.instrumentSelects(F, &I, NumCounters, FuncInfo.FuncNameVar,
845 FuncInfo.FunctionHash);
846 assert(I == NumCounters)((I == NumCounters) ? static_cast<void> (0) : __assert_fail
("I == NumCounters", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 846, __PRETTY_FUNCTION__));
847
848 if (DisableValueProfiling)
849 return;
850
851 unsigned NumIndirectCalls = 0;
852 for (auto &I : FuncInfo.ValueSites[IPVK_IndirectCallTarget]) {
853 CallSite CS(I);
854 Value *Callee = CS.getCalledValue();
855 LLVM_DEBUG(dbgs() << "Instrument one indirect call: CallSite Index = "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Instrument one indirect call: CallSite Index = "
<< NumIndirectCalls << "\n"; } } while (false)
856 << NumIndirectCalls << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Instrument one indirect call: CallSite Index = "
<< NumIndirectCalls << "\n"; } } while (false);
857 IRBuilder<> Builder(I);
858 assert(Builder.GetInsertPoint() != I->getParent()->end() &&((Builder.GetInsertPoint() != I->getParent()->end() &&
"Cannot get the Instrumentation point") ? static_cast<void
> (0) : __assert_fail ("Builder.GetInsertPoint() != I->getParent()->end() && \"Cannot get the Instrumentation point\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 859, __PRETTY_FUNCTION__))
859 "Cannot get the Instrumentation point")((Builder.GetInsertPoint() != I->getParent()->end() &&
"Cannot get the Instrumentation point") ? static_cast<void
> (0) : __assert_fail ("Builder.GetInsertPoint() != I->getParent()->end() && \"Cannot get the Instrumentation point\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 859, __PRETTY_FUNCTION__));
860 Builder.CreateCall(
861 Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
862 {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
863 Builder.getInt64(FuncInfo.FunctionHash),
864 Builder.CreatePtrToInt(Callee, Builder.getInt64Ty()),
865 Builder.getInt32(IPVK_IndirectCallTarget),
866 Builder.getInt32(NumIndirectCalls++)});
867 }
868 NumOfPGOICall += NumIndirectCalls;
869
870 // Now instrument memop intrinsic calls.
871 FuncInfo.MIVisitor.instrumentMemIntrinsics(
872 F, NumCounters, FuncInfo.FuncNameVar, FuncInfo.FunctionHash);
873}
874
875namespace {
876
877// This class represents a CFG edge in profile use compilation.
878struct PGOUseEdge : public PGOEdge {
879 bool CountValid = false;
880 uint64_t CountValue = 0;
881
882 PGOUseEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
883 : PGOEdge(Src, Dest, W) {}
884
885 // Set edge count value
886 void setEdgeCount(uint64_t Value) {
887 CountValue = Value;
888 CountValid = true;
889 }
890
891 // Return the information string for this object.
892 const std::string infoString() const {
893 if (!CountValid)
894 return PGOEdge::infoString();
895 return (Twine(PGOEdge::infoString()) + " Count=" + Twine(CountValue))
896 .str();
897 }
898};
899
900using DirectEdges = SmallVector<PGOUseEdge *, 2>;
901
902// This class stores the auxiliary information for each BB.
903struct UseBBInfo : public BBInfo {
904 uint64_t CountValue = 0;
905 bool CountValid;
906 int32_t UnknownCountInEdge = 0;
907 int32_t UnknownCountOutEdge = 0;
908 DirectEdges InEdges;
909 DirectEdges OutEdges;
910
911 UseBBInfo(unsigned IX) : BBInfo(IX), CountValid(false) {}
912
913 UseBBInfo(unsigned IX, uint64_t C)
914 : BBInfo(IX), CountValue(C), CountValid(true) {}
915
916 // Set the profile count value for this BB.
917 void setBBInfoCount(uint64_t Value) {
918 CountValue = Value;
919 CountValid = true;
920 }
921
922 // Return the information string of this object.
923 const std::string infoString() const {
924 if (!CountValid)
925 return BBInfo::infoString();
926 return (Twine(BBInfo::infoString()) + " Count=" + Twine(CountValue)).str();
927 }
928};
929
930} // end anonymous namespace
931
932// Sum up the count values for all the edges.
933static uint64_t sumEdgeCount(const ArrayRef<PGOUseEdge *> Edges) {
934 uint64_t Total = 0;
935 for (auto &E : Edges) {
936 if (E->Removed)
937 continue;
938 Total += E->CountValue;
939 }
940 return Total;
941}
942
943namespace {
944
945class PGOUseFunc {
946public:
947 PGOUseFunc(Function &Func, Module *Modu,
948 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
949 BranchProbabilityInfo *BPI = nullptr,
950 BlockFrequencyInfo *BFIin = nullptr, bool IsCS = false)
951 : F(Func), M(Modu), BFI(BFIin),
952 FuncInfo(Func, ComdatMembers, false, BPI, BFIin, IsCS),
953 FreqAttr(FFA_Normal), IsCS(IsCS) {}
954
955 // Read counts for the instrumented BB from profile.
956 bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros);
957
958 // Populate the counts for all BBs.
959 void populateCounters();
960
961 // Set the branch weights based on the count values.
962 void setBranchWeights();
963
964 // Annotate the value profile call sites for all value kind.
965 void annotateValueSites();
966
967 // Annotate the value profile call sites for one value kind.
968 void annotateValueSites(uint32_t Kind);
969
970 // Annotate the irreducible loop header weights.
971 void annotateIrrLoopHeaderWeights();
972
973 // The hotness of the function from the profile count.
974 enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };
975
976 // Return the function hotness from the profile.
977 FuncFreqAttr getFuncFreqAttr() const { return FreqAttr; }
978
979 // Return the function hash.
980 uint64_t getFuncHash() const { return FuncInfo.FunctionHash; }
981
982 // Return the profile record for this function;
983 InstrProfRecord &getProfileRecord() { return ProfileRecord; }
984
985 // Return the auxiliary BB information.
986 UseBBInfo &getBBInfo(const BasicBlock *BB) const {
987 return FuncInfo.getBBInfo(BB);
988 }
989
990 // Return the auxiliary BB information if available.
991 UseBBInfo *findBBInfo(const BasicBlock *BB) const {
992 return FuncInfo.findBBInfo(BB);
993 }
994
995 Function &getFunc() const { return F; }
996
997 void dumpInfo(std::string Str = "") const {
998 FuncInfo.dumpInfo(Str);
999 }
1000
1001 uint64_t getProgramMaxCount() const { return ProgramMaxCount; }
1002private:
1003 Function &F;
1004 Module *M;
1005 BlockFrequencyInfo *BFI;
1006
1007 // This member stores the shared information with class PGOGenFunc.
1008 FuncPGOInstrumentation<PGOUseEdge, UseBBInfo> FuncInfo;
1009
1010 // The maximum count value in the profile. This is only used in PGO use
1011 // compilation.
1012 uint64_t ProgramMaxCount;
1013
1014 // Position of counter that remains to be read.
1015 uint32_t CountPosition = 0;
1016
1017 // Total size of the profile count for this function.
1018 uint32_t ProfileCountSize = 0;
1019
1020 // ProfileRecord for this function.
1021 InstrProfRecord ProfileRecord;
1022
1023 // Function hotness info derived from profile.
1024 FuncFreqAttr FreqAttr;
1025
1026 // Is to use the context sensitive profile.
1027 bool IsCS;
1028
1029 // Find the Instrumented BB and set the value. Return false on error.
1030 bool setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
1031
1032 // Set the edge counter value for the unknown edge -- there should be only
1033 // one unknown edge.
1034 void setEdgeCount(DirectEdges &Edges, uint64_t Value);
1035
1036 // Return FuncName string;
1037 const std::string getFuncName() const { return FuncInfo.FuncName; }
1038
1039 // Set the hot/cold inline hints based on the count values.
1040 // FIXME: This function should be removed once the functionality in
1041 // the inliner is implemented.
1042 void markFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) {
1043 if (ProgramMaxCount == 0)
1044 return;
1045 // Threshold of the hot functions.
1046 const BranchProbability HotFunctionThreshold(1, 100);
1047 // Threshold of the cold functions.
1048 const BranchProbability ColdFunctionThreshold(2, 10000);
1049 if (EntryCount >= HotFunctionThreshold.scale(ProgramMaxCount))
1050 FreqAttr = FFA_Hot;
1051 else if (MaxCount <= ColdFunctionThreshold.scale(ProgramMaxCount))
1052 FreqAttr = FFA_Cold;
1053 }
1054};
1055
1056} // end anonymous namespace
1057
1058// Visit all the edges and assign the count value for the instrumented
1059// edges and the BB. Return false on error.
1060bool PGOUseFunc::setInstrumentedCounts(
1061 const std::vector<uint64_t> &CountFromProfile) {
1062
1063 std::vector<BasicBlock *> InstrumentBBs;
1064 FuncInfo.getInstrumentBBs(InstrumentBBs);
1065 unsigned NumCounters =
1066 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
1067 // The number of counters here should match the number of counters
1068 // in profile. Return if they mismatch.
1069 if (NumCounters != CountFromProfile.size()) {
1070 return false;
1071 }
1072 uint32_t I = 0;
1073 for (BasicBlock *InstrBB : InstrumentBBs) {
1074 uint64_t CountValue = CountFromProfile[I++];
1075 UseBBInfo &Info = getBBInfo(InstrBB);
1076 Info.setBBInfoCount(CountValue);
1077 // If only one in-edge, the edge profile count should be the same as BB
1078 // profile count.
1079 if (Info.InEdges.size() == 1) {
1080 Info.InEdges[0]->setEdgeCount(CountValue);
1081 }
1082 // If only one out-edge, the edge profile count should be the same as BB
1083 // profile count.
1084 if (Info.OutEdges.size() == 1) {
1085 Info.OutEdges[0]->setEdgeCount(CountValue);
1086 }
1087 }
1088 ProfileCountSize = CountFromProfile.size();
1089 CountPosition = I;
1090 return true;
1091}
1092
1093// Set the count value for the unknown edge. There should be one and only one
1094// unknown edge in Edges vector.
1095void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) {
1096 for (auto &E : Edges) {
1097 if (E->CountValid)
1098 continue;
1099 E->setEdgeCount(Value);
1100
1101 getBBInfo(E->SrcBB).UnknownCountOutEdge--;
1102 getBBInfo(E->DestBB).UnknownCountInEdge--;
1103 return;
1104 }
1105 llvm_unreachable("Cannot find the unknown count edge")::llvm::llvm_unreachable_internal("Cannot find the unknown count edge"
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1105);
1106}
1107
1108// Read the profile from ProfileFileName and assign the value to the
1109// instrumented BB and the edges. This function also updates ProgramMaxCount.
1110// Return true if the profile are successfully read, and false on errors.
1111bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros) {
1112 auto &Ctx = M->getContext();
1113 Expected<InstrProfRecord> Result =
1114 PGOReader->getInstrProfRecord(FuncInfo.FuncName, FuncInfo.FunctionHash);
1115 if (Error E = Result.takeError()) {
14
←
Assuming the condition is true
→
15
←
Taking true branch
→
1116 handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
16
←
Calling 'handleAllErrors<(lambda at /build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp:1116:35)>'
→
1117 auto Err = IPE.get();
1118 bool SkipWarning = false;
1119 LLVM_DEBUG(dbgs() << "Error in reading profile for Func "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Error in reading profile for Func "
<< FuncInfo.FuncName << ": "; } } while (false)
1120 << FuncInfo.FuncName << ": ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Error in reading profile for Func "
<< FuncInfo.FuncName << ": "; } } while (false);
1121 if (Err == instrprof_error::unknown_function) {
1122 IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++;
1123 SkipWarning = !PGOWarnMissing;
1124 LLVM_DEBUG(dbgs() << "unknown function")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "unknown function"
; } } while (false);
1125 } else if (Err == instrprof_error::hash_mismatch ||
1126 Err == instrprof_error::malformed) {
1127 IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++;
1128 SkipWarning =
1129 NoPGOWarnMismatch ||
1130 (NoPGOWarnMismatchComdat &&
1131 (F.hasComdat() ||
1132 F.getLinkage() == GlobalValue::AvailableExternallyLinkage));
1133 LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "hash mismatch (skip="
<< SkipWarning << ")"; } } while (false);
1134 }
1135
1136 LLVM_DEBUG(dbgs() << " IsCS=" << IsCS << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << " IsCS=" << IsCS
<< "\n"; } } while (false);
1137 if (SkipWarning)
1138 return;
1139
1140 std::string Msg = IPE.message() + std::string(" ") + F.getName().str() +
1141 std::string(" Hash = ") +
1142 std::to_string(FuncInfo.FunctionHash);
1143
1144 Ctx.diagnose(
1145 DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
1146 });
1147 return false;
1148 }
1149 ProfileRecord = std::move(Result.get());
1150 std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts;
1151
1152 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
1153 LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << CountFromProfile.size
() << " counts\n"; } } while (false);
1154 uint64_t ValueSum = 0;
1155 for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {
1156 LLVM_DEBUG(dbgs() << " " << I << ": " << CountFromProfile[I] << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << " " << I <<
": " << CountFromProfile[I] << "\n"; } } while (
false);
1157 ValueSum += CountFromProfile[I];
1158 }
1159 AllZeros = (ValueSum == 0);
1160
1161 LLVM_DEBUG(dbgs() << "SUM = " << ValueSum << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "SUM = " <<
ValueSum << "\n"; } } while (false);
1162
1163 getBBInfo(nullptr).UnknownCountOutEdge = 2;
1164 getBBInfo(nullptr).UnknownCountInEdge = 2;
1165
1166 if (!setInstrumentedCounts(CountFromProfile)) {
1167 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Inconsistent number of counts, skipping this function"
; } } while (false)
1168 dbgs() << "Inconsistent number of counts, skipping this function")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Inconsistent number of counts, skipping this function"
; } } while (false);
1169 Ctx.diagnose(DiagnosticInfoPGOProfile(
1170 M->getName().data(),
1171 Twine("Inconsistent number of counts in ") + F.getName().str()
1172 + Twine(": the profile may be stale or there is a function name collision."),
1173 DS_Warning));
1174 return false;
1175 }
1176 ProgramMaxCount = PGOReader->getMaximumFunctionCount(IsCS);
1177 return true;
1178}
1179
1180// Populate the counters from instrumented BBs to all BBs.
1181// In the end of this operation, all BBs should have a valid count value.
1182void PGOUseFunc::populateCounters() {
1183 // First set up Count variable for all BBs.
1184 for (auto &E : FuncInfo.MST.AllEdges) {
1185 if (E->Removed)
1186 continue;
1187
1188 const BasicBlock *SrcBB = E->SrcBB;
1189 const BasicBlock *DestBB = E->DestBB;
1190 UseBBInfo &SrcInfo = getBBInfo(SrcBB);
1191 UseBBInfo &DestInfo = getBBInfo(DestBB);
1192 SrcInfo.OutEdges.push_back(E.get());
1193 DestInfo.InEdges.push_back(E.get());
1194 SrcInfo.UnknownCountOutEdge++;
1195 DestInfo.UnknownCountInEdge++;
1196
1197 if (!E->CountValid)
1198 continue;
1199 DestInfo.UnknownCountInEdge--;
1200 SrcInfo.UnknownCountOutEdge--;
1201 }
1202
1203 bool Changes = true;
1204 unsigned NumPasses = 0;
1205 while (Changes) {
1206 NumPasses++;
1207 Changes = false;
1208
1209 // For efficient traversal, it's better to start from the end as most
1210 // of the instrumented edges are at the end.
1211 for (auto &BB : reverse(F)) {
1212 UseBBInfo *Count = findBBInfo(&BB);
1213 if (Count == nullptr)
1214 continue;
1215 if (!Count->CountValid) {
1216 if (Count->UnknownCountOutEdge == 0) {
1217 Count->CountValue = sumEdgeCount(Count->OutEdges);
1218 Count->CountValid = true;
1219 Changes = true;
1220 } else if (Count->UnknownCountInEdge == 0) {
1221 Count->CountValue = sumEdgeCount(Count->InEdges);
1222 Count->CountValid = true;
1223 Changes = true;
1224 }
1225 }
1226 if (Count->CountValid) {
1227 if (Count->UnknownCountOutEdge == 1) {
1228 uint64_t Total = 0;
1229 uint64_t OutSum = sumEdgeCount(Count->OutEdges);
1230 // If the one of the successor block can early terminate (no-return),
1231 // we can end up with situation where out edge sum count is larger as
1232 // the source BB's count is collected by a post-dominated block.
1233 if (Count->CountValue > OutSum)
1234 Total = Count->CountValue - OutSum;
1235 setEdgeCount(Count->OutEdges, Total);
1236 Changes = true;
1237 }
1238 if (Count->UnknownCountInEdge == 1) {
1239 uint64_t Total = 0;
1240 uint64_t InSum = sumEdgeCount(Count->InEdges);
1241 if (Count->CountValue > InSum)
1242 Total = Count->CountValue - InSum;
1243 setEdgeCount(Count->InEdges, Total);
1244 Changes = true;
1245 }
1246 }
1247 }
1248 }
1249
1250 LLVM_DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Populate counts in "
<< NumPasses << " passes.\n"; } } while (false);
1251#ifndef NDEBUG
1252 // Assert every BB has a valid counter.
1253 for (auto &BB : F) {
1254 auto BI = findBBInfo(&BB);
1255 if (BI == nullptr)
1256 continue;
1257 assert(BI->CountValid && "BB count is not valid")((BI->CountValid && "BB count is not valid") ? static_cast
<void> (0) : __assert_fail ("BI->CountValid && \"BB count is not valid\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1257, __PRETTY_FUNCTION__));
1258 }
1259#endif
1260 uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue;
1261 F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
1262 uint64_t FuncMaxCount = FuncEntryCount;
1263 for (auto &BB : F) {
1264 auto BI = findBBInfo(&BB);
1265 if (BI == nullptr)
1266 continue;
1267 FuncMaxCount = std::max(FuncMaxCount, BI->CountValue);
1268 }
1269 markFunctionAttributes(FuncEntryCount, FuncMaxCount);
1270
1271 // Now annotate select instructions
1272 FuncInfo.SIVisitor.annotateSelects(F, this, &CountPosition);
1273 assert(CountPosition == ProfileCountSize)((CountPosition == ProfileCountSize) ? static_cast<void>
(0) : __assert_fail ("CountPosition == ProfileCountSize", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1273, __PRETTY_FUNCTION__));
1274
1275 LLVM_DEBUG(FuncInfo.dumpInfo("after reading profile."))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { FuncInfo.dumpInfo("after reading profile."
); } } while (false);
1276}
1277
1278// Assign the scaled count values to the BB with multiple out edges.
1279void PGOUseFunc::setBranchWeights() {
1280 // Generate MD_prof metadata for every branch instruction.
1281 LLVM_DEBUG(dbgs() << "\nSetting branch weights for func " << F.getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "\nSetting branch weights for func "
<< F.getName() << " IsCS=" << IsCS <<
"\n"; } } while (false)
1282 << " IsCS=" << IsCS << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "\nSetting branch weights for func "
<< F.getName() << " IsCS=" << IsCS <<
"\n"; } } while (false);
1283 for (auto &BB : F) {
1284 Instruction *TI = BB.getTerminator();
1285 if (TI->getNumSuccessors() < 2)
1286 continue;
1287 if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
1288 isa<IndirectBrInst>(TI)))
1289 continue;
1290
1291 if (getBBInfo(&BB).CountValue == 0)
1292 continue;
1293
1294 // We have a non-zero Branch BB.
1295 const UseBBInfo &BBCountInfo = getBBInfo(&BB);
1296 unsigned Size = BBCountInfo.OutEdges.size();
1297 SmallVector<uint64_t, 2> EdgeCounts(Size, 0);
1298 uint64_t MaxCount = 0;
1299 for (unsigned s = 0; s < Size; s++) {
1300 const PGOUseEdge *E = BBCountInfo.OutEdges[s];
1301 const BasicBlock *SrcBB = E->SrcBB;
1302 const BasicBlock *DestBB = E->DestBB;
1303 if (DestBB == nullptr)
1304 continue;
1305 unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
1306 uint64_t EdgeCount = E->CountValue;
1307 if (EdgeCount > MaxCount)
1308 MaxCount = EdgeCount;
1309 EdgeCounts[SuccNum] = EdgeCount;
1310 }
1311 setProfMetadata(M, TI, EdgeCounts, MaxCount);
1312 }
1313}
1314
1315static bool isIndirectBrTarget(BasicBlock *BB) {
1316 for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
1317 if (isa<IndirectBrInst>((*PI)->getTerminator()))
1318 return true;
1319 }
1320 return false;
1321}
1322
1323void PGOUseFunc::annotateIrrLoopHeaderWeights() {
1324 LLVM_DEBUG(dbgs() << "\nAnnotating irreducible loop header weights.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "\nAnnotating irreducible loop header weights.\n"
; } } while (false);
1325 // Find irr loop headers
1326 for (auto &BB : F) {
1327 // As a heuristic also annotate indrectbr targets as they have a high chance
1328 // to become an irreducible loop header after the indirectbr tail
1329 // duplication.
1330 if (BFI->isIrrLoopHeader(&BB) || isIndirectBrTarget(&BB)) {
1331 Instruction *TI = BB.getTerminator();
1332 const UseBBInfo &BBCountInfo = getBBInfo(&BB);
1333 setIrrLoopHeaderMetadata(M, TI, BBCountInfo.CountValue);
1334 }
1335 }
1336}
1337
1338void SelectInstVisitor::instrumentOneSelectInst(SelectInst &SI) {
1339 Module *M = F.getParent();
1340 IRBuilder<> Builder(&SI);
1341 Type *Int64Ty = Builder.getInt64Ty();
1342 Type *I8PtrTy = Builder.getInt8PtrTy();
1343 auto *Step = Builder.CreateZExt(SI.getCondition(), Int64Ty);
1344 Builder.CreateCall(
1345 Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment_step),
1346 {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
1347 Builder.getInt64(FuncHash), Builder.getInt32(TotalNumCtrs),
1348 Builder.getInt32(*CurCtrIdx), Step});
1349 ++(*CurCtrIdx);
1350}
1351
1352void SelectInstVisitor::annotateOneSelectInst(SelectInst &SI) {
1353 std::vector<uint64_t> &CountFromProfile = UseFunc->getProfileRecord().Counts;
1354 assert(*CurCtrIdx < CountFromProfile.size() &&((*CurCtrIdx < CountFromProfile.size() && "Out of bound access of counters"
) ? static_cast<void> (0) : __assert_fail ("*CurCtrIdx < CountFromProfile.size() && \"Out of bound access of counters\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1355, __PRETTY_FUNCTION__))
1355 "Out of bound access of counters")((*CurCtrIdx < CountFromProfile.size() && "Out of bound access of counters"
) ? static_cast<void> (0) : __assert_fail ("*CurCtrIdx < CountFromProfile.size() && \"Out of bound access of counters\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1355, __PRETTY_FUNCTION__));
1356 uint64_t SCounts[2];
1357 SCounts[0] = CountFromProfile[*CurCtrIdx]; // True count
1358 ++(*CurCtrIdx);
1359 uint64_t TotalCount = 0;
1360 auto BI = UseFunc->findBBInfo(SI.getParent());
1361 if (BI != nullptr)
1362 TotalCount = BI->CountValue;
1363 // False Count
1364 SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);
1365 uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);
1366 if (MaxCount)
1367 setProfMetadata(F.getParent(), &SI, SCounts, MaxCount);
1368}
1369
1370void SelectInstVisitor::visitSelectInst(SelectInst &SI) {
1371 if (!PGOInstrSelect)
1372 return;
1373 // FIXME: do not handle this yet.
1374 if (SI.getCondition()->getType()->isVectorTy())
1375 return;
1376
1377 switch (Mode) {
1378 case VM_counting:
1379 NSIs++;
1380 return;
1381 case VM_instrument:
1382 instrumentOneSelectInst(SI);
1383 return;
1384 case VM_annotate:
1385 annotateOneSelectInst(SI);
1386 return;
1387 }
1388
1389 llvm_unreachable("Unknown visiting mode")::llvm::llvm_unreachable_internal("Unknown visiting mode", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1389);
1390}
1391
1392void MemIntrinsicVisitor::instrumentOneMemIntrinsic(MemIntrinsic &MI) {
1393 Module *M = F.getParent();
1394 IRBuilder<> Builder(&MI);
1395 Type *Int64Ty = Builder.getInt64Ty();
1396 Type *I8PtrTy = Builder.getInt8PtrTy();
1397 Value *Length = MI.getLength();
1398 assert(!isa<ConstantInt>(Length))((!isa<ConstantInt>(Length)) ? static_cast<void> (
0) : __assert_fail ("!isa<ConstantInt>(Length)", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1398, __PRETTY_FUNCTION__));
1399 Builder.CreateCall(
1400 Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
1401 {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
1402 Builder.getInt64(FuncHash), Builder.CreateZExtOrTrunc(Length, Int64Ty),
1403 Builder.getInt32(IPVK_MemOPSize), Builder.getInt32(CurCtrId)});
1404 ++CurCtrId;
1405}
1406
1407void MemIntrinsicVisitor::visitMemIntrinsic(MemIntrinsic &MI) {
1408 if (!PGOInstrMemOP)
1409 return;
1410 Value *Length = MI.getLength();
1411 // Not instrument constant length calls.
1412 if (dyn_cast<ConstantInt>(Length))
1413 return;
1414
1415 switch (Mode) {
1416 case VM_counting:
1417 NMemIs++;
1418 return;
1419 case VM_instrument:
1420 instrumentOneMemIntrinsic(MI);
1421 return;
1422 case VM_annotate:
1423 Candidates.push_back(&MI);
1424 return;
1425 }
1426 llvm_unreachable("Unknown visiting mode")::llvm::llvm_unreachable_internal("Unknown visiting mode", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1426);
1427}
1428
1429// Traverse all valuesites and annotate the instructions for all value kind.
1430void PGOUseFunc::annotateValueSites() {
1431 if (DisableValueProfiling)
1432 return;
1433
1434 // Create the PGOFuncName meta data.
1435 createPGOFuncNameMetadata(F, FuncInfo.FuncName);
1436
1437 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1438 annotateValueSites(Kind);
1439}
1440
1441static const char *ValueProfKindDescr[] = {
1442#define VALUE_PROF_KIND(Enumerator, Value, Descr) Descr,
1443#include "llvm/ProfileData/InstrProfData.inc"
1444};
1445
1446// Annotate the instructions for a specific value kind.
1447void PGOUseFunc::annotateValueSites(uint32_t Kind) {
1448 assert(Kind <= IPVK_Last)((Kind <= IPVK_Last) ? static_cast<void> (0) : __assert_fail
("Kind <= IPVK_Last", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1448, __PRETTY_FUNCTION__));
1449 unsigned ValueSiteIndex = 0;
1450 auto &ValueSites = FuncInfo.ValueSites[Kind];
1451 unsigned NumValueSites = ProfileRecord.getNumValueSites(Kind);
1452 if (NumValueSites != ValueSites.size()) {
1453 auto &Ctx = M->getContext();
1454 Ctx.diagnose(DiagnosticInfoPGOProfile(
1455 M->getName().data(),
1456 Twine("Inconsistent number of value sites for ") +
1457 Twine(ValueProfKindDescr[Kind]) +
1458 Twine(" profiling in \"") + F.getName().str() +
1459 Twine("\", possibly due to the use of a stale profile."),
1460 DS_Warning));
1461 return;
1462 }
1463
1464 for (auto &I : ValueSites) {
1465 LLVM_DEBUG(dbgs() << "Read one value site profile (kind = " << Kinddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Read one value site profile (kind = "
<< Kind << "): Index = " << ValueSiteIndex
<< " out of " << NumValueSites << "\n"; } }
while (false)
1466 << "): Index = " << ValueSiteIndex << " out of "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Read one value site profile (kind = "
<< Kind << "): Index = " << ValueSiteIndex
<< " out of " << NumValueSites << "\n"; } }
while (false)
1467 << NumValueSites << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Read one value site profile (kind = "
<< Kind << "): Index = " << ValueSiteIndex
<< " out of " << NumValueSites << "\n"; } }
while (false);
1468 annotateValueSite(*M, *I, ProfileRecord,
1469 static_cast<InstrProfValueKind>(Kind), ValueSiteIndex,
1470 Kind == IPVK_MemOPSize ? MaxNumMemOPAnnotations
1471 : MaxNumAnnotations);
1472 ValueSiteIndex++;
1473 }
1474}
1475
1476// Collect the set of members for each Comdat in module M and store
1477// in ComdatMembers.
1478static void collectComdatMembers(
1479 Module &M,
1480 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
1481 if (!DoComdatRenaming)
1482 return;
1483 for (Function &F : M)
1484 if (Comdat *C = F.getComdat())
1485 ComdatMembers.insert(std::make_pair(C, &F));
1486 for (GlobalVariable &GV : M.globals())
1487 if (Comdat *C = GV.getComdat())
1488 ComdatMembers.insert(std::make_pair(C, &GV));
1489 for (GlobalAlias &GA : M.aliases())
1490 if (Comdat *C = GA.getComdat())
1491 ComdatMembers.insert(std::make_pair(C, &GA));
1492}
1493
1494static bool InstrumentAllFunctions(
1495 Module &M, function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
1496 function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS) {
1497 // For the context-sensitve instrumentation, we should have a separated pass
1498 // (before LTO/ThinLTO linking) to create these variables.
1499 if (!IsCS)
1500 createIRLevelProfileFlagVar(M, /* IsCS */ false);
1501 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1502 collectComdatMembers(M, ComdatMembers);
1503
1504 for (auto &F : M) {
1505 if (F.isDeclaration())
1506 continue;
1507 auto *BPI = LookupBPI(F);
1508 auto *BFI = LookupBFI(F);
1509 instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers, IsCS);
1510 }
1511 return true;
1512}
1513
1514PreservedAnalyses
1515PGOInstrumentationGenCreateVar::run(Module &M, ModuleAnalysisManager &AM) {
1516 createProfileFileNameVar(M, CSInstrName);
1517 createIRLevelProfileFlagVar(M, /* IsCS */ true);
1518 return PreservedAnalyses::all();
1519}
1520
1521bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) {
1522 if (skipModule(M))
1523 return false;
1524
1525 auto LookupBPI = [this](Function &F) {
1526 return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI();
1527 };
1528 auto LookupBFI = [this](Function &F) {
1529 return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
1530 };
1531 return InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS);
1532}
1533
1534PreservedAnalyses PGOInstrumentationGen::run(Module &M,
1535 ModuleAnalysisManager &AM) {
1536 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1537 auto LookupBPI = [&FAM](Function &F) {
1538 return &FAM.getResult<BranchProbabilityAnalysis>(F);
1539 };
1540
1541 auto LookupBFI = [&FAM](Function &F) {
1542 return &FAM.getResult<BlockFrequencyAnalysis>(F);
1543 };
1544
1545 if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI, IsCS))
1546 return PreservedAnalyses::all();
1547
1548 return PreservedAnalyses::none();
1549}
1550
1551static bool annotateAllFunctions(
1552 Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName,
1553 function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
1554 function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS) {
1555 LLVM_DEBUG(dbgs() << "Read in profile counters: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Read in profile counters: "
; } } while (false);
4
←
Assuming 'DebugFlag' is 0
→
5
←
Loop condition is false. Exiting loop
→
1556 auto &Ctx = M.getContext();
1557 // Read the counter array from file.
1558 auto ReaderOrErr =
1559 IndexedInstrProfReader::create(ProfileFileName, ProfileRemappingFileName);
1560 if (Error E = ReaderOrErr.takeError()) {
6
←
Taking false branch
→
1561 handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) {
1562 Ctx.diagnose(
1563 DiagnosticInfoPGOProfile(ProfileFileName.data(), EI.message()));
1564 });
1565 return false;
1566 }
1567
1568 std::unique_ptr<IndexedInstrProfReader> PGOReader =
1569 std::move(ReaderOrErr.get());
1570 if (!PGOReader) {
7
←
Taking false branch
→
1571 Ctx.diagnose(DiagnosticInfoPGOProfile(ProfileFileName.data(),
1572 StringRef("Cannot get PGOReader")));
1573 return false;
1574 }
1575 if (!PGOReader->hasCSIRLevelProfile() && IsCS)
8
←
Assuming the condition is false
→
1576 return false;
1577
1578 // TODO: might need to change the warning once the clang option is finalized.
1579 if (!PGOReader->isIRLevelProfile()) {
9
←
Assuming the condition is false
→
10
←
Taking false branch
→
1580 Ctx.diagnose(DiagnosticInfoPGOProfile(
1581 ProfileFileName.data(), "Not an IR level instrumentation profile"));
1582 return false;
1583 }
1584
1585 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1586 collectComdatMembers(M, ComdatMembers);
1587 std::vector<Function *> HotFunctions;
1588 std::vector<Function *> ColdFunctions;
1589 for (auto &F : M) {
1590 if (F.isDeclaration())
11
←
Assuming the condition is false
→
12
←
Taking false branch
→
1591 continue;
1592 auto *BPI = LookupBPI(F);
1593 auto *BFI = LookupBFI(F);
1594 // Split indirectbr critical edges here before computing the MST rather than
1595 // later in getInstrBB() to avoid invalidating it.
1596 SplitIndirectBrCriticalEdges(F, BPI, BFI);
1597 PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, IsCS);
1598 bool AllZeros = false;
1599 if (!Func.readCounters(PGOReader.get(), AllZeros))
13
←
Calling 'PGOUseFunc::readCounters'
→
1600 continue;
1601 if (AllZeros) {
1602 F.setEntryCount(ProfileCount(0, Function::PCT_Real));
1603 if (Func.getProgramMaxCount() != 0)
1604 ColdFunctions.push_back(&F);
1605 continue;
1606 }
1607 Func.populateCounters();
1608 Func.setBranchWeights();
1609 Func.annotateValueSites();
1610 Func.annotateIrrLoopHeaderWeights();
1611 PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();
1612 if (FreqAttr == PGOUseFunc::FFA_Cold)
1613 ColdFunctions.push_back(&F);
1614 else if (FreqAttr == PGOUseFunc::FFA_Hot)
1615 HotFunctions.push_back(&F);
1616 if (PGOViewCounts != PGOVCT_None &&
1617 (ViewBlockFreqFuncName.empty() ||
1618 F.getName().equals(ViewBlockFreqFuncName))) {
1619 LoopInfo LI{DominatorTree(F)};
1620 std::unique_ptr<BranchProbabilityInfo> NewBPI =
1621 llvm::make_unique<BranchProbabilityInfo>(F, LI);
1622 std::unique_ptr<BlockFrequencyInfo> NewBFI =
1623 llvm::make_unique<BlockFrequencyInfo>(F, *NewBPI, LI);
1624 if (PGOViewCounts == PGOVCT_Graph)
1625 NewBFI->view();
1626 else if (PGOViewCounts == PGOVCT_Text) {
1627 dbgs() << "pgo-view-counts: " << Func.getFunc().getName() << "\n";
1628 NewBFI->print(dbgs());
1629 }
1630 }
1631 if (PGOViewRawCounts != PGOVCT_None &&
1632 (ViewBlockFreqFuncName.empty() ||
1633 F.getName().equals(ViewBlockFreqFuncName))) {
1634 if (PGOViewRawCounts == PGOVCT_Graph)
1635 if (ViewBlockFreqFuncName.empty())
1636 WriteGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());
1637 else
1638 ViewGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());
1639 else if (PGOViewRawCounts == PGOVCT_Text) {
1640 dbgs() << "pgo-view-raw-counts: " << Func.getFunc().getName() << "\n";
1641 Func.dumpInfo();
1642 }
1643 }
1644 }
1645 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
1646 IsCS ? ProfileSummary::PSK_CSInstr
1647 : ProfileSummary::PSK_Instr);
1648
1649 // Set function hotness attribute from the profile.
1650 // We have to apply these attributes at the end because their presence
1651 // can affect the BranchProbabilityInfo of any callers, resulting in an
1652 // inconsistent MST between prof-gen and prof-use.
1653 for (auto &F : HotFunctions) {
1654 F->addFnAttr(Attribute::InlineHint);
1655 LLVM_DEBUG(dbgs() << "Set inline attribute to function: " << F->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Set inline attribute to function: "
<< F->getName() << "\n"; } } while (false)
1656 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Set inline attribute to function: "
<< F->getName() << "\n"; } } while (false);
1657 }
1658 for (auto &F : ColdFunctions) {
1659 F->addFnAttr(Attribute::Cold);
1660 LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Set cold attribute to function: "
<< F->getName() << "\n"; } } while (false)
1661 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Set cold attribute to function: "
<< F->getName() << "\n"; } } while (false);
1662 }
1663 return true;
1664}
1665
1666PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename,
1667 std::string RemappingFilename,
1668 bool IsCS)
1669 : ProfileFileName(std::move(Filename)),
1670 ProfileRemappingFileName(std::move(RemappingFilename)), IsCS(IsCS) {
1671 if (!PGOTestProfileFile.empty())
1672 ProfileFileName = PGOTestProfileFile;
1673 if (!PGOTestProfileRemappingFile.empty())
1674 ProfileRemappingFileName = PGOTestProfileRemappingFile;
1675}
1676
1677PreservedAnalyses PGOInstrumentationUse::run(Module &M,
1678 ModuleAnalysisManager &AM) {
1679
1680 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1681 auto LookupBPI = [&FAM](Function &F) {
1682 return &FAM.getResult<BranchProbabilityAnalysis>(F);
1683 };
1684
1685 auto LookupBFI = [&FAM](Function &F) {
1686 return &FAM.getResult<BlockFrequencyAnalysis>(F);
1687 };
1688
1689 if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName,
1690 LookupBPI, LookupBFI, IsCS))
1691 return PreservedAnalyses::all();
1692
1693 return PreservedAnalyses::none();
1694}
1695
1696bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) {
1697 if (skipModule(M))
1
Assuming the condition is false
→
2
←
Taking false branch
→
1698 return false;
1699
1700 auto LookupBPI = [this](Function &F) {
1701 return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI();
1702 };
1703 auto LookupBFI = [this](Function &F) {
1704 return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
1705 };
1706
1707 return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI,
3
←
Calling 'annotateAllFunctions'
→
1708 IsCS);
1709}
1710
1711static std::string getSimpleNodeName(const BasicBlock *Node) {
1712 if (!Node->getName().empty())
1713 return Node->getName();
1714
1715 std::string SimpleNodeName;
1716 raw_string_ostream OS(SimpleNodeName);
1717 Node->printAsOperand(OS, false);
1718 return OS.str();
1719}
1720
1721void llvm::setProfMetadata(Module *M, Instruction *TI,
1722 ArrayRef<uint64_t> EdgeCounts,
1723 uint64_t MaxCount) {
1724 MDBuilder MDB(M->getContext());
1725 assert(MaxCount > 0 && "Bad max count")((MaxCount > 0 && "Bad max count") ? static_cast<
void> (0) : __assert_fail ("MaxCount > 0 && \"Bad max count\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1725, __PRETTY_FUNCTION__));
1726 uint64_t Scale = calculateCountScale(MaxCount);
1727 SmallVector<unsigned, 4> Weights;
1728 for (const auto &ECI : EdgeCounts)
1729 Weights.push_back(scaleBranchCount(ECI, Scale));
1730
1731 LLVM_DEBUG(dbgs() << "Weight is: "; for (const auto &Wdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Weight is: "; for
(const auto &W : Weights) { dbgs() << W << " "
; } dbgs() << "\n";; } } while (false)
1732 : Weights) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Weight is: "; for
(const auto &W : Weights) { dbgs() << W << " "
; } dbgs() << "\n";; } } while (false)
1733 dbgs() << W << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Weight is: "; for
(const auto &W : Weights) { dbgs() << W << " "
; } dbgs() << "\n";; } } while (false)
1734 } dbgs() << "\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Weight is: "; for
(const auto &W : Weights) { dbgs() << W << " "
; } dbgs() << "\n";; } } while (false);
1735 TI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
1736 if (EmitBranchProbability) {
1737 std::string BrCondStr = getBranchCondString(TI);
1738 if (BrCondStr.empty())
1739 return;
1740
1741 uint64_t WSum =
1742 std::accumulate(Weights.begin(), Weights.end(), (uint64_t)0,
1743 [](uint64_t w1, uint64_t w2) { return w1 + w2; });
1744 uint64_t TotalCount =
1745 std::accumulate(EdgeCounts.begin(), EdgeCounts.end(), (uint64_t)0,
1746 [](uint64_t c1, uint64_t c2) { return c1 + c2; });
1747 Scale = calculateCountScale(WSum);
1748 BranchProbability BP(scaleBranchCount(Weights[0], Scale),
1749 scaleBranchCount(WSum, Scale));
1750 std::string BranchProbStr;
1751 raw_string_ostream OS(BranchProbStr);
1752 OS << BP;
1753 OS << " (total count : " << TotalCount << ")";
1754 OS.flush();
1755 Function *F = TI->getParent()->getParent();
1756 OptimizationRemarkEmitter ORE(F);
1757 ORE.emit([&]() {
1758 return OptimizationRemark(DEBUG_TYPE"pgo-instrumentation", "pgo-instrumentation", TI)
1759 << BrCondStr << " is true with probability : " << BranchProbStr;
1760 });
1761 }
1762}
1763
1764namespace llvm {
1765
1766void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count) {
1767 MDBuilder MDB(M->getContext());
1768 TI->setMetadata(llvm::LLVMContext::MD_irr_loop,
1769 MDB.createIrrLoopHeaderWeight(Count));
1770}
1771
1772template <> struct GraphTraits<PGOUseFunc *> {
1773 using NodeRef = const BasicBlock *;
1774 using ChildIteratorType = succ_const_iterator;
1775 using nodes_iterator = pointer_iterator<Function::const_iterator>;
1776
1777 static NodeRef getEntryNode(const PGOUseFunc *G) {
1778 return &G->getFunc().front();
1779 }
1780
1781 static ChildIteratorType child_begin(const NodeRef N) {
1782 return succ_begin(N);
1783 }
1784
1785 static ChildIteratorType child_end(const NodeRef N) { return succ_end(N); }
1786
1787 static nodes_iterator nodes_begin(const PGOUseFunc *G) {
1788 return nodes_iterator(G->getFunc().begin());
1789 }
1790
1791 static nodes_iterator nodes_end(const PGOUseFunc *G) {
1792 return nodes_iterator(G->getFunc().end());
1793 }
1794};
1795
1796template <> struct DOTGraphTraits<PGOUseFunc *> : DefaultDOTGraphTraits {
1797 explicit DOTGraphTraits(bool isSimple = false)
1798 : DefaultDOTGraphTraits(isSimple) {}
1799
1800 static std::string getGraphName(const PGOUseFunc *G) {
1801 return G->getFunc().getName();
1802 }
1803
1804 std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph) {
1805 std::string Result;
1806 raw_string_ostream OS(Result);
1807
1808 OS << getSimpleNodeName(Node) << ":\\l";
1809 UseBBInfo *BI = Graph->findBBInfo(Node);
1810 OS << "Count : ";
1811 if (BI && BI->CountValid)
1812 OS << BI->CountValue << "\\l";
1813 else
1814 OS << "Unknown\\l";
1815
1816 if (!PGOInstrSelect)
1817 return Result;
1818
1819 for (auto BI = Node->begin(); BI != Node->end(); ++BI) {
1820 auto *I = &*BI;
1821 if (!isa<SelectInst>(I))
1822 continue;
1823 // Display scaled counts for SELECT instruction:
1824 OS << "SELECT : { T = ";
1825 uint64_t TC, FC;
1826 bool HasProf = I->extractProfMetadata(TC, FC);
1827 if (!HasProf)
1828 OS << "Unknown, F = Unknown }\\l";
1829 else
1830 OS << TC << ", F = " << FC << " }\\l";
1831 }
1832 return Result;
1833 }
1834};
1835
1836} // end namespace llvm
←

/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h

1//===- llvm/Support/Error.h - Recoverable error handling --------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an API used to report recoverable errors.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_SUPPORT_ERROR_H
14#define LLVM_SUPPORT_ERROR_H
15
16#include "llvm-c/Error.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/StringExtras.h"
20#include "llvm/ADT/Twine.h"
21#include "llvm/Config/abi-breaking.h"
22#include "llvm/Support/AlignOf.h"
23#include "llvm/Support/Compiler.h"
24#include "llvm/Support/Debug.h"
25#include "llvm/Support/ErrorHandling.h"
26#include "llvm/Support/ErrorOr.h"
27#include "llvm/Support/Format.h"
28#include "llvm/Support/raw_ostream.h"
29#include <algorithm>
30#include <cassert>
31#include <cstdint>
32#include <cstdlib>
33#include <functional>
34#include <memory>
35#include <new>
36#include <string>
37#include <system_error>
38#include <type_traits>
39#include <utility>
40#include <vector>
41
42namespace llvm {
43
44class ErrorSuccess;
45
46/// Base class for error info classes. Do not extend this directly: Extend
47/// the ErrorInfo template subclass instead.
48class ErrorInfoBase {
49public:
50 virtual ~ErrorInfoBase() = default;
51
52 /// Print an error message to an output stream.
53 virtual void log(raw_ostream &OS) const = 0;
54
55 /// Return the error message as a string.
56 virtual std::string message() const {
57 std::string Msg;
58 raw_string_ostream OS(Msg);
59 log(OS);
60 return OS.str();
61 }
62
63 /// Convert this error to a std::error_code.
64 ///
65 /// This is a temporary crutch to enable interaction with code still
66 /// using std::error_code. It will be removed in the future.
67 virtual std::error_code convertToErrorCode() const = 0;
68
69 // Returns the class ID for this type.
70 static const void *classID() { return &ID; }
71
72 // Returns the class ID for the dynamic type of this ErrorInfoBase instance.
73 virtual const void *dynamicClassID() const = 0;
74
75 // Check whether this instance is a subclass of the class identified by
76 // ClassID.
77 virtual bool isA(const void *const ClassID) const {
78 return ClassID == classID();
79 }
80
81 // Check whether this instance is a subclass of ErrorInfoT.
82 template <typename ErrorInfoT> bool isA() const {
83 return isA(ErrorInfoT::classID());
84 }
85
86private:
87 virtual void anchor();
88
89 static char ID;
90};
91
92/// Lightweight error class with error context and mandatory checking.
93///
94/// Instances of this class wrap a ErrorInfoBase pointer. Failure states
95/// are represented by setting the pointer to a ErrorInfoBase subclass
96/// instance containing information describing the failure. Success is
97/// represented by a null pointer value.
98///
99/// Instances of Error also contains a 'Checked' flag, which must be set
100/// before the destructor is called, otherwise the destructor will trigger a
101/// runtime error. This enforces at runtime the requirement that all Error
102/// instances be checked or returned to the caller.
103///
104/// There are two ways to set the checked flag, depending on what state the
105/// Error instance is in. For Error instances indicating success, it
106/// is sufficient to invoke the boolean conversion operator. E.g.:
107///
108/// @code{.cpp}
109/// Error foo(<...>);
110///
111/// if (auto E = foo(<...>))
112/// return E; // <- Return E if it is in the error state.
113/// // We have verified that E was in the success state. It can now be safely
114/// // destroyed.
115/// @endcode
116///
117/// A success value *can not* be dropped. For example, just calling 'foo(<...>)'
118/// without testing the return value will raise a runtime error, even if foo
119/// returns success.
120///
121/// For Error instances representing failure, you must use either the
122/// handleErrors or handleAllErrors function with a typed handler. E.g.:
123///
124/// @code{.cpp}
125/// class MyErrorInfo : public ErrorInfo<MyErrorInfo> {
126/// // Custom error info.
127/// };
128///
129/// Error foo(<...>) { return make_error<MyErrorInfo>(...); }
130///
131/// auto E = foo(<...>); // <- foo returns failure with MyErrorInfo.
132/// auto NewE =
133/// handleErrors(E,
134/// [](const MyErrorInfo &M) {
135/// // Deal with the error.
136/// },
137/// [](std::unique_ptr<OtherError> M) -> Error {
138/// if (canHandle(*M)) {
139/// // handle error.
140/// return Error::success();
141/// }
142/// // Couldn't handle this error instance. Pass it up the stack.
143/// return Error(std::move(M));
144/// );
145/// // Note - we must check or return NewE in case any of the handlers
146/// // returned a new error.
147/// @endcode
148///
149/// The handleAllErrors function is identical to handleErrors, except
150/// that it has a void return type, and requires all errors to be handled and
151/// no new errors be returned. It prevents errors (assuming they can all be
152/// handled) from having to be bubbled all the way to the top-level.
153///
154/// *All* Error instances must be checked before destruction, even if
155/// they're moved-assigned or constructed from Success values that have already
156/// been checked. This enforces checking through all levels of the call stack.
157class LLVM_NODISCARD[[clang::warn_unused_result]] Error {
158 // Both ErrorList and FileError need to be able to yank ErrorInfoBase
159 // pointers out of this class to add to the error list.
160 friend class ErrorList;
161 friend class FileError;
162
163 // handleErrors needs to be able to set the Checked flag.
164 template <typename... HandlerTs>
165 friend Error handleErrors(Error E, HandlerTs &&... Handlers);
166
167 // Expected<T> needs to be able to steal the payload when constructed from an
168 // error.
169 template <typename T> friend class Expected;
170
171 // wrap needs to be able to steal the payload.
172 friend LLVMErrorRef wrap(Error);
173
174protected:
175 /// Create a success value. Prefer using 'Error::success()' for readability
176 Error() {
177 setPtr(nullptr);
178 setChecked(false);
179 }
180
181public:
182 /// Create a success value.
183 static ErrorSuccess success();
184
185 // Errors are not copy-constructable.
186 Error(const Error &Other) = delete;
187
188 /// Move-construct an error value. The newly constructed error is considered
189 /// unchecked, even if the source error had been checked. The original error
190 /// becomes a checked Success value, regardless of its original state.
191 Error(Error &&Other) {
192 setChecked(true);
193 *this = std::move(Other);
194 }
195
196 /// Create an error value. Prefer using the 'make_error' function, but
197 /// this constructor can be useful when "re-throwing" errors from handlers.
198 Error(std::unique_ptr<ErrorInfoBase> Payload) {
199 setPtr(Payload.release());
200 setChecked(false);
33
←
Potential leak of memory pointed to by 'Payload._M_t._M_head_impl'
201 }
202
203 // Errors are not copy-assignable.
204 Error &operator=(const Error &Other) = delete;
205
206 /// Move-assign an error value. The current error must represent success, you
207 /// you cannot overwrite an unhandled error. The current error is then
208 /// considered unchecked. The source error becomes a checked success value,
209 /// regardless of its original state.
210 Error &operator=(Error &&Other) {
211 // Don't allow overwriting of unchecked values.
212 assertIsChecked();
213 setPtr(Other.getPtr());
214
215 // This Error is unchecked, even if the source error was checked.
216 setChecked(false);
217
218 // Null out Other's payload and set its checked bit.
219 Other.setPtr(nullptr);
220 Other.setChecked(true);
221
222 return *this;
223 }
224
225 /// Destroy a Error. Fails with a call to abort() if the error is
226 /// unchecked.
227 ~Error() {
228 assertIsChecked();
229 delete getPtr();
230 }
231
232 /// Bool conversion. Returns true if this Error is in a failure state,
233 /// and false if it is in an accept state. If the error is in a Success state
234 /// it will be considered checked.
235 explicit operator bool() {
236 setChecked(getPtr() == nullptr);
237 return getPtr() != nullptr;
238 }
239
240 /// Check whether one error is a subclass of another.
241 template <typename ErrT> bool isA() const {
242 return getPtr() && getPtr()->isA(ErrT::classID());
243 }
244
245 /// Returns the dynamic class id of this error, or null if this is a success
246 /// value.
247 const void* dynamicClassID() const {
248 if (!getPtr())
249 return nullptr;
250 return getPtr()->dynamicClassID();
251 }
252
253private:
254#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
255 // assertIsChecked() happens very frequently, but under normal circumstances
256 // is supposed to be a no-op. So we want it to be inlined, but having a bunch
257 // of debug prints can cause the function to be too large for inlining. So
258 // it's important that we define this function out of line so that it can't be
259 // inlined.
260 LLVM_ATTRIBUTE_NORETURN__attribute__((noreturn))
261 void fatalUncheckedError() const;
262#endif
263
264 void assertIsChecked() {
265#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
266 if (LLVM_UNLIKELY(!getChecked() || getPtr())__builtin_expect((bool)(!getChecked() || getPtr()), false))
267 fatalUncheckedError();
268#endif
269 }
270
271 ErrorInfoBase *getPtr() const {
272 return reinterpret_cast<ErrorInfoBase*>(
273 reinterpret_cast<uintptr_t>(Payload) &
274 ~static_cast<uintptr_t>(0x1));
275 }
276
277 void setPtr(ErrorInfoBase *EI) {
278#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
279 Payload = reinterpret_cast<ErrorInfoBase*>(
280 (reinterpret_cast<uintptr_t>(EI) &
281 ~static_cast<uintptr_t>(0x1)) |
282 (reinterpret_cast<uintptr_t>(Payload) & 0x1));
283#else
284 Payload = EI;
285#endif
286 }
287
288 bool getChecked() const {
289#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
290 return (reinterpret_cast<uintptr_t>(Payload) & 0x1) == 0;
291#else
292 return true;
293#endif
294 }
295
296 void setChecked(bool V) {
297 Payload = reinterpret_cast<ErrorInfoBase*>(
298 (reinterpret_cast<uintptr_t>(Payload) &
299 ~static_cast<uintptr_t>(0x1)) |
300 (V ? 0 : 1));
301 }
302
303 std::unique_ptr<ErrorInfoBase> takePayload() {
304 std::unique_ptr<ErrorInfoBase> Tmp(getPtr());
305 setPtr(nullptr);
306 setChecked(true);
307 return Tmp;
308 }
309
310 friend raw_ostream &operator<<(raw_ostream &OS, const Error &E) {
311 if (auto P = E.getPtr())
312 P->log(OS);
313 else
314 OS << "success";
315 return OS;
316 }
317
318 ErrorInfoBase *Payload = nullptr;
319};
320
321/// Subclass of Error for the sole purpose of identifying the success path in
322/// the type system. This allows to catch invalid conversion to Expected<T> at
323/// compile time.
324class ErrorSuccess final : public Error {};
325
326inline ErrorSuccess Error::success() { return ErrorSuccess(); }
327
328/// Make a Error instance representing failure using the given error info
329/// type.
330template <typename ErrT, typename... ArgTs> Error make_error(ArgTs &&... Args) {
331 return Error(llvm::make_unique<ErrT>(std::forward<ArgTs>(Args)...));
332}
333
334/// Base class for user error types. Users should declare their error types
335/// like:
336///
337/// class MyError : public ErrorInfo<MyError> {
338/// ....
339/// };
340///
341/// This class provides an implementation of the ErrorInfoBase::kind
342/// method, which is used by the Error RTTI system.
343template <typename ThisErrT, typename ParentErrT = ErrorInfoBase>
344class ErrorInfo : public ParentErrT {
345public:
346 using ParentErrT::ParentErrT; // inherit constructors
347
348 static const void *classID() { return &ThisErrT::ID; }
349
350 const void *dynamicClassID() const override { return &ThisErrT::ID; }
351
352 bool isA(const void *const ClassID) const override {
353 return ClassID == classID() || ParentErrT::isA(ClassID);
354 }
355};
356
357/// Special ErrorInfo subclass representing a list of ErrorInfos.
358/// Instances of this class are constructed by joinError.
359class ErrorList final : public ErrorInfo<ErrorList> {
360 // handleErrors needs to be able to iterate the payload list of an
361 // ErrorList.
362 template <typename... HandlerTs>
363 friend Error handleErrors(Error E, HandlerTs &&... Handlers);
364
365 // joinErrors is implemented in terms of join.
366 friend Error joinErrors(Error, Error);
367
368public:
369 void log(raw_ostream &OS) const override {
370 OS << "Multiple errors:\n";
371 for (auto &ErrPayload : Payloads) {
372 ErrPayload->log(OS);
373 OS << "\n";
374 }
375 }
376
377 std::error_code convertToErrorCode() const override;
378
379 // Used by ErrorInfo::classID.
380 static char ID;
381
382private:
383 ErrorList(std::unique_ptr<ErrorInfoBase> Payload1,
384 std::unique_ptr<ErrorInfoBase> Payload2) {
385 assert(!Payload1->isA<ErrorList>() && !Payload2->isA<ErrorList>() &&((!Payload1->isA<ErrorList>() && !Payload2->
isA<ErrorList>() && "ErrorList constructor payloads should be singleton errors"
) ? static_cast<void> (0) : __assert_fail ("!Payload1->isA<ErrorList>() && !Payload2->isA<ErrorList>() && \"ErrorList constructor payloads should be singleton errors\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 386, __PRETTY_FUNCTION__))
386 "ErrorList constructor payloads should be singleton errors")((!Payload1->isA<ErrorList>() && !Payload2->
isA<ErrorList>() && "ErrorList constructor payloads should be singleton errors"
) ? static_cast<void> (0) : __assert_fail ("!Payload1->isA<ErrorList>() && !Payload2->isA<ErrorList>() && \"ErrorList constructor payloads should be singleton errors\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 386, __PRETTY_FUNCTION__));
387 Payloads.push_back(std::move(Payload1));
388 Payloads.push_back(std::move(Payload2));
389 }
390
391 static Error join(Error E1, Error E2) {
392 if (!E1)
23
←
Assuming the condition is false
→
24
←
Taking false branch
→
393 return E2;
394 if (!E2)
25
←
Assuming the condition is false
→
26
←
Taking false branch
→
395 return E1;
396 if (E1.isA<ErrorList>()) {
27
←
Assuming the condition is false
→
28
←
Taking false branch
→
397 auto &E1List = static_cast<ErrorList &>(*E1.getPtr());
398 if (E2.isA<ErrorList>()) {
399 auto E2Payload = E2.takePayload();
400 auto &E2List = static_cast<ErrorList &>(*E2Payload);
401 for (auto &Payload : E2List.Payloads)
402 E1List.Payloads.push_back(std::move(Payload));
403 } else
404 E1List.Payloads.push_back(E2.takePayload());
405
406 return E1;
407 }
408 if (E2.isA<ErrorList>()) {
29
←
Assuming the condition is false
→
30
←
Taking false branch
→
409 auto &E2List = static_cast<ErrorList &>(*E2.getPtr());
410 E2List.Payloads.insert(E2List.Payloads.begin(), E1.takePayload());
411 return E2;
412 }
413 return Error(std::unique_ptr<ErrorList>(
32
←
Calling constructor for 'Error'
→
414 new ErrorList(E1.takePayload(), E2.takePayload())));
31
←
Memory is allocated
→
415 }
416
417 std::vector<std::unique_ptr<ErrorInfoBase>> Payloads;
418};
419
420/// Concatenate errors. The resulting Error is unchecked, and contains the
421/// ErrorInfo(s), if any, contained in E1, followed by the
422/// ErrorInfo(s), if any, contained in E2.
423inline Error joinErrors(Error E1, Error E2) {
424 return ErrorList::join(std::move(E1), std::move(E2));
425}
426
427/// Tagged union holding either a T or a Error.
428///
429/// This class parallels ErrorOr, but replaces error_code with Error. Since
430/// Error cannot be copied, this class replaces getError() with
431/// takeError(). It also adds an bool errorIsA<ErrT>() method for testing the
432/// error class type.
433template <class T> class LLVM_NODISCARD[[clang::warn_unused_result]] Expected {
434 template <class T1> friend class ExpectedAsOutParameter;
435 template <class OtherT> friend class Expected;
436
437 static const bool isRef = std::is_reference<T>::value;
438
439 using wrap = std::reference_wrapper<typename std::remove_reference<T>::type>;
440
441 using error_type = std::unique_ptr<ErrorInfoBase>;
442
443public:
444 using storage_type = typename std::conditional<isRef, wrap, T>::type;
445 using value_type = T;
446
447private:
448 using reference = typename std::remove_reference<T>::type &;
449 using const_reference = const typename std::remove_reference<T>::type &;
450 using pointer = typename std::remove_reference<T>::type *;
451 using const_pointer = const typename std::remove_reference<T>::type *;
452
453public:
454 /// Create an Expected<T> error value from the given Error.
455 Expected(Error Err)
456 : HasError(true)
457#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
458 // Expected is unchecked upon construction in Debug builds.
459 , Unchecked(true)
460#endif
461 {
462 assert(Err && "Cannot create Expected<T> from Error success value.")((Err && "Cannot create Expected<T> from Error success value."
) ? static_cast<void> (0) : __assert_fail ("Err && \"Cannot create Expected<T> from Error success value.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 462, __PRETTY_FUNCTION__));
463 new (getErrorStorage()) error_type(Err.takePayload());
464 }
465
466 /// Forbid to convert from Error::success() implicitly, this avoids having
467 /// Expected<T> foo() { return Error::success(); } which compiles otherwise
468 /// but triggers the assertion above.
469 Expected(ErrorSuccess) = delete;
470
471 /// Create an Expected<T> success value from the given OtherT value, which
472 /// must be convertible to T.
473 template <typename OtherT>
474 Expected(OtherT &&Val,
475 typename std::enable_if<std::is_convertible<OtherT, T>::value>::type
476 * = nullptr)
477 : HasError(false)
478#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
479 // Expected is unchecked upon construction in Debug builds.
480 , Unchecked(true)
481#endif
482 {
483 new (getStorage()) storage_type(std::forward<OtherT>(Val));
484 }
485
486 /// Move construct an Expected<T> value.
487 Expected(Expected &&Other) { moveConstruct(std::move(Other)); }
488
489 /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT
490 /// must be convertible to T.
491 template <class OtherT>
492 Expected(Expected<OtherT> &&Other,
493 typename std::enable_if<std::is_convertible<OtherT, T>::value>::type
494 * = nullptr) {
495 moveConstruct(std::move(Other));
496 }
497
498 /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT
499 /// isn't convertible to T.
500 template <class OtherT>
501 explicit Expected(
502 Expected<OtherT> &&Other,
503 typename std::enable_if<!std::is_convertible<OtherT, T>::value>::type * =
504 nullptr) {
505 moveConstruct(std::move(Other));
506 }
507
508 /// Move-assign from another Expected<T>.
509 Expected &operator=(Expected &&Other) {
510 moveAssign(std::move(Other));
511 return *this;
512 }
513
514 /// Destroy an Expected<T>.
515 ~Expected() {
516 assertIsChecked();
517 if (!HasError)
518 getStorage()->~storage_type();
519 else
520 getErrorStorage()->~error_type();
521 }
522
523 /// Return false if there is an error.
524 explicit operator bool() {
525#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
526 Unchecked = HasError;
527#endif
528 return !HasError;
529 }
530
531 /// Returns a reference to the stored T value.
532 reference get() {
533 assertIsChecked();
534 return *getStorage();
535 }
536
537 /// Returns a const reference to the stored T value.
538 const_reference get() const {
539 assertIsChecked();
540 return const_cast<Expected<T> *>(this)->get();
541 }
542
543 /// Check that this Expected<T> is an error of type ErrT.
544 template <typename ErrT> bool errorIsA() const {
545 return HasError && (*getErrorStorage())->template isA<ErrT>();
546 }
547
548 /// Take ownership of the stored error.
549 /// After calling this the Expected<T> is in an indeterminate state that can
550 /// only be safely destructed. No further calls (beside the destructor) should
551 /// be made on the Expected<T> vaule.
552 Error takeError() {
553#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
554 Unchecked = false;
555#endif
556 return HasError ? Error(std::move(*getErrorStorage())) : Error::success();
557 }
558
559 /// Returns a pointer to the stored T value.
560 pointer operator->() {
561 assertIsChecked();
562 return toPointer(getStorage());
563 }
564
565 /// Returns a const pointer to the stored T value.
566 const_pointer operator->() const {
567 assertIsChecked();
568 return toPointer(getStorage());
569 }
570
571 /// Returns a reference to the stored T value.
572 reference operator*() {
573 assertIsChecked();
574 return *getStorage();
575 }
576
577 /// Returns a const reference to the stored T value.
578 const_reference operator*() const {
579 assertIsChecked();
580 return *getStorage();
581 }
582
583private:
584 template <class T1>
585 static bool compareThisIfSameType(const T1 &a, const T1 &b) {
586 return &a == &b;
587 }
588
589 template <class T1, class T2>
590 static bool compareThisIfSameType(const T1 &a, const T2 &b) {
591 return false;
592 }
593
594 template <class OtherT> void moveConstruct(Expected<OtherT> &&Other) {
595 HasError = Other.HasError;
596#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
597 Unchecked = true;
598 Other.Unchecked = false;
599#endif
600
601 if (!HasError)
602 new (getStorage()) storage_type(std::move(*Other.getStorage()));
603 else
604 new (getErrorStorage()) error_type(std::move(*Other.getErrorStorage()));
605 }
606
607 template <class OtherT> void moveAssign(Expected<OtherT> &&Other) {
608 assertIsChecked();
609
610 if (compareThisIfSameType(*this, Other))
611 return;
612
613 this->~Expected();
614 new (this) Expected(std::move(Other));
615 }
616
617 pointer toPointer(pointer Val) { return Val; }
618
619 const_pointer toPointer(const_pointer Val) const { return Val; }
620
621 pointer toPointer(wrap *Val) { return &Val->get(); }
622
623 const_pointer toPointer(const wrap *Val) const { return &Val->get(); }
624
625 storage_type *getStorage() {
626 assert(!HasError && "Cannot get value when an error exists!")((!HasError && "Cannot get value when an error exists!"
) ? static_cast<void> (0) : __assert_fail ("!HasError && \"Cannot get value when an error exists!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 626, __PRETTY_FUNCTION__));
627 return reinterpret_cast<storage_type *>(TStorage.buffer);
628 }
629
630 const storage_type *getStorage() const {
631 assert(!HasError && "Cannot get value when an error exists!")((!HasError && "Cannot get value when an error exists!"
) ? static_cast<void> (0) : __assert_fail ("!HasError && \"Cannot get value when an error exists!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 631, __PRETTY_FUNCTION__));
632 return reinterpret_cast<const storage_type *>(TStorage.buffer);
633 }
634
635 error_type *getErrorStorage() {
636 assert(HasError && "Cannot get error when a value exists!")((HasError && "Cannot get error when a value exists!"
) ? static_cast<void> (0) : __assert_fail ("HasError && \"Cannot get error when a value exists!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 636, __PRETTY_FUNCTION__));
637 return reinterpret_cast<error_type *>(ErrorStorage.buffer);
638 }
639
640 const error_type *getErrorStorage() const {
641 assert(HasError && "Cannot get error when a value exists!")((HasError && "Cannot get error when a value exists!"
) ? static_cast<void> (0) : __assert_fail ("HasError && \"Cannot get error when a value exists!\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 641, __PRETTY_FUNCTION__));
642 return reinterpret_cast<const error_type *>(ErrorStorage.buffer);
643 }
644
645 // Used by ExpectedAsOutParameter to reset the checked flag.
646 void setUnchecked() {
647#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
648 Unchecked = true;
649#endif
650 }
651
652#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
653 LLVM_ATTRIBUTE_NORETURN__attribute__((noreturn))
654 LLVM_ATTRIBUTE_NOINLINE__attribute__((noinline))
655 void fatalUncheckedExpected() const {
656 dbgs() << "Expected<T> must be checked before access or destruction.\n";
657 if (HasError) {
658 dbgs() << "Unchecked Expected<T> contained error:\n";
659 (*getErrorStorage())->log(dbgs());
660 } else
661 dbgs() << "Expected<T> value was in success state. (Note: Expected<T> "
662 "values in success mode must still be checked prior to being "
663 "destroyed).\n";
664 abort();
665 }
666#endif
667
668 void assertIsChecked() {
669#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
670 if (LLVM_UNLIKELY(Unchecked)__builtin_expect((bool)(Unchecked), false))
671 fatalUncheckedExpected();
672#endif
673 }
674
675 union {
676 AlignedCharArrayUnion<storage_type> TStorage;
677 AlignedCharArrayUnion<error_type> ErrorStorage;
678 };
679 bool HasError : 1;
680#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
681 bool Unchecked : 1;
682#endif
683};
684
685/// Report a serious error, calling any installed error handler. See
686/// ErrorHandling.h.
687LLVM_ATTRIBUTE_NORETURN__attribute__((noreturn)) void report_fatal_error(Error Err,
688 bool gen_crash_diag = true);
689
690/// Report a fatal error if Err is a failure value.
691///
692/// This function can be used to wrap calls to fallible functions ONLY when it
693/// is known that the Error will always be a success value. E.g.
694///
695/// @code{.cpp}
696/// // foo only attempts the fallible operation if DoFallibleOperation is
697/// // true. If DoFallibleOperation is false then foo always returns
698/// // Error::success().
699/// Error foo(bool DoFallibleOperation);
700///
701/// cantFail(foo(false));
702/// @endcode
703inline void cantFail(Error Err, const char *Msg = nullptr) {
704 if (Err) {
705 if (!Msg)
706 Msg = "Failure value returned from cantFail wrapped call";
707 llvm_unreachable(Msg)::llvm::llvm_unreachable_internal(Msg, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 707);
708 }
709}
710
711/// Report a fatal error if ValOrErr is a failure value, otherwise unwraps and
712/// returns the contained value.
713///
714/// This function can be used to wrap calls to fallible functions ONLY when it
715/// is known that the Error will always be a success value. E.g.
716///
717/// @code{.cpp}
718/// // foo only attempts the fallible operation if DoFallibleOperation is
719/// // true. If DoFallibleOperation is false then foo always returns an int.
720/// Expected<int> foo(bool DoFallibleOperation);
721///
722/// int X = cantFail(foo(false));
723/// @endcode
724template <typename T>
725T cantFail(Expected<T> ValOrErr, const char *Msg = nullptr) {
726 if (ValOrErr)
727 return std::move(*ValOrErr);
728 else {
729 if (!Msg)
730 Msg = "Failure value returned from cantFail wrapped call";
731 llvm_unreachable(Msg)::llvm::llvm_unreachable_internal(Msg, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 731);
732 }
733}
734
735/// Report a fatal error if ValOrErr is a failure value, otherwise unwraps and
736/// returns the contained reference.
737///
738/// This function can be used to wrap calls to fallible functions ONLY when it
739/// is known that the Error will always be a success value. E.g.
740///
741/// @code{.cpp}
742/// // foo only attempts the fallible operation if DoFallibleOperation is
743/// // true. If DoFallibleOperation is false then foo always returns a Bar&.
744/// Expected<Bar&> foo(bool DoFallibleOperation);
745///
746/// Bar &X = cantFail(foo(false));
747/// @endcode
748template <typename T>
749T& cantFail(Expected<T&> ValOrErr, const char *Msg = nullptr) {
750 if (ValOrErr)
751 return *ValOrErr;
752 else {
753 if (!Msg)
754 Msg = "Failure value returned from cantFail wrapped call";
755 llvm_unreachable(Msg)::llvm::llvm_unreachable_internal(Msg, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 755);
756 }
757}
758
759/// Helper for testing applicability of, and applying, handlers for
760/// ErrorInfo types.
761template <typename HandlerT>
762class ErrorHandlerTraits
763 : public ErrorHandlerTraits<decltype(
764 &std::remove_reference<HandlerT>::type::operator())> {};
765
766// Specialization functions of the form 'Error (const ErrT&)'.
767template <typename ErrT> class ErrorHandlerTraits<Error (&)(ErrT &)> {
768public:
769 static bool appliesTo(const ErrorInfoBase &E) {
770 return E.template isA<ErrT>();
771 }
772
773 template <typename HandlerT>
774 static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
775 assert(appliesTo(*E) && "Applying incorrect handler")((appliesTo(*E) && "Applying incorrect handler") ? static_cast
<void> (0) : __assert_fail ("appliesTo(*E) && \"Applying incorrect handler\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 775, __PRETTY_FUNCTION__));
776 return H(static_cast<ErrT &>(*E));
777 }
778};
779
780// Specialization functions of the form 'void (const ErrT&)'.
781template <typename ErrT> class ErrorHandlerTraits<void (&)(ErrT &)> {
782public:
783 static bool appliesTo(const ErrorInfoBase &E) {
784 return E.template isA<ErrT>();
785 }
786
787 template <typename HandlerT>
788 static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
789 assert(appliesTo(*E) && "Applying incorrect handler")((appliesTo(*E) && "Applying incorrect handler") ? static_cast
<void> (0) : __assert_fail ("appliesTo(*E) && \"Applying incorrect handler\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 789, __PRETTY_FUNCTION__));
790 H(static_cast<ErrT &>(*E));
791 return Error::success();
792 }
793};
794
795/// Specialization for functions of the form 'Error (std::unique_ptr<ErrT>)'.
796template <typename ErrT>
797class ErrorHandlerTraits<Error (&)(std::unique_ptr<ErrT>)> {
798public:
799 static bool appliesTo(const ErrorInfoBase &E) {
800 return E.template isA<ErrT>();
801 }
802
803 template <typename HandlerT>
804 static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
805 assert(appliesTo(*E) && "Applying incorrect handler")((appliesTo(*E) && "Applying incorrect handler") ? static_cast
<void> (0) : __assert_fail ("appliesTo(*E) && \"Applying incorrect handler\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 805, __PRETTY_FUNCTION__));
806 std::unique_ptr<ErrT> SubE(static_cast<ErrT *>(E.release()));
807 return H(std::move(SubE));
808 }
809};
810
811/// Specialization for functions of the form 'void (std::unique_ptr<ErrT>)'.
812template <typename ErrT>
813class ErrorHandlerTraits<void (&)(std::unique_ptr<ErrT>)> {
814public:
815 static bool appliesTo(const ErrorInfoBase &E) {
816 return E.template isA<ErrT>();
817 }
818
819 template <typename HandlerT>
820 static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
821 assert(appliesTo(*E) && "Applying incorrect handler")((appliesTo(*E) && "Applying incorrect handler") ? static_cast
<void> (0) : __assert_fail ("appliesTo(*E) && \"Applying incorrect handler\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 821, __PRETTY_FUNCTION__));
822 std::unique_ptr<ErrT> SubE(static_cast<ErrT *>(E.release()));
823 H(std::move(SubE));
824 return Error::success();
825 }
826};
827
828// Specialization for member functions of the form 'RetT (const ErrT&)'.
829template <typename C, typename RetT, typename ErrT>
830class ErrorHandlerTraits<RetT (C::*)(ErrT &)>
831 : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
832
833// Specialization for member functions of the form 'RetT (const ErrT&) const'.
834template <typename C, typename RetT, typename ErrT>
835class ErrorHandlerTraits<RetT (C::*)(ErrT &) const>
836 : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
837
838// Specialization for member functions of the form 'RetT (const ErrT&)'.
839template <typename C, typename RetT, typename ErrT>
840class ErrorHandlerTraits<RetT (C::*)(const ErrT &)>
841 : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
842
843// Specialization for member functions of the form 'RetT (const ErrT&) const'.
844template <typename C, typename RetT, typename ErrT>
845class ErrorHandlerTraits<RetT (C::*)(const ErrT &) const>
846 : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
847
848/// Specialization for member functions of the form
849/// 'RetT (std::unique_ptr<ErrT>)'.
850template <typename C, typename RetT, typename ErrT>
851class ErrorHandlerTraits<RetT (C::*)(std::unique_ptr<ErrT>)>
852 : public ErrorHandlerTraits<RetT (&)(std::unique_ptr<ErrT>)> {};
853
854/// Specialization for member functions of the form
855/// 'RetT (std::unique_ptr<ErrT>) const'.
856template <typename C, typename RetT, typename ErrT>
857class ErrorHandlerTraits<RetT (C::*)(std::unique_ptr<ErrT>) const>
858 : public ErrorHandlerTraits<RetT (&)(std::unique_ptr<ErrT>)> {};
859
860inline Error handleErrorImpl(std::unique_ptr<ErrorInfoBase> Payload) {
861 return Error(std::move(Payload));
862}
863
864template <typename HandlerT, typename... HandlerTs>
865Error handleErrorImpl(std::unique_ptr<ErrorInfoBase> Payload,
866 HandlerT &&Handler, HandlerTs &&... Handlers) {
867 if (ErrorHandlerTraits<HandlerT>::appliesTo(*Payload))
868 return ErrorHandlerTraits<HandlerT>::apply(std::forward<HandlerT>(Handler),
869 std::move(Payload));
870 return handleErrorImpl(std::move(Payload),
871 std::forward<HandlerTs>(Handlers)...);
872}
873
874/// Pass the ErrorInfo(s) contained in E to their respective handlers. Any
875/// unhandled errors (or Errors returned by handlers) are re-concatenated and
876/// returned.
877/// Because this function returns an error, its result must also be checked
878/// or returned. If you intend to handle all errors use handleAllErrors
879/// (which returns void, and will abort() on unhandled errors) instead.
880template <typename... HandlerTs>
881Error handleErrors(Error E, HandlerTs &&... Hs) {
882 if (!E)
18
←
Assuming the condition is false
→
19
←
Taking false branch
→
883 return Error::success();
884
885 std::unique_ptr<ErrorInfoBase> Payload = E.takePayload();
886
887 if (Payload->isA<ErrorList>()) {
20
←
Assuming the condition is true
→
21
←
Taking true branch
→
888 ErrorList &List = static_cast<ErrorList &>(*Payload);
889 Error R;
890 for (auto &P : List.Payloads)
891 R = ErrorList::join(
22
←
Calling 'ErrorList::join'
→
892 std::move(R),
893 handleErrorImpl(std::move(P), std::forward<HandlerTs>(Hs)...));
894 return R;
895 }
896
897 return handleErrorImpl(std::move(Payload), std::forward<HandlerTs>(Hs)...);
898}
899
900/// Behaves the same as handleErrors, except that by contract all errors
901/// *must* be handled by the given handlers (i.e. there must be no remaining
902/// errors after running the handlers, or llvm_unreachable is called).
903template <typename... HandlerTs>
904void handleAllErrors(Error E, HandlerTs &&... Handlers) {
905 cantFail(handleErrors(std::move(E), std::forward<HandlerTs>(Handlers)...));
17
←
Calling 'handleErrors<(lambda at /build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Instrumentation/PGOInstrumentation.cpp:1116:35)>'
→
906}
907
908/// Check that E is a non-error, then drop it.
909/// If E is an error, llvm_unreachable will be called.
910inline void handleAllErrors(Error E) {
911 cantFail(std::move(E));
912}
913
914/// Handle any errors (if present) in an Expected<T>, then try a recovery path.
915///
916/// If the incoming value is a success value it is returned unmodified. If it
917/// is a failure value then it the contained error is passed to handleErrors.
918/// If handleErrors is able to handle the error then the RecoveryPath functor
919/// is called to supply the final result. If handleErrors is not able to
920/// handle all errors then the unhandled errors are returned.
921///
922/// This utility enables the follow pattern:
923///
924/// @code{.cpp}
925/// enum FooStrategy { Aggressive, Conservative };
926/// Expected<Foo> foo(FooStrategy S);
927///
928/// auto ResultOrErr =
929/// handleExpected(
930/// foo(Aggressive),
931/// []() { return foo(Conservative); },
932/// [](AggressiveStrategyError&) {
933/// // Implicitly conusme this - we'll recover by using a conservative
934/// // strategy.
935/// });
936///
937/// @endcode
938template <typename T, typename RecoveryFtor, typename... HandlerTs>
939Expected<T> handleExpected(Expected<T> ValOrErr, RecoveryFtor &&RecoveryPath,
940 HandlerTs &&... Handlers) {
941 if (ValOrErr)
942 return ValOrErr;
943
944 if (auto Err = handleErrors(ValOrErr.takeError(),
945 std::forward<HandlerTs>(Handlers)...))
946 return std::move(Err);
947
948 return RecoveryPath();
949}
950
951/// Log all errors (if any) in E to OS. If there are any errors, ErrorBanner
952/// will be printed before the first one is logged. A newline will be printed
953/// after each error.
954///
955/// This function is compatible with the helpers from Support/WithColor.h. You
956/// can pass any of them as the OS. Please consider using them instead of
957/// including 'error: ' in the ErrorBanner.
958///
959/// This is useful in the base level of your program to allow clean termination
960/// (allowing clean deallocation of resources, etc.), while reporting error
961/// information to the user.
962void logAllUnhandledErrors(Error E, raw_ostream &OS, Twine ErrorBanner = {});
963
964/// Write all error messages (if any) in E to a string. The newline character
965/// is used to separate error messages.
966inline std::string toString(Error E) {
967 SmallVector<std::string, 2> Errors;
968 handleAllErrors(std::move(E), [&Errors](const ErrorInfoBase &EI) {
969 Errors.push_back(EI.message());
970 });
971 return join(Errors.begin(), Errors.end(), "\n");
972}
973
974/// Consume a Error without doing anything. This method should be used
975/// only where an error can be considered a reasonable and expected return
976/// value.
977///
978/// Uses of this method are potentially indicative of design problems: If it's
979/// legitimate to do nothing while processing an "error", the error-producer
980/// might be more clearly refactored to return an Optional<T>.
981inline void consumeError(Error Err) {
982 handleAllErrors(std::move(Err), [](const ErrorInfoBase &) {});
983}
984
985/// Helper for converting an Error to a bool.
986///
987/// This method returns true if Err is in an error state, or false if it is
988/// in a success state. Puts Err in a checked state in both cases (unlike
989/// Error::operator bool(), which only does this for success states).
990inline bool errorToBool(Error Err) {
991 bool IsError = static_cast<bool>(Err);
992 if (IsError)
993 consumeError(std::move(Err));
994 return IsError;
995}
996
997/// Helper for Errors used as out-parameters.
998///
999/// This helper is for use with the Error-as-out-parameter idiom, where an error
1000/// is passed to a function or method by reference, rather than being returned.
1001/// In such cases it is helpful to set the checked bit on entry to the function
1002/// so that the error can be written to (unchecked Errors abort on assignment)
1003/// and clear the checked bit on exit so that clients cannot accidentally forget
1004/// to check the result. This helper performs these actions automatically using
1005/// RAII:
1006///
1007/// @code{.cpp}
1008/// Result foo(Error &Err) {
1009/// ErrorAsOutParameter ErrAsOutParam(&Err); // 'Checked' flag set
1010/// // <body of foo>
1011/// // <- 'Checked' flag auto-cleared when ErrAsOutParam is destructed.
1012/// }
1013/// @endcode
1014///
1015/// ErrorAsOutParameter takes an Error* rather than Error& so that it can be
1016/// used with optional Errors (Error pointers that are allowed to be null). If
1017/// ErrorAsOutParameter took an Error reference, an instance would have to be
1018/// created inside every condition that verified that Error was non-null. By
1019/// taking an Error pointer we can just create one instance at the top of the
1020/// function.
1021class ErrorAsOutParameter {
1022public:
1023 ErrorAsOutParameter(Error *Err) : Err(Err) {
1024 // Raise the checked bit if Err is success.
1025 if (Err)
1026 (void)!!*Err;
1027 }
1028
1029 ~ErrorAsOutParameter() {
1030 // Clear the checked bit.
1031 if (Err && !*Err)
1032 *Err = Error::success();
1033 }
1034
1035private:
1036 Error *Err;
1037};
1038
1039/// Helper for Expected<T>s used as out-parameters.
1040///
1041/// See ErrorAsOutParameter.
1042template <typename T>
1043class ExpectedAsOutParameter {
1044public:
1045 ExpectedAsOutParameter(Expected<T> *ValOrErr)
1046 : ValOrErr(ValOrErr) {
1047 if (ValOrErr)
1048 (void)!!*ValOrErr;
1049 }
1050
1051 ~ExpectedAsOutParameter() {
1052 if (ValOrErr)
1053 ValOrErr->setUnchecked();
1054 }
1055
1056private:
1057 Expected<T> *ValOrErr;
1058};
1059
1060/// This class wraps a std::error_code in a Error.
1061///
1062/// This is useful if you're writing an interface that returns a Error
1063/// (or Expected) and you want to call code that still returns
1064/// std::error_codes.
1065class ECError : public ErrorInfo<ECError> {
1066 friend Error errorCodeToError(std::error_code);
1067
1068 virtual void anchor() override;
1069
1070public:
1071 void setErrorCode(std::error_code EC) { this->EC = EC; }
1072 std::error_code convertToErrorCode() const override { return EC; }
1073 void log(raw_ostream &OS) const override { OS << EC.message(); }
1074
1075 // Used by ErrorInfo::classID.
1076 static char ID;
1077
1078protected:
1079 ECError() = default;
1080 ECError(std::error_code EC) : EC(EC) {}
1081
1082 std::error_code EC;
1083};
1084
1085/// The value returned by this function can be returned from convertToErrorCode
1086/// for Error values where no sensible translation to std::error_code exists.
1087/// It should only be used in this situation, and should never be used where a
1088/// sensible conversion to std::error_code is available, as attempts to convert
1089/// to/from this error will result in a fatal error. (i.e. it is a programmatic
1090///error to try to convert such a value).
1091std::error_code inconvertibleErrorCode();
1092
1093/// Helper for converting an std::error_code to a Error.
1094Error errorCodeToError(std::error_code EC);
1095
1096/// Helper for converting an ECError to a std::error_code.
1097///
1098/// This method requires that Err be Error() or an ECError, otherwise it
1099/// will trigger a call to abort().
1100std::error_code errorToErrorCode(Error Err);
1101
1102/// Convert an ErrorOr<T> to an Expected<T>.
1103template <typename T> Expected<T> errorOrToExpected(ErrorOr<T> &&EO) {
1104 if (auto EC = EO.getError())
1105 return errorCodeToError(EC);
1106 return std::move(*EO);
1107}
1108
1109/// Convert an Expected<T> to an ErrorOr<T>.
1110template <typename T> ErrorOr<T> expectedToErrorOr(Expected<T> &&E) {
1111 if (auto Err = E.takeError())
1112 return errorToErrorCode(std::move(Err));
1113 return std::move(*E);
1114}
1115
1116/// This class wraps a string in an Error.
1117///
1118/// StringError is useful in cases where the client is not expected to be able
1119/// to consume the specific error message programmatically (for example, if the
1120/// error message is to be presented to the user).
1121///
1122/// StringError can also be used when additional information is to be printed
1123/// along with a error_code message. Depending on the constructor called, this
1124/// class can either display:
1125/// 1. the error_code message (ECError behavior)
1126/// 2. a string
1127/// 3. the error_code message and a string
1128///
1129/// These behaviors are useful when subtyping is required; for example, when a
1130/// specific library needs an explicit error type. In the example below,
1131/// PDBError is derived from StringError:
1132///
1133/// @code{.cpp}
1134/// Expected<int> foo() {
1135/// return llvm::make_error<PDBError>(pdb_error_code::dia_failed_loading,
1136/// "Additional information");
1137/// }
1138/// @endcode
1139///
1140class StringError : public ErrorInfo<StringError> {
1141public:
1142 static char ID;
1143
1144 // Prints EC + S and converts to EC
1145 StringError(std::error_code EC, const Twine &S = Twine());
1146
1147 // Prints S and converts to EC
1148 StringError(const Twine &S, std::error_code EC);
1149
1150 void log(raw_ostream &OS) const override;
1151 std::error_code convertToErrorCode() const override;
1152
1153 const std::string &getMessage() const { return Msg; }
1154
1155private:
1156 std::string Msg;
1157 std::error_code EC;
1158 const bool PrintMsgOnly = false;
1159};
1160
1161/// Create formatted StringError object.
1162template <typename... Ts>
1163Error createStringError(std::error_code EC, char const *Fmt,
1164 const Ts &... Vals) {
1165 std::string Buffer;
1166 raw_string_ostream Stream(Buffer);
1167 Stream << format(Fmt, Vals...);
1168 return make_error<StringError>(Stream.str(), EC);
1169}
1170
1171Error createStringError(std::error_code EC, char const *Msg);
1172
1173/// This class wraps a filename and another Error.
1174///
1175/// In some cases, an error needs to live along a 'source' name, in order to
1176/// show more detailed information to the user.
1177class FileError final : public ErrorInfo<FileError> {
1178
1179 friend Error createFileError(const Twine &, Error);
1180 friend Error createFileError(const Twine &, size_t, Error);
1181
1182public:
1183 void log(raw_ostream &OS) const override {
1184 assert(Err && !FileName.empty() && "Trying to log after takeError().")((Err && !FileName.empty() && "Trying to log after takeError()."
) ? static_cast<void> (0) : __assert_fail ("Err && !FileName.empty() && \"Trying to log after takeError().\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 1184, __PRETTY_FUNCTION__));
1185 OS << "'" << FileName << "': ";
1186 if (Line.hasValue())
1187 OS << "line " << Line.getValue() << ": ";
1188 Err->log(OS);
1189 }
1190
1191 Error takeError() { return Error(std::move(Err)); }
1192
1193 std::error_code convertToErrorCode() const override;
1194
1195 // Used by ErrorInfo::classID.
1196 static char ID;
1197
1198private:
1199 FileError(const Twine &F, Optional<size_t> LineNum,
1200 std::unique_ptr<ErrorInfoBase> E) {
1201 assert(E && "Cannot create FileError from Error success value.")((E && "Cannot create FileError from Error success value."
) ? static_cast<void> (0) : __assert_fail ("E && \"Cannot create FileError from Error success value.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 1201, __PRETTY_FUNCTION__));
1202 assert(!F.isTriviallyEmpty() &&((!F.isTriviallyEmpty() && "The file name provided to FileError must not be empty."
) ? static_cast<void> (0) : __assert_fail ("!F.isTriviallyEmpty() && \"The file name provided to FileError must not be empty.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 1203, __PRETTY_FUNCTION__))
1203 "The file name provided to FileError must not be empty.")((!F.isTriviallyEmpty() && "The file name provided to FileError must not be empty."
) ? static_cast<void> (0) : __assert_fail ("!F.isTriviallyEmpty() && \"The file name provided to FileError must not be empty.\""
, "/build/llvm-toolchain-snapshot-9~svn362543/include/llvm/Support/Error.h"
, 1203, __PRETTY_FUNCTION__));
1204 FileName = F.str();
1205 Err = std::move(E);
1206 Line = std::move(LineNum);
1207 }
1208
1209 static Error build(const Twine &F, Optional<size_t> Line, Error E) {
1210 return Error(
1211 std::unique_ptr<FileError>(new FileError(F, Line, E.takePayload())));
1212 }
1213
1214 std::string FileName;
1215 Optional<size_t> Line;
1216 std::unique_ptr<ErrorInfoBase> Err;
1217};
1218
1219/// Concatenate a source file path and/or name with an Error. The resulting
1220/// Error is unchecked.
1221inline Error createFileError(const Twine &F, Error E) {
1222 return FileError::build(F, Optional<size_t>(), std::move(E));
1223}
1224
1225/// Concatenate a source file path and/or name with line number and an Error.
1226/// The resulting Error is unchecked.
1227inline Error createFileError(const Twine &F, size_t Line, Error E) {
1228 return FileError::build(F, Optional<size_t>(Line), std::move(E));
1229}
1230
1231/// Concatenate a source file path and/or name with a std::error_code
1232/// to form an Error object.
1233inline Error createFileError(const Twine &F, std::error_code EC) {
1234 return createFileError(F, errorCodeToError(EC));
1235}
1236
1237/// Concatenate a source file path and/or name with line number and
1238/// std::error_code to form an Error object.
1239inline Error createFileError(const Twine &F, size_t Line, std::error_code EC) {
1240 return createFileError(F, Line, errorCodeToError(EC));
1241}
1242
1243Error createFileError(const Twine &F, ErrorSuccess) = delete;
1244
1245/// Helper for check-and-exit error handling.
1246///
1247/// For tool use only. NOT FOR USE IN LIBRARY CODE.
1248///
1249class ExitOnError {
1250public:
1251 /// Create an error on exit helper.
1252 ExitOnError(std::string Banner = "", int DefaultErrorExitCode = 1)
1253 : Banner(std::move(Banner)),
1254 GetExitCode([=](const Error &) { return DefaultErrorExitCode; }) {}
1255
1256 /// Set the banner string for any errors caught by operator().
1257 void setBanner(std::string Banner) { this->Banner = std::move(Banner); }
1258
1259 /// Set the exit-code mapper function.
1260 void setExitCodeMapper(std::function<int(const Error &)> GetExitCode) {
1261 this->GetExitCode = std::move(GetExitCode);
1262 }
1263
1264 /// Check Err. If it's in a failure state log the error(s) and exit.
1265 void operator()(Error Err) const { checkError(std::move(Err)); }
1266
1267 /// Check E. If it's in a success state then return the contained value. If
1268 /// it's in a failure state log the error(s) and exit.
1269 template <typename T> T operator()(Expected<T> &&E) const {
1270 checkError(E.takeError());
1271 return std::move(*E);
1272 }
1273
1274 /// Check E. If it's in a success state then return the contained reference. If
1275 /// it's in a failure state log the error(s) and exit.
1276 template <typename T> T& operator()(Expected<T&> &&E) const {
1277 checkError(E.takeError());
1278 return *E;
1279 }
1280
1281private:
1282 void checkError(Error Err) const {
1283 if (Err) {
1284 int ExitCode = GetExitCode(Err);
1285 logAllUnhandledErrors(std::move(Err), errs(), Banner);
1286 exit(ExitCode);
1287 }
1288 }
1289
1290 std::string Banner;
1291 std::function<int(const Error &)> GetExitCode;
1292};
1293
1294/// Conversion from Error to LLVMErrorRef for C error bindings.
1295inline LLVMErrorRef wrap(Error Err) {
1296 return reinterpret_cast<LLVMErrorRef>(Err.takePayload().release());
1297}
1298
1299/// Conversion from LLVMErrorRef to Error for C error bindings.
1300inline Error unwrap(LLVMErrorRef ErrRef) {
1301 return Error(std::unique_ptr<ErrorInfoBase>(
1302 reinterpret_cast<ErrorInfoBase *>(ErrRef)));
1303}
1304
1305} // end namespace llvm
1306
1307#endif // LLVM_SUPPORT_ERROR_H