LLVM  10.0.0svn
InstrProfiling.cpp
Go to the documentation of this file.
1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10 // It also builds the data structures and initialization code needed for
11 // updating execution counts and emitting the profile at runtime.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/ADT/Twine.h"
23 #include "llvm/Analysis/LoopInfo.h"
25 #include "llvm/IR/Attributes.h"
26 #include "llvm/IR/BasicBlock.h"
27 #include "llvm/IR/Constant.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DerivedTypes.h"
30 #include "llvm/IR/Dominators.h"
31 #include "llvm/IR/Function.h"
32 #include "llvm/IR/GlobalValue.h"
33 #include "llvm/IR/GlobalVariable.h"
34 #include "llvm/IR/IRBuilder.h"
35 #include "llvm/IR/Instruction.h"
36 #include "llvm/IR/Instructions.h"
37 #include "llvm/IR/IntrinsicInst.h"
38 #include "llvm/IR/Module.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/Pass.h"
42 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/Error.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstddef>
52 #include <cstdint>
53 #include <string>
54 
55 using namespace llvm;
56 
57 #define DEBUG_TYPE "instrprof"
58 
59 // The start and end values of precise value profile range for memory
60 // intrinsic sizes
62  "memop-size-range",
63  cl::desc("Set the range of size in memory intrinsic calls to be profiled "
64  "precisely, in a format of <start_val>:<end_val>"),
65  cl::init(""));
66 
67 // The value that considered to be large value in memory intrinsic.
69  "memop-size-large",
70  cl::desc("Set large value thresthold in memory intrinsic size profiling. "
71  "Value of 0 disables the large value profiling."),
72  cl::init(8192));
73 
74 namespace {
75 
76 cl::opt<bool> DoNameCompression("enable-name-compression",
77  cl::desc("Enable name string compression"),
78  cl::init(true));
79 
80 cl::opt<bool> DoHashBasedCounterSplit(
81  "hash-based-counter-split",
82  cl::desc("Rename counter variable of a comdat function based on cfg hash"),
83  cl::init(true));
84 
85 cl::opt<bool> ValueProfileStaticAlloc(
86  "vp-static-alloc",
87  cl::desc("Do static counter allocation for value profiler"),
88  cl::init(true));
89 
90 cl::opt<double> NumCountersPerValueSite(
91  "vp-counters-per-site",
92  cl::desc("The average number of profile counters allocated "
93  "per value profiling site."),
94  // This is set to a very small value because in real programs, only
95  // a very small percentage of value sites have non-zero targets, e.g, 1/30.
96  // For those sites with non-zero profile, the average number of targets
97  // is usually smaller than 2.
98  cl::init(1.0));
99 
100 cl::opt<bool> AtomicCounterUpdateAll(
101  "instrprof-atomic-counter-update-all", cl::ZeroOrMore,
102  cl::desc("Make all profile counter updates atomic (for testing only)"),
103  cl::init(false));
104 
105 cl::opt<bool> AtomicCounterUpdatePromoted(
106  "atomic-counter-update-promoted", cl::ZeroOrMore,
107  cl::desc("Do counter update using atomic fetch add "
108  " for promoted counters only"),
109  cl::init(false));
110 
111 // If the option is not specified, the default behavior about whether
112 // counter promotion is done depends on how instrumentaiton lowering
113 // pipeline is setup, i.e., the default value of true of this option
114 // does not mean the promotion will be done by default. Explicitly
115 // setting this option can override the default behavior.
116 cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
117  cl::desc("Do counter register promotion"),
118  cl::init(false));
119 cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
120  cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20),
121  cl::desc("Max number counter promotions per loop to avoid"
122  " increasing register pressure too much"));
123 
124 // A debug option
126  MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
127  cl::desc("Max number of allowed counter promotions"));
128 
129 cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
130  cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3),
131  cl::desc("The max number of exiting blocks of a loop to allow "
132  " speculative counter promotion"));
133 
134 cl::opt<bool> SpeculativeCounterPromotionToLoop(
135  cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false),
136  cl::desc("When the option is false, if the target block is in a loop, "
137  "the promotion will be disallowed unless the promoted counter "
138  " update can be further/iteratively promoted into an acyclic "
139  " region."));
140 
141 cl::opt<bool> IterativeCounterPromotion(
142  cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
143  cl::desc("Allow counter promotion across the whole loop nest."));
144 
145 class InstrProfilingLegacyPass : public ModulePass {
146  InstrProfiling InstrProf;
147 
148 public:
149  static char ID;
150 
151  InstrProfilingLegacyPass() : ModulePass(ID) {}
152  InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false)
153  : ModulePass(ID), InstrProf(Options, IsCS) {}
154 
155  StringRef getPassName() const override {
156  return "Frontend instrumentation-based coverage lowering";
157  }
158 
159  bool runOnModule(Module &M) override {
160  auto GetTLI = [this](Function &F) -> TargetLibraryInfo & {
161  return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
162  };
163  return InstrProf.run(M, GetTLI);
164  }
165 
166  void getAnalysisUsage(AnalysisUsage &AU) const override {
167  AU.setPreservesCFG();
169  }
170 };
171 
172 ///
173 /// A helper class to promote one counter RMW operation in the loop
174 /// into register update.
175 ///
176 /// RWM update for the counter will be sinked out of the loop after
177 /// the transformation.
178 ///
179 class PGOCounterPromoterHelper : public LoadAndStorePromoter {
180 public:
181  PGOCounterPromoterHelper(
183  BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
184  ArrayRef<Instruction *> InsertPts,
186  LoopInfo &LI)
187  : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
188  InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
189  assert(isa<LoadInst>(L));
190  assert(isa<StoreInst>(S));
191  SSA.AddAvailableValue(PH, Init);
192  }
193 
194  void doExtraRewritesBeforeFinalDeletion() override {
195  for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
196  BasicBlock *ExitBlock = ExitBlocks[i];
197  Instruction *InsertPos = InsertPts[i];
198  // Get LiveIn value into the ExitBlock. If there are multiple
199  // predecessors, the value is defined by a PHI node in this
200  // block.
201  Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
202  Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
203  Type *Ty = LiveInValue->getType();
204  IRBuilder<> Builder(InsertPos);
205  if (AtomicCounterUpdatePromoted)
206  // automic update currently can only be promoted across the current
207  // loop, not the whole loop nest.
208  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
210  else {
211  LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
212  auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
213  auto *NewStore = Builder.CreateStore(NewVal, Addr);
214 
215  // Now update the parent loop's candidate list:
216  if (IterativeCounterPromotion) {
217  auto *TargetLoop = LI.getLoopFor(ExitBlock);
218  if (TargetLoop)
219  LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
220  }
221  }
222  }
223  }
224 
225 private:
227  ArrayRef<BasicBlock *> ExitBlocks;
228  ArrayRef<Instruction *> InsertPts;
230  LoopInfo &LI;
231 };
232 
233 /// A helper class to do register promotion for all profile counter
234 /// updates in a loop.
235 ///
236 class PGOCounterPromoter {
237 public:
238  PGOCounterPromoter(
240  Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
241  : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop),
242  LI(LI), BFI(BFI) {
243 
244  SmallVector<BasicBlock *, 8> LoopExitBlocks;
246  L.getExitBlocks(LoopExitBlocks);
247 
248  for (BasicBlock *ExitBlock : LoopExitBlocks) {
249  if (BlockSet.insert(ExitBlock).second) {
250  ExitBlocks.push_back(ExitBlock);
251  InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
252  }
253  }
254  }
255 
256  bool run(int64_t *NumPromoted) {
257  // Skip 'infinite' loops:
258  if (ExitBlocks.size() == 0)
259  return false;
260  unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
261  if (MaxProm == 0)
262  return false;
263 
264  unsigned Promoted = 0;
265  for (auto &Cand : LoopToCandidates[&L]) {
266 
268  SSAUpdater SSA(&NewPHIs);
269  Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
270 
271  // If BFI is set, we will use it to guide the promotions.
272  if (BFI) {
273  auto *BB = Cand.first->getParent();
274  auto InstrCount = BFI->getBlockProfileCount(BB);
275  if (!InstrCount)
276  continue;
277  auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
278  // If the average loop trip count is not greater than 1.5, we skip
279  // promotion.
280  if (PreheaderCount &&
281  (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2))
282  continue;
283  }
284 
285  PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
286  L.getLoopPreheader(), ExitBlocks,
287  InsertPts, LoopToCandidates, LI);
288  Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
289  Promoted++;
290  if (Promoted >= MaxProm)
291  break;
292 
293  (*NumPromoted)++;
294  if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
295  break;
296  }
297 
298  LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
299  << L.getLoopDepth() << ")\n");
300  return Promoted != 0;
301  }
302 
303 private:
304  bool allowSpeculativeCounterPromotion(Loop *LP) {
305  SmallVector<BasicBlock *, 8> ExitingBlocks;
306  L.getExitingBlocks(ExitingBlocks);
307  // Not considierered speculative.
308  if (ExitingBlocks.size() == 1)
309  return true;
310  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
311  return false;
312  return true;
313  }
314 
315  // Returns the max number of Counter Promotions for LP.
316  unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
317  // We can't insert into a catchswitch.
318  SmallVector<BasicBlock *, 8> LoopExitBlocks;
319  LP->getExitBlocks(LoopExitBlocks);
320  if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
321  return isa<CatchSwitchInst>(Exit->getTerminator());
322  }))
323  return 0;
324 
325  if (!LP->hasDedicatedExits())
326  return 0;
327 
328  BasicBlock *PH = LP->getLoopPreheader();
329  if (!PH)
330  return 0;
331 
332  SmallVector<BasicBlock *, 8> ExitingBlocks;
333  LP->getExitingBlocks(ExitingBlocks);
334 
335  // If BFI is set, we do more aggressive promotions based on BFI.
336  if (BFI)
337  return (unsigned)-1;
338 
339  // Not considierered speculative.
340  if (ExitingBlocks.size() == 1)
341  return MaxNumOfPromotionsPerLoop;
342 
343  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
344  return 0;
345 
346  // Whether the target block is in a loop does not matter:
347  if (SpeculativeCounterPromotionToLoop)
348  return MaxNumOfPromotionsPerLoop;
349 
350  // Now check the target block:
351  unsigned MaxProm = MaxNumOfPromotionsPerLoop;
352  for (auto *TargetBlock : LoopExitBlocks) {
353  auto *TargetLoop = LI.getLoopFor(TargetBlock);
354  if (!TargetLoop)
355  continue;
356  unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
357  unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
358  MaxProm =
359  std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
360  PendingCandsInTarget);
361  }
362  return MaxProm;
363  }
364 
366  SmallVector<BasicBlock *, 8> ExitBlocks;
368  Loop &L;
369  LoopInfo &LI;
371 };
372 
373 } // end anonymous namespace
374 
377  AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
378  auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
379  return FAM.getResult<TargetLibraryAnalysis>(F);
380  };
381  if (!run(M, GetTLI))
382  return PreservedAnalyses::all();
383 
384  return PreservedAnalyses::none();
385 }
386 
389  InstrProfilingLegacyPass, "instrprof",
390  "Frontend instrumentation-based coverage lowering.", false, false)
393  InstrProfilingLegacyPass, "instrprof",
394  "Frontend instrumentation-based coverage lowering.", false, false)
395 
396 ModulePass *
398  bool IsCS) {
399  return new InstrProfilingLegacyPass(Options, IsCS);
400 }
401 
404  if (Inc)
405  return Inc;
406  return dyn_cast<InstrProfIncrementInst>(Instr);
407 }
408 
409 bool InstrProfiling::lowerIntrinsics(Function *F) {
410  bool MadeChange = false;
411  PromotionCandidates.clear();
412  for (BasicBlock &BB : *F) {
413  for (auto I = BB.begin(), E = BB.end(); I != E;) {
414  auto Instr = I++;
416  if (Inc) {
417  lowerIncrement(Inc);
418  MadeChange = true;
419  } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
420  lowerValueProfileInst(Ind);
421  MadeChange = true;
422  }
423  }
424  }
425 
426  if (!MadeChange)
427  return false;
428 
429  promoteCounterLoadStores(F);
430  return true;
431 }
432 
433 bool InstrProfiling::isCounterPromotionEnabled() const {
434  if (DoCounterPromotion.getNumOccurrences() > 0)
435  return DoCounterPromotion;
436 
437  return Options.DoCounterPromotion;
438 }
439 
440 void InstrProfiling::promoteCounterLoadStores(Function *F) {
441  if (!isCounterPromotionEnabled())
442  return;
443 
444  DominatorTree DT(*F);
445  LoopInfo LI(DT);
446  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
447 
448  std::unique_ptr<BlockFrequencyInfo> BFI;
449  if (Options.UseBFIInPromotion) {
450  std::unique_ptr<BranchProbabilityInfo> BPI;
451  BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
452  BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
453  }
454 
455  for (const auto &LoadStore : PromotionCandidates) {
456  auto *CounterLoad = LoadStore.first;
457  auto *CounterStore = LoadStore.second;
458  BasicBlock *BB = CounterLoad->getParent();
459  Loop *ParentLoop = LI.getLoopFor(BB);
460  if (!ParentLoop)
461  continue;
462  LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
463  }
464 
466 
467  // Do a post-order traversal of the loops so that counter updates can be
468  // iteratively hoisted outside the loop nest.
469  for (auto *Loop : llvm::reverse(Loops)) {
470  PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
471  Promoter.run(&TotalCountersPromoted);
472  }
473 }
474 
475 /// Check if the module contains uses of any profiling intrinsics.
477  if (auto *F = M.getFunction(
478  Intrinsic::getName(llvm::Intrinsic::instrprof_increment)))
479  if (!F->use_empty())
480  return true;
481  if (auto *F = M.getFunction(
482  Intrinsic::getName(llvm::Intrinsic::instrprof_increment_step)))
483  if (!F->use_empty())
484  return true;
485  if (auto *F = M.getFunction(
486  Intrinsic::getName(llvm::Intrinsic::instrprof_value_profile)))
487  if (!F->use_empty())
488  return true;
489  return false;
490 }
491 
493  Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
494  this->M = &M;
495  this->GetTLI = std::move(GetTLI);
496  NamesVar = nullptr;
497  NamesSize = 0;
498  ProfileDataMap.clear();
499  UsedVars.clear();
500  getMemOPSizeRangeFromOption(MemOPSizeRange, MemOPSizeRangeStart,
501  MemOPSizeRangeLast);
502  TT = Triple(M.getTargetTriple());
503 
504  // Emit the runtime hook even if no counters are present.
505  bool MadeChange = emitRuntimeHook();
506 
507  // Improve compile time by avoiding linear scans when there is no work.
508  GlobalVariable *CoverageNamesVar =
510  if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
511  return MadeChange;
512 
513  // We did not know how many value sites there would be inside
514  // the instrumented function. This is counting the number of instrumented
515  // target value sites to enter it as field in the profile data variable.
516  for (Function &F : M) {
517  InstrProfIncrementInst *FirstProfIncInst = nullptr;
518  for (BasicBlock &BB : F)
519  for (auto I = BB.begin(), E = BB.end(); I != E; I++)
520  if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
521  computeNumValueSiteCounts(Ind);
522  else if (FirstProfIncInst == nullptr)
523  FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
524 
525  // Value profiling intrinsic lowering requires per-function profile data
526  // variable to be created first.
527  if (FirstProfIncInst != nullptr)
528  static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
529  }
530 
531  for (Function &F : M)
532  MadeChange |= lowerIntrinsics(&F);
533 
534  if (CoverageNamesVar) {
535  lowerCoverageData(CoverageNamesVar);
536  MadeChange = true;
537  }
538 
539  if (!MadeChange)
540  return false;
541 
542  emitVNodes();
543  emitNameData();
544  emitRegistration();
545  emitUses();
546  emitInitialization();
547  return true;
548 }
549 
550 static FunctionCallee
552  bool IsRange = false) {
553  LLVMContext &Ctx = M.getContext();
554  auto *ReturnTy = Type::getVoidTy(M.getContext());
555 
557  if (auto AK = TLI.getExtAttrForI32Param(false))
558  AL = AL.addParamAttribute(M.getContext(), 2, AK);
559 
560  if (!IsRange) {
561  Type *ParamTypes[] = {
562 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
564  };
565  auto *ValueProfilingCallTy =
566  FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
568  ValueProfilingCallTy, AL);
569  } else {
570  Type *RangeParamTypes[] = {
571 #define VALUE_RANGE_PROF 1
572 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
574 #undef VALUE_RANGE_PROF
575  };
576  auto *ValueRangeProfilingCallTy =
577  FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
579  ValueRangeProfilingCallTy, AL);
580  }
581 }
582 
583 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
584  GlobalVariable *Name = Ind->getName();
585  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
586  uint64_t Index = Ind->getIndex()->getZExtValue();
587  auto It = ProfileDataMap.find(Name);
588  if (It == ProfileDataMap.end()) {
589  PerFunctionProfileData PD;
590  PD.NumValueSites[ValueKind] = Index + 1;
591  ProfileDataMap[Name] = PD;
592  } else if (It->second.NumValueSites[ValueKind] <= Index)
593  It->second.NumValueSites[ValueKind] = Index + 1;
594 }
595 
596 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
597  GlobalVariable *Name = Ind->getName();
598  auto It = ProfileDataMap.find(Name);
599  assert(It != ProfileDataMap.end() && It->second.DataVar &&
600  "value profiling detected in function with no counter incerement");
601 
602  GlobalVariable *DataVar = It->second.DataVar;
603  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
604  uint64_t Index = Ind->getIndex()->getZExtValue();
605  for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
606  Index += It->second.NumValueSites[Kind];
607 
608  IRBuilder<> Builder(Ind);
609  bool IsRange = (Ind->getValueKind()->getZExtValue() ==
610  llvm::InstrProfValueKind::IPVK_MemOPSize);
611  CallInst *Call = nullptr;
612  auto *TLI = &GetTLI(*Ind->getFunction());
613  if (!IsRange) {
614  Value *Args[3] = {Ind->getTargetValue(),
615  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
616  Builder.getInt32(Index)};
617  Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args);
618  } else {
619  Value *Args[6] = {
620  Ind->getTargetValue(),
621  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
622  Builder.getInt32(Index),
623  Builder.getInt64(MemOPSizeRangeStart),
624  Builder.getInt64(MemOPSizeRangeLast),
625  Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
626  Call =
627  Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true), Args);
628  }
629  if (auto AK = TLI->getExtAttrForI32Param(false))
630  Call->addParamAttr(2, AK);
631  Ind->replaceAllUsesWith(Call);
632  Ind->eraseFromParent();
633 }
634 
635 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
636  GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
637 
638  IRBuilder<> Builder(Inc);
639  uint64_t Index = Inc->getIndex()->getZExtValue();
640  Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(),
641  Counters, 0, Index);
642 
643  if (Options.Atomic || AtomicCounterUpdateAll) {
644  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
646  } else {
647  Value *IncStep = Inc->getStep();
648  Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
649  auto *Count = Builder.CreateAdd(Load, Inc->getStep());
650  auto *Store = Builder.CreateStore(Count, Addr);
651  if (isCounterPromotionEnabled())
652  PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
653  }
654  Inc->eraseFromParent();
655 }
656 
657 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
658  ConstantArray *Names =
659  cast<ConstantArray>(CoverageNamesVar->getInitializer());
660  for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
661  Constant *NC = Names->getOperand(I);
662  Value *V = NC->stripPointerCasts();
663  assert(isa<GlobalVariable>(V) && "Missing reference to function name");
664  GlobalVariable *Name = cast<GlobalVariable>(V);
665 
667  ReferencedNames.push_back(Name);
668  NC->dropAllReferences();
669  }
670  CoverageNamesVar->eraseFromParent();
671 }
672 
673 /// Get the name of a profiling variable for a particular function.
675  StringRef NamePrefix = getInstrProfNameVarPrefix();
676  StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
677  Function *F = Inc->getParent()->getParent();
678  Module *M = F->getParent();
679  if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
680  !canRenameComdatFunc(*F))
681  return (Prefix + Name).str();
682  uint64_t FuncHash = Inc->getHash()->getZExtValue();
683  SmallVector<char, 24> HashPostfix;
684  if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
685  return (Prefix + Name).str();
686  return (Prefix + Name + "." + Twine(FuncHash)).str();
687 }
688 
689 static inline bool shouldRecordFunctionAddr(Function *F) {
690  // Check the linkage
691  bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
692  if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
693  !HasAvailableExternallyLinkage)
694  return true;
695 
696  // A function marked 'alwaysinline' with available_externally linkage can't
697  // have its address taken. Doing so would create an undefined external ref to
698  // the function, which would fail to link.
699  if (HasAvailableExternallyLinkage &&
700  F->hasFnAttribute(Attribute::AlwaysInline))
701  return false;
702 
703  // Prohibit function address recording if the function is both internal and
704  // COMDAT. This avoids the profile data variable referencing internal symbols
705  // in COMDAT.
706  if (F->hasLocalLinkage() && F->hasComdat())
707  return false;
708 
709  // Check uses of this function for other than direct calls or invokes to it.
710  // Inline virtual functions have linkeOnceODR linkage. When a key method
711  // exists, the vtable will only be emitted in the TU where the key method
712  // is defined. In a TU where vtable is not available, the function won't
713  // be 'addresstaken'. If its address is not recorded here, the profile data
714  // with missing address may be picked by the linker leading to missing
715  // indirect call target info.
716  return F->hasAddressTaken() || F->hasLinkOnceLinkage();
717 }
718 
720  // Don't do this for Darwin. compiler-rt uses linker magic.
721  if (TT.isOSDarwin())
722  return false;
723  // Use linker script magic to get data/cnts/name start/end.
724  if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
725  TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS4CPU() ||
726  TT.isOSWindows())
727  return false;
728 
729  return true;
730 }
731 
733 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
734  GlobalVariable *NamePtr = Inc->getName();
735  auto It = ProfileDataMap.find(NamePtr);
736  PerFunctionProfileData PD;
737  if (It != ProfileDataMap.end()) {
738  if (It->second.RegionCounters)
739  return It->second.RegionCounters;
740  PD = It->second;
741  }
742 
743  // Match the linkage and visibility of the name global. COFF supports using
744  // comdats with internal symbols, so do that if we can.
745  Function *Fn = Inc->getParent()->getParent();
747  GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
748  if (TT.isOSBinFormatCOFF()) {
750  Visibility = GlobalValue::DefaultVisibility;
751  }
752 
753  // Move the name variable to the right section. Place them in a COMDAT group
754  // if the associated function is a COMDAT. This will make sure that only one
755  // copy of counters of the COMDAT function will be emitted after linking. Keep
756  // in mind that this pass may run before the inliner, so we need to create a
757  // new comdat group for the counters and profiling data. If we use the comdat
758  // of the parent function, that will result in relocations against discarded
759  // sections.
760  bool NeedComdat = needsComdatForCounter(*Fn, *M);
761  if (NeedComdat) {
762  if (TT.isOSBinFormatCOFF()) {
763  // For COFF, put the counters, data, and values each into their own
764  // comdats. We can't use a group because the Visual C++ linker will
765  // report duplicate symbol errors if there are multiple external symbols
766  // with the same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
768  Visibility = GlobalValue::HiddenVisibility;
769  }
770  }
771  auto MaybeSetComdat = [=](GlobalVariable *GV) {
772  if (NeedComdat)
773  GV->setComdat(M->getOrInsertComdat(GV->getName()));
774  };
775 
776  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
777  LLVMContext &Ctx = M->getContext();
778  ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
779 
780  // Create the counters variable.
781  auto *CounterPtr =
782  new GlobalVariable(*M, CounterTy, false, Linkage,
783  Constant::getNullValue(CounterTy),
785  CounterPtr->setVisibility(Visibility);
786  CounterPtr->setSection(
787  getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
788  CounterPtr->setAlignment(Align(8));
789  MaybeSetComdat(CounterPtr);
790  CounterPtr->setLinkage(Linkage);
791 
792  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
793  // Allocate statically the array of pointers to value profile nodes for
794  // the current function.
795  Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
796  if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(TT)) {
797  uint64_t NS = 0;
798  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
799  NS += PD.NumValueSites[Kind];
800  if (NS) {
801  ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
802 
803  auto *ValuesVar =
804  new GlobalVariable(*M, ValuesTy, false, Linkage,
805  Constant::getNullValue(ValuesTy),
807  ValuesVar->setVisibility(Visibility);
808  ValuesVar->setSection(
809  getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
810  ValuesVar->setAlignment(Align(8));
811  MaybeSetComdat(ValuesVar);
812  ValuesPtrExpr =
814  }
815  }
816 
817  // Create data variable.
818  auto *Int16Ty = Type::getInt16Ty(Ctx);
819  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
820  Type *DataTypes[] = {
821 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
823  };
824  auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
825 
826  Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
827  ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
828  : ConstantPointerNull::get(Int8PtrTy);
829 
830  Constant *Int16ArrayVals[IPVK_Last + 1];
831  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
832  Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
833 
834  Constant *DataVals[] = {
835 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
837  };
838  auto *Data = new GlobalVariable(*M, DataTy, false, Linkage,
839  ConstantStruct::get(DataTy, DataVals),
841  Data->setVisibility(Visibility);
842  Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
843  Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
844  MaybeSetComdat(Data);
845  Data->setLinkage(Linkage);
846 
847  PD.RegionCounters = CounterPtr;
848  PD.DataVar = Data;
849  ProfileDataMap[NamePtr] = PD;
850 
851  // Mark the data variable as used so that it isn't stripped out.
852  UsedVars.push_back(Data);
853  // Now that the linkage set by the FE has been passed to the data and counter
854  // variables, reset Name variable's linkage and visibility to private so that
855  // it can be removed later by the compiler.
857  // Collect the referenced names to be used by emitNameData.
858  ReferencedNames.push_back(NamePtr);
859 
860  return CounterPtr;
861 }
862 
863 void InstrProfiling::emitVNodes() {
864  if (!ValueProfileStaticAlloc)
865  return;
866 
867  // For now only support this on platforms that do
868  // not require runtime registration to discover
869  // named section start/end.
871  return;
872 
873  size_t TotalNS = 0;
874  for (auto &PD : ProfileDataMap) {
875  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
876  TotalNS += PD.second.NumValueSites[Kind];
877  }
878 
879  if (!TotalNS)
880  return;
881 
882  uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
883 // Heuristic for small programs with very few total value sites.
884 // The default value of vp-counters-per-site is chosen based on
885 // the observation that large apps usually have a low percentage
886 // of value sites that actually have any profile data, and thus
887 // the average number of counters per site is low. For small
888 // apps with very few sites, this may not be true. Bump up the
889 // number of counters in this case.
890 #define INSTR_PROF_MIN_VAL_COUNTS 10
891  if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
892  NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
893 
894  auto &Ctx = M->getContext();
895  Type *VNodeTypes[] = {
896 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
898  };
899  auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
900 
901  ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
902  auto *VNodesVar = new GlobalVariable(
903  *M, VNodesTy, false, GlobalValue::PrivateLinkage,
905  VNodesVar->setSection(
906  getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
907  UsedVars.push_back(VNodesVar);
908 }
909 
910 void InstrProfiling::emitNameData() {
911  std::string UncompressedData;
912 
913  if (ReferencedNames.empty())
914  return;
915 
916  std::string CompressedNameStr;
917  if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
918  DoNameCompression)) {
919  report_fatal_error(toString(std::move(E)), false);
920  }
921 
922  auto &Ctx = M->getContext();
923  auto *NamesVal = ConstantDataArray::getString(
924  Ctx, StringRef(CompressedNameStr), false);
925  NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
926  GlobalValue::PrivateLinkage, NamesVal,
928  NamesSize = CompressedNameStr.size();
929  NamesVar->setSection(
930  getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
931  // On COFF, it's important to reduce the alignment down to 1 to prevent the
932  // linker from inserting padding before the start of the names section or
933  // between names entries.
934  NamesVar->setAlignment(Align::None());
935  UsedVars.push_back(NamesVar);
936 
937  for (auto *NamePtr : ReferencedNames)
938  NamePtr->eraseFromParent();
939 }
940 
941 void InstrProfiling::emitRegistration() {
943  return;
944 
945  // Construct the function.
946  auto *VoidTy = Type::getVoidTy(M->getContext());
947  auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
948  auto *Int64Ty = Type::getInt64Ty(M->getContext());
949  auto *RegisterFTy = FunctionType::get(VoidTy, false);
950  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
952  RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
953  if (Options.NoRedZone)
954  RegisterF->addFnAttr(Attribute::NoRedZone);
955 
956  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
957  auto *RuntimeRegisterF =
960 
961  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
962  for (Value *Data : UsedVars)
963  if (Data != NamesVar && !isa<Function>(Data))
964  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
965 
966  if (NamesVar) {
967  Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
968  auto *NamesRegisterTy =
969  FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
970  auto *NamesRegisterF =
973  IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
974  IRB.getInt64(NamesSize)});
975  }
976 
977  IRB.CreateRetVoid();
978 }
979 
980 bool InstrProfiling::emitRuntimeHook() {
981  // We expect the linker to be invoked with -u<hook_var> flag for linux,
982  // for which case there is no need to emit the user function.
983  if (TT.isOSLinux())
984  return false;
985 
986  // If the module's provided its own runtime, we don't need to do anything.
988  return false;
989 
990  // Declare an external variable that will pull in the runtime initialization.
991  auto *Int32Ty = Type::getInt32Ty(M->getContext());
992  auto *Var =
994  nullptr, getInstrProfRuntimeHookVarName());
995 
996  // Make a function that uses it.
1000  User->addFnAttr(Attribute::NoInline);
1001  if (Options.NoRedZone)
1002  User->addFnAttr(Attribute::NoRedZone);
1003  User->setVisibility(GlobalValue::HiddenVisibility);
1004  if (TT.supportsCOMDAT())
1005  User->setComdat(M->getOrInsertComdat(User->getName()));
1006 
1008  auto *Load = IRB.CreateLoad(Int32Ty, Var);
1009  IRB.CreateRet(Load);
1010 
1011  // Mark the user variable as used so that it isn't stripped out.
1012  UsedVars.push_back(User);
1013  return true;
1014 }
1015 
1016 void InstrProfiling::emitUses() {
1017  if (!UsedVars.empty())
1018  appendToUsed(*M, UsedVars);
1019 }
1020 
1021 void InstrProfiling::emitInitialization() {
1022  // Create ProfileFileName variable. Don't don't this for the
1023  // context-sensitive instrumentation lowering: This lowering is after
1024  // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1025  // have already create the variable before LTO/ThinLTO linking.
1026  if (!IsCS)
1027  createProfileFileNameVar(*M, Options.InstrProfileOutput);
1028  Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1029  if (!RegisterF)
1030  return;
1031 
1032  // Create the initialization function.
1033  auto *VoidTy = Type::getVoidTy(M->getContext());
1034  auto *F = Function::Create(FunctionType::get(VoidTy, false),
1037  F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1038  F->addFnAttr(Attribute::NoInline);
1039  if (Options.NoRedZone)
1040  F->addFnAttr(Attribute::NoRedZone);
1041 
1042  // Add the basic block and the necessary calls.
1043  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1044  IRB.CreateCall(RegisterF, {});
1045  IRB.CreateRetVoid();
1046 
1047  appendToGlobalCtors(*M, F, 0);
1048 }
cl::opt< std::string > MemOPSizeRange("memop-size-range", cl::desc("Set the range of size in memory intrinsic calls to be profiled " "precisely, in a format of <start_val>:<end_val>"), cl::init(""))
bool isOSDarwin() const
isOSDarwin - Is this a "Darwin" OS (OS X, iOS, or watchOS).
Definition: Triple.h:481
Optional< uint64_t > getBlockProfileCount(const BasicBlock *BB, bool AllowSynthetic=false) const
Returns the estimated profile count of BB.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:67
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable...
Definition: InstrProf.h:152
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
Definition: Module.h:241
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2579
bool hasLocalLinkage() const
Definition: GlobalValue.h:445
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:38
ConstantInt * getIndex() const
void dropAllReferences()
Drop all references to operands.
Definition: User.h:294
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:82
LLVM_NODISCARD bool endswith(StringRef Suffix) const
Check if this string ends with the given Suffix.
Definition: StringRef.h:281
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:777
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
cl::opt< unsigned > MemOPSizeLarge("memop-size-large", cl::desc("Set large value thresthold in memory intrinsic size profiling. " "Value of 0 disables the large value profiling."), cl::init(8192))
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:122
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve &#39;CreateLoad(Ty, Ptr, "...")&#39; correctly, instead of converting the string to &#39;bool...
Definition: IRBuilder.h:1575
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:66
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
Definition: LoopInfoImpl.h:85
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:170
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
#define INSTR_PROF_MIN_VAL_COUNTS
This class represents a function call, abstracting a target machine&#39;s calling convention.
bool hasAvailableExternallyLinkage() const
Definition: GlobalValue.h:432
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1088
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:160
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
Definition: Module.h:406
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Externally visible function.
Definition: GlobalValue.h:48
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:323
GlobalVariable * getGlobalVariable(StringRef Name) const
Look up the specified global variable in the module symbol table.
Definition: Module.h:391
This file provides the interface for LLVM&#39;s PGO Instrumentation lowering pass.
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:165
F(f)
bool isOSFuchsia() const
Definition: Triple.h:505
An instruction for reading from memory.
Definition: Instructions.h:169
static unsigned InstrCount
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:181
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:144
static IntegerType * getInt16Ty(LLVMContext &C)
Definition: Type.cpp:179
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:91
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:289
Attribute::AttrKind getExtAttrForI32Param(bool Signed=true) const
Returns extension attribute kind to be used for i32 parameters corresponding to C-level int or unsign...
ModulePass * createInstrProfilingLegacyPass(const InstrProfOptions &Options=InstrProfOptions(), bool IsCS=false)
Insert frontend instrumentation based profiling. Parameter IsCS indicates if.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
Hexagon Hardware Loops
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:141
std::string toString(Error E)
Write all error messages (if any) in E to a string.
Definition: Error.h:986
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:640
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:928
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:85
#define INT64_MIN
Definition: DataTypes.h:80
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:245
Instrumentation based profiling lowering pass.
ReturnInst * CreateRet(Value *V)
Create a &#39;ret <val>&#39; instruction.
Definition: IRBuilder.h:865
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:779
This file contains the simple types necessary to represent the attributes associated with functions a...
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:378
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1118
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:113
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:592
bool isOSSolaris() const
Definition: Triple.h:511
static StructType * get(LLVMContext &Context, ArrayRef< Type *> Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:346
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:133
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1604
void getExitBlocks(SmallVectorImpl< BlockT *> &ExitBlocks) const
Return all of the successor blocks of this loop.
Definition: LoopInfoImpl.h:62
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:261
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1963
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:246
void appendToUsed(Module &M, ArrayRef< GlobalValue *> Values)
Adds global values to the llvm.used list.
Class to represent array types.
Definition: DerivedTypes.h:408
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
Memory SSA
Definition: MemorySSA.cpp:65
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:144
LinkageTypes getLinkage() const
Definition: GlobalValue.h:460
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:429
bool isOSNetBSD() const
Definition: Triple.h:493
bool hasLinkOnceLinkage() const
Definition: GlobalValue.h:435
GlobalVariable * getName() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:144
Value * getOperand(unsigned i) const
Definition: User.h:169
ConstantInt * getNumCounters() const
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:157
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1070
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, bool IsRange=false)
static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix)
Get the name of a profiling variable for a particular function.
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1804
bool isOSWindows() const
Tests whether the OS is Windows.
Definition: Triple.h:538
void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, int64_t &RangeLast)
Definition: InstrProf.cpp:1111
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1044
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:135
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:148
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:154
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:223
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1432
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:236
ConstantInt * getIndex() const
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1151
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:72
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
ValueKind
Value kinds.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:370
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function&#39;s PGO names...
Definition: InstrProf.h:98
void eraseFromParent()
eraseFromParent - This method unlinks &#39;this&#39; from the containing module and deletes it...
Definition: Globals.cpp:384
Represent the analysis usage information of a pass.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:165
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1172
bool isPS4CPU() const
Tests whether the target is the PS4 CPU.
Definition: Triple.h:648
static bool shouldRecordFunctionAddr(Function *F)
constexpr double e
Definition: MathExtras.h:57
static FunctionType * get(Type *Result, ArrayRef< Type *> Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:301
static Constant * get(StructType *T, ArrayRef< Constant *> V)
Definition: Constants.cpp:1075
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:99
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:348
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:59
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Definition: IRBuilder.h:1670
void getExitingBlocks(SmallVectorImpl< BlockT *> &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
Definition: LoopInfoImpl.h:34
StringRef toStringRef(bool B)
Construct a string ref from a boolean.
Definition: StringExtras.h:52
const Constant * stripPointerCasts() const
Definition: Constant.h:183
static constexpr const Align None()
Returns a default constructed Align which corresponds to no alignment.
Definition: Alignment.h:93
Comdat * getOrInsertComdat(StringRef Name)
Return the Comdat in the module with the specified name.
Definition: Module.cpp:482
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:160
size_t size() const
Definition: SmallVector.h:52
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:224
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:43
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:88
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1397
unsigned getNumOperands() const
Definition: User.h:191
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:417
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:390
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater...
Definition: SSAUpdater.h:136
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
Module.h This file contains the declarations for the Module class.
Provides information about what library functions are available for the current target.
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:586
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:653
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:63
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:301
FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T, AttributeList AttributeList)
Look up the specified function in the module symbol table.
Definition: Module.cpp:143
#define NC
Definition: regutils.h:42
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:454
bool isOSFreeBSD() const
Definition: Triple.h:501
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition: Module.cpp:174
ConstantArray - Constant Array Declarations.
Definition: Constants.h:413
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:146
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:128
Options for the frontend instrumentation based profiling pass.
bool hasComdat() const
Definition: GlobalObject.h:119
This represents the llvm.instrprof_increment intrinsic.
ReturnInst * CreateRetVoid()
Create a &#39;ret void&#39; instruction.
Definition: IRBuilder.h:860
Analysis providing branch probability information.
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:180
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:509
ConstantInt * getHash() const
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:106
#define I(x, y, z)
Definition: MD5.cpp:58
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:224
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:587
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
Type * getValueType() const
Definition: GlobalValue.h:279
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2239
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
SmallVector< LoopT *, 4 > getLoopsInPreorder()
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
Definition: LoopInfoImpl.h:567
Analysis pass providing the TargetLibraryInfo.
bool hasAddressTaken(const User **=nullptr) const
hasAddressTaken - returns true if there are any uses of this function other than direct calls or invo...
Definition: Function.cpp:1420
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
StringRef getInstrProfValueRangeProfFuncName()
Return the name profile runtime entry point to do value range profiling.
Definition: InstrProf.h:77
aarch64 promote const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:575
LLVM Value Representation.
Definition: Value.h:74
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:94
GlobalVariable * getName() const
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
ConstantInt * getValueKind() const
print Print MemDeps of function
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
A container for analyses that lazily runs them and caches their results.
static bool lowerIntrinsics(Module &M)
#define LLVM_DEBUG(X)
Definition: Debug.h:122
This represents the llvm.instrprof_value_profile intrinsic.
static InstrProfIncrementInst * castToIncrementInst(Instruction *Instr)
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:215
bool use_empty() const
Definition: Value.h:343
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
IntegerType * Int32Ty
Value * CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1, const Twine &Name="")
Definition: IRBuilder.h:1831
INITIALIZE_PASS_BEGIN(InstrProfilingLegacyPass, "instrprof", "Frontend instrumentation-based coverage lowering.", false, false) INITIALIZE_PASS_END(InstrProfilingLegacyPass
const BasicBlock * getParent() const
Definition: Instruction.h:66
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:1045