LLVM  14.0.0git
InstrProfiling.cpp
Go to the documentation of this file.
1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10 // It also builds the data structures and initialization code needed for
11 // updating execution counts and emitting the profile at runtime.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/ADT/Twine.h"
23 #include "llvm/Analysis/LoopInfo.h"
25 #include "llvm/IR/Attributes.h"
26 #include "llvm/IR/BasicBlock.h"
27 #include "llvm/IR/Constant.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DIBuilder.h"
30 #include "llvm/IR/DerivedTypes.h"
31 #include "llvm/IR/DiagnosticInfo.h"
32 #include "llvm/IR/Dominators.h"
33 #include "llvm/IR/Function.h"
34 #include "llvm/IR/GlobalValue.h"
35 #include "llvm/IR/GlobalVariable.h"
36 #include "llvm/IR/IRBuilder.h"
37 #include "llvm/IR/Instruction.h"
38 #include "llvm/IR/Instructions.h"
39 #include "llvm/IR/IntrinsicInst.h"
40 #include "llvm/IR/Module.h"
41 #include "llvm/IR/Type.h"
42 #include "llvm/InitializePasses.h"
43 #include "llvm/Pass.h"
46 #include "llvm/Support/Casting.h"
48 #include "llvm/Support/Error.h"
53 #include <algorithm>
54 #include <cassert>
55 #include <cstddef>
56 #include <cstdint>
57 #include <string>
58 
59 using namespace llvm;
60 
61 #define DEBUG_TYPE "instrprof"
62 
63 namespace llvm {
65  DebugInfoCorrelate("debug-info-correlate", cl::ZeroOrMore,
66  cl::desc("Use debug info to correlate profiles."),
67  cl::init(false));
68 } // namespace llvm
69 
70 namespace {
71 
72 cl::opt<bool> DoHashBasedCounterSplit(
73  "hash-based-counter-split",
74  cl::desc("Rename counter variable of a comdat function based on cfg hash"),
75  cl::init(true));
76 
78  RuntimeCounterRelocation("runtime-counter-relocation",
79  cl::desc("Enable relocating counters at runtime."),
80  cl::init(false));
81 
82 cl::opt<bool> ValueProfileStaticAlloc(
83  "vp-static-alloc",
84  cl::desc("Do static counter allocation for value profiler"),
85  cl::init(true));
86 
87 cl::opt<double> NumCountersPerValueSite(
88  "vp-counters-per-site",
89  cl::desc("The average number of profile counters allocated "
90  "per value profiling site."),
91  // This is set to a very small value because in real programs, only
92  // a very small percentage of value sites have non-zero targets, e.g, 1/30.
93  // For those sites with non-zero profile, the average number of targets
94  // is usually smaller than 2.
95  cl::init(1.0));
96 
97 cl::opt<bool> AtomicCounterUpdateAll(
98  "instrprof-atomic-counter-update-all", cl::ZeroOrMore,
99  cl::desc("Make all profile counter updates atomic (for testing only)"),
100  cl::init(false));
101 
102 cl::opt<bool> AtomicCounterUpdatePromoted(
103  "atomic-counter-update-promoted", cl::ZeroOrMore,
104  cl::desc("Do counter update using atomic fetch add "
105  " for promoted counters only"),
106  cl::init(false));
107 
108 cl::opt<bool> AtomicFirstCounter(
109  "atomic-first-counter", cl::ZeroOrMore,
110  cl::desc("Use atomic fetch add for first counter in a function (usually "
111  "the entry counter)"),
112  cl::init(false));
113 
114 // If the option is not specified, the default behavior about whether
115 // counter promotion is done depends on how instrumentaiton lowering
116 // pipeline is setup, i.e., the default value of true of this option
117 // does not mean the promotion will be done by default. Explicitly
118 // setting this option can override the default behavior.
119 cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
120  cl::desc("Do counter register promotion"),
121  cl::init(false));
122 cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
123  cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20),
124  cl::desc("Max number counter promotions per loop to avoid"
125  " increasing register pressure too much"));
126 
127 // A debug option
129  MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
130  cl::desc("Max number of allowed counter promotions"));
131 
132 cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
133  cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3),
134  cl::desc("The max number of exiting blocks of a loop to allow "
135  " speculative counter promotion"));
136 
137 cl::opt<bool> SpeculativeCounterPromotionToLoop(
138  cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false),
139  cl::desc("When the option is false, if the target block is in a loop, "
140  "the promotion will be disallowed unless the promoted counter "
141  " update can be further/iteratively promoted into an acyclic "
142  " region."));
143 
144 cl::opt<bool> IterativeCounterPromotion(
145  cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
146  cl::desc("Allow counter promotion across the whole loop nest."));
147 
148 cl::opt<bool> SkipRetExitBlock(
149  cl::ZeroOrMore, "skip-ret-exit-block", cl::init(true),
150  cl::desc("Suppress counter promotion if exit blocks contain ret."));
151 
152 class InstrProfilingLegacyPass : public ModulePass {
153  InstrProfiling InstrProf;
154 
155 public:
156  static char ID;
157 
158  InstrProfilingLegacyPass() : ModulePass(ID) {}
159  InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false)
160  : ModulePass(ID), InstrProf(Options, IsCS) {
162  }
163 
164  StringRef getPassName() const override {
165  return "Frontend instrumentation-based coverage lowering";
166  }
167 
168  bool runOnModule(Module &M) override {
169  auto GetTLI = [this](Function &F) -> TargetLibraryInfo & {
170  return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
171  };
172  return InstrProf.run(M, GetTLI);
173  }
174 
175  void getAnalysisUsage(AnalysisUsage &AU) const override {
176  AU.setPreservesCFG();
178  }
179 };
180 
181 ///
182 /// A helper class to promote one counter RMW operation in the loop
183 /// into register update.
184 ///
185 /// RWM update for the counter will be sinked out of the loop after
186 /// the transformation.
187 ///
188 class PGOCounterPromoterHelper : public LoadAndStorePromoter {
189 public:
190  PGOCounterPromoterHelper(
192  BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
193  ArrayRef<Instruction *> InsertPts,
195  LoopInfo &LI)
196  : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
197  InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
198  assert(isa<LoadInst>(L));
199  assert(isa<StoreInst>(S));
200  SSA.AddAvailableValue(PH, Init);
201  }
202 
203  void doExtraRewritesBeforeFinalDeletion() override {
204  for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
205  BasicBlock *ExitBlock = ExitBlocks[i];
206  Instruction *InsertPos = InsertPts[i];
207  // Get LiveIn value into the ExitBlock. If there are multiple
208  // predecessors, the value is defined by a PHI node in this
209  // block.
210  Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
211  Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
212  Type *Ty = LiveInValue->getType();
213  IRBuilder<> Builder(InsertPos);
214  if (AtomicCounterUpdatePromoted)
215  // automic update currently can only be promoted across the current
216  // loop, not the whole loop nest.
217  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
218  MaybeAlign(),
220  else {
221  LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
222  auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
223  auto *NewStore = Builder.CreateStore(NewVal, Addr);
224 
225  // Now update the parent loop's candidate list:
226  if (IterativeCounterPromotion) {
227  auto *TargetLoop = LI.getLoopFor(ExitBlock);
228  if (TargetLoop)
229  LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
230  }
231  }
232  }
233  }
234 
235 private:
237  ArrayRef<BasicBlock *> ExitBlocks;
238  ArrayRef<Instruction *> InsertPts;
240  LoopInfo &LI;
241 };
242 
243 /// A helper class to do register promotion for all profile counter
244 /// updates in a loop.
245 ///
246 class PGOCounterPromoter {
247 public:
248  PGOCounterPromoter(
250  Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
251  : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
252 
253  // Skip collection of ExitBlocks and InsertPts for loops that will not be
254  // able to have counters promoted.
255  SmallVector<BasicBlock *, 8> LoopExitBlocks;
257 
258  L.getExitBlocks(LoopExitBlocks);
259  if (!isPromotionPossible(&L, LoopExitBlocks))
260  return;
261 
262  for (BasicBlock *ExitBlock : LoopExitBlocks) {
263  if (BlockSet.insert(ExitBlock).second) {
264  ExitBlocks.push_back(ExitBlock);
265  InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
266  }
267  }
268  }
269 
270  bool run(int64_t *NumPromoted) {
271  // Skip 'infinite' loops:
272  if (ExitBlocks.size() == 0)
273  return false;
274 
275  // Skip if any of the ExitBlocks contains a ret instruction.
276  // This is to prevent dumping of incomplete profile -- if the
277  // the loop is a long running loop and dump is called in the middle
278  // of the loop, the result profile is incomplete.
279  // FIXME: add other heuristics to detect long running loops.
280  if (SkipRetExitBlock) {
281  for (auto BB : ExitBlocks)
282  if (isa<ReturnInst>(BB->getTerminator()))
283  return false;
284  }
285 
286  unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
287  if (MaxProm == 0)
288  return false;
289 
290  unsigned Promoted = 0;
291  for (auto &Cand : LoopToCandidates[&L]) {
292 
294  SSAUpdater SSA(&NewPHIs);
295  Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
296 
297  // If BFI is set, we will use it to guide the promotions.
298  if (BFI) {
299  auto *BB = Cand.first->getParent();
300  auto InstrCount = BFI->getBlockProfileCount(BB);
301  if (!InstrCount)
302  continue;
303  auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
304  // If the average loop trip count is not greater than 1.5, we skip
305  // promotion.
306  if (PreheaderCount &&
307  (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2))
308  continue;
309  }
310 
311  PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
312  L.getLoopPreheader(), ExitBlocks,
313  InsertPts, LoopToCandidates, LI);
314  Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
315  Promoted++;
316  if (Promoted >= MaxProm)
317  break;
318 
319  (*NumPromoted)++;
320  if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
321  break;
322  }
323 
324  LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
325  << L.getLoopDepth() << ")\n");
326  return Promoted != 0;
327  }
328 
329 private:
330  bool allowSpeculativeCounterPromotion(Loop *LP) {
331  SmallVector<BasicBlock *, 8> ExitingBlocks;
332  L.getExitingBlocks(ExitingBlocks);
333  // Not considierered speculative.
334  if (ExitingBlocks.size() == 1)
335  return true;
336  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
337  return false;
338  return true;
339  }
340 
341  // Check whether the loop satisfies the basic conditions needed to perform
342  // Counter Promotions.
343  bool
344  isPromotionPossible(Loop *LP,
345  const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
346  // We can't insert into a catchswitch.
347  if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
348  return isa<CatchSwitchInst>(Exit->getTerminator());
349  }))
350  return false;
351 
352  if (!LP->hasDedicatedExits())
353  return false;
354 
355  BasicBlock *PH = LP->getLoopPreheader();
356  if (!PH)
357  return false;
358 
359  return true;
360  }
361 
362  // Returns the max number of Counter Promotions for LP.
363  unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
364  SmallVector<BasicBlock *, 8> LoopExitBlocks;
365  LP->getExitBlocks(LoopExitBlocks);
366  if (!isPromotionPossible(LP, LoopExitBlocks))
367  return 0;
368 
369  SmallVector<BasicBlock *, 8> ExitingBlocks;
370  LP->getExitingBlocks(ExitingBlocks);
371 
372  // If BFI is set, we do more aggressive promotions based on BFI.
373  if (BFI)
374  return (unsigned)-1;
375 
376  // Not considierered speculative.
377  if (ExitingBlocks.size() == 1)
378  return MaxNumOfPromotionsPerLoop;
379 
380  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
381  return 0;
382 
383  // Whether the target block is in a loop does not matter:
384  if (SpeculativeCounterPromotionToLoop)
385  return MaxNumOfPromotionsPerLoop;
386 
387  // Now check the target block:
388  unsigned MaxProm = MaxNumOfPromotionsPerLoop;
389  for (auto *TargetBlock : LoopExitBlocks) {
390  auto *TargetLoop = LI.getLoopFor(TargetBlock);
391  if (!TargetLoop)
392  continue;
393  unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
394  unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
395  MaxProm =
396  std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
397  PendingCandsInTarget);
398  }
399  return MaxProm;
400  }
401 
403  SmallVector<BasicBlock *, 8> ExitBlocks;
405  Loop &L;
406  LoopInfo &LI;
408 };
409 
410 enum class ValueProfilingCallType {
411  // Individual values are tracked. Currently used for indiret call target
412  // profiling.
413  Default,
414 
415  // MemOp: the memop size value profiling.
416  MemOp
417 };
418 
419 } // end anonymous namespace
420 
424  auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
426  };
427  if (!run(M, GetTLI))
428  return PreservedAnalyses::all();
429 
430  return PreservedAnalyses::none();
431 }
432 
434 INITIALIZE_PASS_BEGIN(InstrProfilingLegacyPass, "instrprof",
435  "Frontend instrumentation-based coverage lowering.",
436  false, false)
438 INITIALIZE_PASS_END(InstrProfilingLegacyPass, "instrprof",
439  "Frontend instrumentation-based coverage lowering.", false,
440  false)
441 
442 ModulePass *
444  bool IsCS) {
445  return new InstrProfilingLegacyPass(Options, IsCS);
446 }
447 
448 bool InstrProfiling::lowerIntrinsics(Function *F) {
449  bool MadeChange = false;
450  PromotionCandidates.clear();
451  for (BasicBlock &BB : *F) {
452  for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
453  if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
454  lowerIncrement(IPIS);
455  MadeChange = true;
456  } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
457  lowerIncrement(IPI);
458  MadeChange = true;
459  } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
460  lowerValueProfileInst(IPVP);
461  MadeChange = true;
462  }
463  }
464  }
465 
466  if (!MadeChange)
467  return false;
468 
469  promoteCounterLoadStores(F);
470  return true;
471 }
472 
473 bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
474  // Mach-O don't support weak external references.
475  if (TT.isOSBinFormatMachO())
476  return false;
477 
478  if (RuntimeCounterRelocation.getNumOccurrences() > 0)
479  return RuntimeCounterRelocation;
480 
481  // Fuchsia uses runtime counter relocation by default.
482  return TT.isOSFuchsia();
483 }
484 
485 bool InstrProfiling::isCounterPromotionEnabled() const {
486  if (DoCounterPromotion.getNumOccurrences() > 0)
487  return DoCounterPromotion;
488 
489  return Options.DoCounterPromotion;
490 }
491 
492 void InstrProfiling::promoteCounterLoadStores(Function *F) {
493  if (!isCounterPromotionEnabled())
494  return;
495 
496  DominatorTree DT(*F);
497  LoopInfo LI(DT);
498  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
499 
500  std::unique_ptr<BlockFrequencyInfo> BFI;
501  if (Options.UseBFIInPromotion) {
502  std::unique_ptr<BranchProbabilityInfo> BPI;
503  BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
504  BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
505  }
506 
507  for (const auto &LoadStore : PromotionCandidates) {
508  auto *CounterLoad = LoadStore.first;
509  auto *CounterStore = LoadStore.second;
510  BasicBlock *BB = CounterLoad->getParent();
511  Loop *ParentLoop = LI.getLoopFor(BB);
512  if (!ParentLoop)
513  continue;
514  LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
515  }
516 
518 
519  // Do a post-order traversal of the loops so that counter updates can be
520  // iteratively hoisted outside the loop nest.
521  for (auto *Loop : llvm::reverse(Loops)) {
522  PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
523  Promoter.run(&TotalCountersPromoted);
524  }
525 }
526 
527 static bool needsRuntimeHookUnconditionally(const Triple &TT) {
528  // On Fuchsia, we only need runtime hook if any counters are present.
529  if (TT.isOSFuchsia())
530  return false;
531 
532  return true;
533 }
534 
535 /// Check if the module contains uses of any profiling intrinsics.
537  auto containsIntrinsic = [&](int ID) {
538  if (auto *F = M.getFunction(Intrinsic::getName(ID)))
539  return !F->use_empty();
540  return false;
541  };
542  return containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
543  containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
544  containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
545 }
546 
548  Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
549  this->M = &M;
550  this->GetTLI = std::move(GetTLI);
551  NamesVar = nullptr;
552  NamesSize = 0;
553  ProfileDataMap.clear();
554  CompilerUsedVars.clear();
555  UsedVars.clear();
556  TT = Triple(M.getTargetTriple());
557 
558  bool MadeChange = false;
559 
560  // Emit the runtime hook even if no counters are present.
562  MadeChange = emitRuntimeHook();
563 
564  // Improve compile time by avoiding linear scans when there is no work.
565  GlobalVariable *CoverageNamesVar =
566  M.getNamedGlobal(getCoverageUnusedNamesVarName());
567  if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
568  return MadeChange;
569 
570  // We did not know how many value sites there would be inside
571  // the instrumented function. This is counting the number of instrumented
572  // target value sites to enter it as field in the profile data variable.
573  for (Function &F : M) {
574  InstrProfIncrementInst *FirstProfIncInst = nullptr;
575  for (BasicBlock &BB : F)
576  for (auto I = BB.begin(), E = BB.end(); I != E; I++)
577  if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
578  computeNumValueSiteCounts(Ind);
579  else if (FirstProfIncInst == nullptr)
580  FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
581 
582  // Value profiling intrinsic lowering requires per-function profile data
583  // variable to be created first.
584  if (FirstProfIncInst != nullptr)
585  static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
586  }
587 
588  for (Function &F : M)
589  MadeChange |= lowerIntrinsics(&F);
590 
591  if (CoverageNamesVar) {
592  lowerCoverageData(CoverageNamesVar);
593  MadeChange = true;
594  }
595 
596  if (!MadeChange)
597  return false;
598 
599  emitVNodes();
600  emitNameData();
601  emitRuntimeHook();
602  emitRegistration();
603  emitUses();
604  emitInitialization();
605  return true;
606 }
607 
609  Module &M, const TargetLibraryInfo &TLI,
610  ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
611  LLVMContext &Ctx = M.getContext();
612  auto *ReturnTy = Type::getVoidTy(M.getContext());
613 
615  if (auto AK = TLI.getExtAttrForI32Param(false))
616  AL = AL.addParamAttribute(M.getContext(), 2, AK);
617 
619  CallType == ValueProfilingCallType::MemOp) &&
620  "Must be Default or MemOp");
621  Type *ParamTypes[] = {
622 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
624  };
625  auto *ValueProfilingCallTy =
626  FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
627  StringRef FuncName = CallType == ValueProfilingCallType::Default
630  return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
631 }
632 
633 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
634  GlobalVariable *Name = Ind->getName();
636  uint64_t Index = Ind->getIndex()->getZExtValue();
637  auto &PD = ProfileDataMap[Name];
638  PD.NumValueSites[ValueKind] =
639  std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));
640 }
641 
642 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
643  // TODO: Value profiling heavily depends on the data section which is omitted
644  // in lightweight mode. We need to move the value profile pointer to the
645  // Counter struct to get this working.
646  assert(
648  "Value profiling is not yet supported with lightweight instrumentation");
649  GlobalVariable *Name = Ind->getName();
650  auto It = ProfileDataMap.find(Name);
651  assert(It != ProfileDataMap.end() && It->second.DataVar &&
652  "value profiling detected in function with no counter incerement");
653 
654  GlobalVariable *DataVar = It->second.DataVar;
656  uint64_t Index = Ind->getIndex()->getZExtValue();
657  for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
658  Index += It->second.NumValueSites[Kind];
659 
660  IRBuilder<> Builder(Ind);
661  bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
662  llvm::InstrProfValueKind::IPVK_MemOPSize);
663  CallInst *Call = nullptr;
664  auto *TLI = &GetTLI(*Ind->getFunction());
665 
666  // To support value profiling calls within Windows exception handlers, funclet
667  // information contained within operand bundles needs to be copied over to
668  // the library call. This is required for the IR to be processed by the
669  // WinEHPrepare pass.
671  Ind->getOperandBundlesAsDefs(OpBundles);
672  if (!IsMemOpSize) {
673  Value *Args[3] = {Ind->getTargetValue(),
674  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
675  Builder.getInt32(Index)};
676  Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
677  OpBundles);
678  } else {
679  Value *Args[3] = {Ind->getTargetValue(),
680  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
681  Builder.getInt32(Index)};
682  Call = Builder.CreateCall(
683  getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
684  Args, OpBundles);
685  }
686  if (auto AK = TLI->getExtAttrForI32Param(false))
687  Call->addParamAttr(2, AK);
688  Ind->replaceAllUsesWith(Call);
689  Ind->eraseFromParent();
690 }
691 
692 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
693  GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
694 
695  IRBuilder<> Builder(Inc);
696  uint64_t Index = Inc->getIndex()->getZExtValue();
697  Value *Addr = Builder.CreateConstInBoundsGEP2_32(Counters->getValueType(),
698  Counters, 0, Index);
699 
700  if (isRuntimeCounterRelocationEnabled()) {
701  Type *Int64Ty = Type::getInt64Ty(M->getContext());
702  Type *Int64PtrTy = Type::getInt64PtrTy(M->getContext());
703  Function *Fn = Inc->getParent()->getParent();
704  Instruction &I = Fn->getEntryBlock().front();
705  LoadInst *LI = dyn_cast<LoadInst>(&I);
706  if (!LI) {
708  GlobalVariable *Bias =
709  M->getGlobalVariable(getInstrProfCounterBiasVarName());
710  if (!Bias) {
711  // Compiler must define this variable when runtime counter relocation
712  // is being used. Runtime has a weak external reference that is used
713  // to check whether that's the case or not.
714  Bias = new GlobalVariable(
715  *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
718  // A definition that's weak (linkonce_odr) without being in a COMDAT
719  // section wouldn't lead to link errors, but it would lead to a dead
720  // data word from every TU but one. Putting it in COMDAT ensures there
721  // will be exactly one data slot in the link.
722  if (TT.supportsCOMDAT())
723  Bias->setComdat(M->getOrInsertComdat(Bias->getName()));
724  }
725  LI = Builder.CreateLoad(Int64Ty, Bias);
726  }
727  auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), LI);
728  Addr = Builder.CreateIntToPtr(Add, Int64PtrTy);
729  }
730 
731  if (Options.Atomic || AtomicCounterUpdateAll ||
732  (Index == 0 && AtomicFirstCounter)) {
733  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
735  } else {
736  Value *IncStep = Inc->getStep();
737  Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
738  auto *Count = Builder.CreateAdd(Load, Inc->getStep());
739  auto *Store = Builder.CreateStore(Count, Addr);
740  if (isCounterPromotionEnabled())
741  PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
742  }
743  Inc->eraseFromParent();
744 }
745 
746 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
747  ConstantArray *Names =
748  cast<ConstantArray>(CoverageNamesVar->getInitializer());
749  for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
750  Constant *NC = Names->getOperand(I);
751  Value *V = NC->stripPointerCasts();
752  assert(isa<GlobalVariable>(V) && "Missing reference to function name");
753  GlobalVariable *Name = cast<GlobalVariable>(V);
754 
755  Name->setLinkage(GlobalValue::PrivateLinkage);
756  ReferencedNames.push_back(Name);
757  NC->dropAllReferences();
758  }
759  CoverageNamesVar->eraseFromParent();
760 }
761 
762 /// Get the name of a profiling variable for a particular function.
763 static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
764  bool &Renamed) {
765  StringRef NamePrefix = getInstrProfNameVarPrefix();
766  StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
767  Function *F = Inc->getParent()->getParent();
768  Module *M = F->getParent();
769  if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
770  !canRenameComdatFunc(*F)) {
771  Renamed = false;
772  return (Prefix + Name).str();
773  }
774  Renamed = true;
775  uint64_t FuncHash = Inc->getHash()->getZExtValue();
776  SmallVector<char, 24> HashPostfix;
777  if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
778  return (Prefix + Name).str();
779  return (Prefix + Name + "." + Twine(FuncHash)).str();
780 }
781 
783  auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
784  if (!MD)
785  return 0;
786 
787  // If the flag is a ConstantAsMetadata, it should be an integer representable
788  // in 64-bits.
789  return cast<ConstantInt>(MD->getValue())->getZExtValue();
790 }
791 
792 static bool enablesValueProfiling(const Module &M) {
793  return isIRPGOFlagSet(&M) ||
794  getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
795 }
796 
797 // Conservatively returns true if data variables may be referenced by code.
798 static bool profDataReferencedByCode(const Module &M) {
799  return enablesValueProfiling(M);
800 }
801 
802 static inline bool shouldRecordFunctionAddr(Function *F) {
803  // Only record function addresses if IR PGO is enabled or if clang value
804  // profiling is enabled. Recording function addresses greatly increases object
805  // file size, because it prevents the inliner from deleting functions that
806  // have been inlined everywhere.
807  if (!profDataReferencedByCode(*F->getParent()))
808  return false;
809 
810  // Check the linkage
811  bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
812  if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
813  !HasAvailableExternallyLinkage)
814  return true;
815 
816  // A function marked 'alwaysinline' with available_externally linkage can't
817  // have its address taken. Doing so would create an undefined external ref to
818  // the function, which would fail to link.
819  if (HasAvailableExternallyLinkage &&
820  F->hasFnAttribute(Attribute::AlwaysInline))
821  return false;
822 
823  // Prohibit function address recording if the function is both internal and
824  // COMDAT. This avoids the profile data variable referencing internal symbols
825  // in COMDAT.
826  if (F->hasLocalLinkage() && F->hasComdat())
827  return false;
828 
829  // Check uses of this function for other than direct calls or invokes to it.
830  // Inline virtual functions have linkeOnceODR linkage. When a key method
831  // exists, the vtable will only be emitted in the TU where the key method
832  // is defined. In a TU where vtable is not available, the function won't
833  // be 'addresstaken'. If its address is not recorded here, the profile data
834  // with missing address may be picked by the linker leading to missing
835  // indirect call target info.
836  return F->hasAddressTaken() || F->hasLinkOnceLinkage();
837 }
838 
840  // Don't do this for Darwin. compiler-rt uses linker magic.
841  if (TT.isOSDarwin())
842  return false;
843  // Use linker script magic to get data/cnts/name start/end.
844  if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
845  TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS4CPU() || TT.isOSWindows())
846  return false;
847 
848  return true;
849 }
850 
852 InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
853  GlobalVariable *NamePtr = Inc->getName();
854  auto &PD = ProfileDataMap[NamePtr];
855  if (PD.RegionCounters)
856  return PD.RegionCounters;
857 
858  // Match the linkage and visibility of the name global.
859  Function *Fn = Inc->getParent()->getParent();
861  GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
862 
863  // Use internal rather than private linkage so the counter variable shows up
864  // in the symbol table when using debug info for correlation.
868 
869  // Due to the limitation of binder as of 2021/09/28, the duplicate weak
870  // symbols in the same csect won't be discarded. When there are duplicate weak
871  // symbols, we can NOT guarantee that the relocations get resolved to the
872  // intended weak symbol, so we can not ensure the correctness of the relative
873  // CounterPtr, so we have to use private linkage for counter and data symbols.
874  if (TT.isOSBinFormatXCOFF()) {
876  Visibility = GlobalValue::DefaultVisibility;
877  }
878  // Move the name variable to the right section. Place them in a COMDAT group
879  // if the associated function is a COMDAT. This will make sure that only one
880  // copy of counters of the COMDAT function will be emitted after linking. Keep
881  // in mind that this pass may run before the inliner, so we need to create a
882  // new comdat group for the counters and profiling data. If we use the comdat
883  // of the parent function, that will result in relocations against discarded
884  // sections.
885  //
886  // If the data variable is referenced by code, counters and data have to be
887  // in different comdats for COFF because the Visual C++ linker will report
888  // duplicate symbol errors if there are multiple external symbols with the
889  // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
890  //
891  // For ELF, when not using COMDAT, put counters, data and values into a
892  // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
893  // allows -z start-stop-gc to discard the entire group when the function is
894  // discarded.
895  bool DataReferencedByCode = profDataReferencedByCode(*M);
896  bool NeedComdat = needsComdatForCounter(*Fn, *M);
897  bool Renamed;
898  std::string CntsVarName =
899  getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);
900  std::string DataVarName =
901  getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
902  auto MaybeSetComdat = [&](GlobalVariable *GV) {
903  bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
904  if (UseComdat) {
905  StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
906  ? GV->getName()
907  : CntsVarName;
908  Comdat *C = M->getOrInsertComdat(GroupName);
909  if (!NeedComdat)
910  C->setSelectionKind(Comdat::NoDeduplicate);
911  GV->setComdat(C);
912  }
913  };
914 
915  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
916  LLVMContext &Ctx = M->getContext();
917  ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
918 
919  // Create the counters variable.
920  auto *CounterPtr =
921  new GlobalVariable(*M, CounterTy, false, Linkage,
922  Constant::getNullValue(CounterTy), CntsVarName);
923  CounterPtr->setVisibility(Visibility);
924  CounterPtr->setSection(
925  getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
926  CounterPtr->setAlignment(Align(8));
927  MaybeSetComdat(CounterPtr);
928  CounterPtr->setLinkage(Linkage);
929  PD.RegionCounters = CounterPtr;
930  if (DebugInfoCorrelate) {
931  if (auto *SP = Fn->getSubprogram()) {
932  DIBuilder DB(*M, true, SP->getUnit());
933  Metadata *FunctionNameAnnotation[] = {
936  };
937  Metadata *CFGHashAnnotation[] = {
940  };
941  Metadata *NumCountersAnnotation[] = {
944  };
945  auto Annotations = DB.getOrCreateArray({
946  MDNode::get(Ctx, FunctionNameAnnotation),
947  MDNode::get(Ctx, CFGHashAnnotation),
948  MDNode::get(Ctx, NumCountersAnnotation),
949  });
950  auto *DICounter = DB.createGlobalVariableExpression(
951  SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
952  /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
953  CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
954  /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
955  Annotations);
956  CounterPtr->addDebugInfo(DICounter);
957  DB.finalize();
958  } else {
959  std::string Msg = ("Missing debug info for function " + Fn->getName() +
960  "; required for profile correlation.")
961  .str();
962  Ctx.diagnose(
963  DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
964  }
965  }
966 
967  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
968  // Allocate statically the array of pointers to value profile nodes for
969  // the current function.
970  Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
971  uint64_t NS = 0;
972  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
973  NS += PD.NumValueSites[Kind];
974  if (NS > 0 && ValueProfileStaticAlloc &&
976  ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
977  auto *ValuesVar = new GlobalVariable(
978  *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
979  getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
980  ValuesVar->setVisibility(Visibility);
981  ValuesVar->setSection(
982  getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
983  ValuesVar->setAlignment(Align(8));
984  MaybeSetComdat(ValuesVar);
985  ValuesPtrExpr =
987  }
988 
989  if (DebugInfoCorrelate) {
990  // Mark the counter variable as used so that it isn't optimized out.
991  CompilerUsedVars.push_back(PD.RegionCounters);
992  return PD.RegionCounters;
993  }
994 
995  // Create data variable.
996  auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
997  auto *Int16Ty = Type::getInt16Ty(Ctx);
998  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
999  Type *DataTypes[] = {
1000 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1002  };
1003  auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
1004 
1005  Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
1006  ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
1007  : ConstantPointerNull::get(Int8PtrTy);
1008 
1009  Constant *Int16ArrayVals[IPVK_Last + 1];
1010  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1011  Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
1012 
1013  // If the data variable is not referenced by code (if we don't emit
1014  // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1015  // data variable live under linker GC, the data variable can be private. This
1016  // optimization applies to ELF.
1017  //
1018  // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1019  // to be false.
1020  //
1021  // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1022  // that other copies must have the same CFG and cannot have value profiling.
1023  // If no hash suffix, other profd copies may be referenced by code.
1024  if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1025  (TT.isOSBinFormatELF() ||
1026  (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1028  Visibility = GlobalValue::DefaultVisibility;
1029  }
1030  auto *Data =
1031  new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName);
1032  // Reference the counter variable with a label difference (link-time
1033  // constant).
1034  auto *RelativeCounterPtr =
1035  ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
1036  ConstantExpr::getPtrToInt(Data, IntPtrTy));
1037 
1038  Constant *DataVals[] = {
1039 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1041  };
1042  Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1043 
1044  Data->setVisibility(Visibility);
1045  Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
1046  Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
1047  MaybeSetComdat(Data);
1048  Data->setLinkage(Linkage);
1049 
1050  PD.DataVar = Data;
1051 
1052  // Mark the data variable as used so that it isn't stripped out.
1053  CompilerUsedVars.push_back(Data);
1054  // Now that the linkage set by the FE has been passed to the data and counter
1055  // variables, reset Name variable's linkage and visibility to private so that
1056  // it can be removed later by the compiler.
1058  // Collect the referenced names to be used by emitNameData.
1059  ReferencedNames.push_back(NamePtr);
1060 
1061  return PD.RegionCounters;
1062 }
1063 
1064 void InstrProfiling::emitVNodes() {
1065  if (!ValueProfileStaticAlloc)
1066  return;
1067 
1068  // For now only support this on platforms that do
1069  // not require runtime registration to discover
1070  // named section start/end.
1072  return;
1073 
1074  size_t TotalNS = 0;
1075  for (auto &PD : ProfileDataMap) {
1076  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1077  TotalNS += PD.second.NumValueSites[Kind];
1078  }
1079 
1080  if (!TotalNS)
1081  return;
1082 
1083  uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1084 // Heuristic for small programs with very few total value sites.
1085 // The default value of vp-counters-per-site is chosen based on
1086 // the observation that large apps usually have a low percentage
1087 // of value sites that actually have any profile data, and thus
1088 // the average number of counters per site is low. For small
1089 // apps with very few sites, this may not be true. Bump up the
1090 // number of counters in this case.
1091 #define INSTR_PROF_MIN_VAL_COUNTS 10
1092  if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
1093  NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
1094 
1095  auto &Ctx = M->getContext();
1096  Type *VNodeTypes[] = {
1097 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1099  };
1100  auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
1101 
1102  ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
1103  auto *VNodesVar = new GlobalVariable(
1104  *M, VNodesTy, false, GlobalValue::PrivateLinkage,
1106  VNodesVar->setSection(
1107  getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
1108  // VNodesVar is used by runtime but not referenced via relocation by other
1109  // sections. Conservatively make it linker retained.
1110  UsedVars.push_back(VNodesVar);
1111 }
1112 
1113 void InstrProfiling::emitNameData() {
1114  std::string UncompressedData;
1115 
1116  if (ReferencedNames.empty())
1117  return;
1118 
1119  std::string CompressedNameStr;
1120  if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1123  }
1124 
1125  auto &Ctx = M->getContext();
1126  auto *NamesVal =
1127  ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);
1128  NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
1129  GlobalValue::PrivateLinkage, NamesVal,
1131  NamesSize = CompressedNameStr.size();
1132  NamesVar->setSection(
1133  getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1134  // On COFF, it's important to reduce the alignment down to 1 to prevent the
1135  // linker from inserting padding before the start of the names section or
1136  // between names entries.
1137  NamesVar->setAlignment(Align(1));
1138  // NamesVar is used by runtime but not referenced via relocation by other
1139  // sections. Conservatively make it linker retained.
1140  UsedVars.push_back(NamesVar);
1141 
1142  for (auto *NamePtr : ReferencedNames)
1143  NamePtr->eraseFromParent();
1144 }
1145 
1146 void InstrProfiling::emitRegistration() {
1148  return;
1149 
1150  // Construct the function.
1151  auto *VoidTy = Type::getVoidTy(M->getContext());
1152  auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
1153  auto *Int64Ty = Type::getInt64Ty(M->getContext());
1154  auto *RegisterFTy = FunctionType::get(VoidTy, false);
1155  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1157  RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1158  if (Options.NoRedZone)
1159  RegisterF->addFnAttr(Attribute::NoRedZone);
1160 
1161  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1162  auto *RuntimeRegisterF =
1165 
1166  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1167  for (Value *Data : CompilerUsedVars)
1168  if (!isa<Function>(Data))
1169  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1170  for (Value *Data : UsedVars)
1171  if (Data != NamesVar && !isa<Function>(Data))
1172  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1173 
1174  if (NamesVar) {
1175  Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1176  auto *NamesRegisterTy =
1177  FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
1178  auto *NamesRegisterF =
1181  IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
1182  IRB.getInt64(NamesSize)});
1183  }
1184 
1185  IRB.CreateRetVoid();
1186 }
1187 
1188 bool InstrProfiling::emitRuntimeHook() {
1189  // We expect the linker to be invoked with -u<hook_var> flag for Linux
1190  // in which case there is no need to emit the external variable.
1191  if (TT.isOSLinux())
1192  return false;
1193 
1194  // If the module's provided its own runtime, we don't need to do anything.
1195  if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1196  return false;
1197 
1198  // Declare an external variable that will pull in the runtime initialization.
1199  auto *Int32Ty = Type::getInt32Ty(M->getContext());
1200  auto *Var =
1202  nullptr, getInstrProfRuntimeHookVarName());
1203 
1204  if (TT.isOSBinFormatELF()) {
1205  // Mark the user variable as used so that it isn't stripped out.
1206  CompilerUsedVars.push_back(Var);
1207  } else {
1208  // Make a function that uses it.
1212  User->addFnAttr(Attribute::NoInline);
1213  if (Options.NoRedZone)
1214  User->addFnAttr(Attribute::NoRedZone);
1215  User->setVisibility(GlobalValue::HiddenVisibility);
1216  if (TT.supportsCOMDAT())
1217  User->setComdat(M->getOrInsertComdat(User->getName()));
1218 
1219  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1220  auto *Load = IRB.CreateLoad(Int32Ty, Var);
1221  IRB.CreateRet(Load);
1222 
1223  // Mark the function as used so that it isn't stripped out.
1224  CompilerUsedVars.push_back(User);
1225  }
1226  return true;
1227 }
1228 
1229 void InstrProfiling::emitUses() {
1230  // The metadata sections are parallel arrays. Optimizers (e.g.
1231  // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1232  // we conservatively retain all unconditionally in the compiler.
1233  //
1234  // On ELF and Mach-O, the linker can guarantee the associated sections will be
1235  // retained or discarded as a unit, so llvm.compiler.used is sufficient.
1236  // Similarly on COFF, if prof data is not referenced by code we use one comdat
1237  // and ensure this GC property as well. Otherwise, we have to conservatively
1238  // make all of the sections retained by the linker.
1239  if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
1241  appendToCompilerUsed(*M, CompilerUsedVars);
1242  else
1243  appendToUsed(*M, CompilerUsedVars);
1244 
1245  // We do not add proper references from used metadata sections to NamesVar and
1246  // VNodesVar, so we have to be conservative and place them in llvm.used
1247  // regardless of the target,
1248  appendToUsed(*M, UsedVars);
1249 }
1250 
1251 void InstrProfiling::emitInitialization() {
1252  // Create ProfileFileName variable. Don't don't this for the
1253  // context-sensitive instrumentation lowering: This lowering is after
1254  // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1255  // have already create the variable before LTO/ThinLTO linking.
1256  if (!IsCS)
1257  createProfileFileNameVar(*M, Options.InstrProfileOutput);
1258  Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1259  if (!RegisterF)
1260  return;
1261 
1262  // Create the initialization function.
1263  auto *VoidTy = Type::getVoidTy(M->getContext());
1264  auto *F = Function::Create(FunctionType::get(VoidTy, false),
1267  F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1268  F->addFnAttr(Attribute::NoInline);
1269  if (Options.NoRedZone)
1270  F->addFnAttr(Attribute::NoRedZone);
1271 
1272  // Add the basic block and the necessary calls.
1273  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1274  IRB.CreateCall(RegisterF, {});
1275  IRB.CreateRetVoid();
1276 
1277  appendToGlobalCtors(*M, F, 0);
1278 }
llvm::createInstrProfilingLegacyPass
ModulePass * createInstrProfilingLegacyPass(const InstrProfOptions &Options=InstrProfOptions(), bool IsCS=false)
Insert frontend instrumentation based profiling. Parameter IsCS indicates if.
Definition: InstrProfiling.cpp:443
INSTR_PROF_MIN_VAL_COUNTS
#define INSTR_PROF_MIN_VAL_COUNTS
i
i
Definition: README.txt:29
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
Int32Ty
IntegerType * Int32Ty
Definition: NVVMIntrRange.cpp:67
llvm::GlobalVariable::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:430
instrumentation
xray instrumentation
Definition: XRayInstrumentation.cpp:268
llvm::GlobalObject::setComdat
void setComdat(Comdat *C)
Definition: Globals.cpp:192
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AllocatorList.h:22
llvm::CallBase::getOperandBundlesAsDefs
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition: Instructions.cpp:375
llvm::InstrProfIncrementInst::getStep
Value * getStep() const
Definition: IntrinsicInst.cpp:193
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(InstrProfilingLegacyPass, "instrprof", "Frontend instrumentation-based coverage lowering.", false, false) INITIALIZE_PASS_END(InstrProfilingLegacyPass
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::AArch64CC::AL
@ AL
Definition: AArch64BaseInfo.h:269
llvm::LoopBase::getExitBlocks
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
Definition: LoopInfoImpl.h:62
needsRuntimeHookUnconditionally
static bool needsRuntimeHookUnconditionally(const Triple &TT)
Definition: InstrProfiling.cpp:527
llvm::StructType::get
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:408
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:293
llvm::getInstrProfNamesVarName
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:100
getVarName
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
Definition: InstrProfiling.cpp:763
llvm::ModulePass
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:238
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:107
IntrinsicInst.h
llvm::DIBuilder
Definition: DIBuilder.h:41
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:783
llvm::Triple::isOSBinFormatCOFF
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
Definition: Triple.h:621
llvm::cl::Prefix
@ Prefix
Definition: CommandLine.h:164
llvm::InstrProfCorrelator::NumCountersAttributeName
static const char * NumCountersAttributeName
Definition: InstrProfCorrelator.h:40
llvm::LoadAndStorePromoter
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater.
Definition: SSAUpdater.h:136
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition: GlobalValue.h:467
llvm::GlobalValue::HiddenVisibility
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
llvm::Function
Definition: Function.h:62
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:530
StringRef.h
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1353
llvm::NVPTX::LoadStore
LoadStore
Definition: NVPTX.h:99
Pass.h
llvm::InstrProfInstBase
A base class for all instrprof intrinsics.
Definition: IntrinsicInst.h:1180
Loops
Hexagon Hardware Loops
Definition: HexagonHardwareLoops.cpp:372
llvm::needsComdatForCounter
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1117
llvm::toString
std::string toString(Error E)
Write all error messages (if any) in E to a string.
Definition: Error.h:1028
llvm::Comdat::NoDeduplicate
@ NoDeduplicate
No deduplication is performed.
Definition: Comdat.h:39
llvm::getPGOFuncNameVarInitializer
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:483
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1175
llvm::Function::getSubprogram
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1541
ErrorHandling.h
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:707
llvm::Intrinsic::getName
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:880
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::ConstantExpr::getBitCast
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2253
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:363
Error.h
llvm::AtomicOrdering::SequentiallyConsistent
@ SequentiallyConsistent
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
FAM
FunctionAnalysisManager FAM
Definition: PassBuilderBindings.cpp:59
llvm::getInstrProfRuntimeHookVarName
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:148
llvm::MemOp
Definition: TargetLowering.h:111
InstrCount
static unsigned InstrCount
Definition: DFAPacketizer.cpp:53
llvm::PreservedAnalyses::none
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:158
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
containsProfilingIntrinsics
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
Definition: InstrProfiling.cpp:536
Module.h
llvm::reverse
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Definition: STLExtras.h:357
llvm::AttributeList
Definition: Attributes.h:403
llvm::TargetLibraryInfo::getExtAttrForI32Param
Attribute::AttrKind getExtAttrForI32Param(bool Signed=true) const
Returns extension attribute kind to be used for i32 parameters corresponding to C-level int or unsign...
Definition: TargetLibraryInfo.h:384
llvm::getInstrProfNamesRegFuncName
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:135
llvm::InstrProfInstBase::getIndex
ConstantInt * getIndex() const
Definition: IntrinsicInst.cpp:187
llvm::InstrProfCorrelator::CFGHashAttributeName
static const char * CFGHashAttributeName
Definition: InstrProfCorrelator.h:39
llvm::Triple::isOSBinFormatXCOFF
bool isOSBinFormatXCOFF() const
Tests whether the OS uses the XCOFF binary format.
Definition: Triple.h:639
llvm::GlobalValue::UnnamedAddr::Global
@ Global
llvm::InstrProfInstBase::getNumCounters
ConstantInt * getNumCounters() const
Definition: IntrinsicInst.cpp:181
llvm::GlobalValue::DefaultVisibility
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:63
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:419
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
llvm::Triple::isOSLinux
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:579
llvm::InstrProfInstBase::getHash
ConstantInt * getHash() const
Definition: IntrinsicInst.h:1188
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::GlobalValue::LinkageTypes
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
llvm::ArrayType
Class to represent array types.
Definition: DerivedTypes.h:357
llvm::createProfileFileNameVar
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1214
llvm::Data
@ Data
Definition: SIMachineScheduler.h:55
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:241
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1233
llvm::InstrProfiling
Instrumentation based profiling lowering pass.
Definition: InstrProfiling.h:35
F
#define F(x, y, z)
Definition: MD5.cpp:55
InstrProfData.inc
llvm::AtomicOrdering::Monotonic
@ Monotonic
getIntModuleFlagOrZero
static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag)
Definition: InstrProfiling.cpp:782
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::Annotations
Annotations lets you mark points and ranges inside source code, for tests:
Definition: Annotations.h:48
llvm::InstrProfValueProfileInst::getValueKind
ConstantInt * getValueKind() const
Definition: IntrinsicInst.h:1234
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ConstantExpr::getSub
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2721
llvm::StringRef::substr
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:611
llvm::ConstantPointerNull
A constant pointer value that points to null.
Definition: Constants.h:534
Instruction.h
CommandLine.h
llvm::AMDGPU::HSAMD::ValueKind
ValueKind
Value kinds.
Definition: AMDGPUMetadata.h:73
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition: BlockFrequencyInfo.h:37
llvm::Triple::isOSBinFormatELF
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:616
GlobalValue.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::GlobalValue::VisibilityTypes
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
Constants.h
InstrProfiling.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::getInstrProfValuesVarPrefix
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:93
llvm::DS_Warning
@ DS_Warning
Definition: DiagnosticInfo.h:47
llvm::GlobalObject::setSection
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:227
llvm::User
Definition: User.h:44
instrprof
instrprof
Definition: InstrProfiling.cpp:438
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::InstrProfCorrelator::FunctionNameAttributeName
static const char * FunctionNameAttributeName
Definition: InstrProfCorrelator.h:38
Twine.h
llvm::InstrProfValueProfileInst::getTargetValue
Value * getTargetValue() const
Definition: IntrinsicInst.h:1230
llvm::BranchProbabilityInfo
Analysis providing branch probability information.
Definition: BranchProbabilityInfo.h:115
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::ConstantExpr::getPtrToInt
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2225
llvm::canRenameComdatFunc
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1165
TargetLibraryInfo.h
llvm::getInstrProfCountersVarPrefix
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:90
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::Triple::supportsCOMDAT
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:816
llvm::BasicBlock::getFirstInsertionPt
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:253
false
Definition: StackSlotColoring.cpp:142
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:109
llvm::dwarf::toStringRef
StringRef toStringRef(const Optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
Definition: DWARFFormValue.h:191
llvm::getInstrProfInitFuncName
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:143
llvm::ConstantArray
ConstantArray - Constant Array Declarations.
Definition: Constants.h:409
llvm::Triple::isOSBinFormatMachO
bool isOSBinFormatMachO() const
Tests whether the environment is MachO.
Definition: Triple.h:629
llvm::Instruction
Definition: Instruction.h:45
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:146
InstrProf.h
llvm::Triple::getObjectFormat
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
Definition: Triple.h:337
llvm::appendToCompilerUsed
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
Definition: ModuleUtils.cpp:110
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:143
Options
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
Definition: PassBuilderBindings.cpp:48
llvm::LoopBase::getExitingBlocks
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
Definition: LoopInfoImpl.h:34
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:925
llvm::InstrProfValueProfileInst::getIndex
ConstantInt * getIndex() const
Definition: IntrinsicInst.h:1239
Align
uint64_t Align
Definition: ELFObjHandler.cpp:82
llvm::GlobalValue::InternalLinkage
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
llvm::Comdat
Definition: Comdat.h:33
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:153
llvm::Metadata
Root of the metadata hierarchy.
Definition: Metadata.h:62
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:747
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
Type.h
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
LoopInfo.h
llvm::Triple::isOSFuchsia
bool isOSFuchsia() const
Definition: Triple.h:498
llvm::getInstrProfVNodesVarName
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:96
llvm::cl::ZeroOrMore
@ ZeroOrMore
Definition: CommandLine.h:120
llvm::getInstrProfNameVarPrefix
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:84
BasicBlock.h
llvm::cl::opt< bool >
SSA
Memory SSA
Definition: MemorySSA.cpp:73
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition: GlobalVariable.h:135
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:78
BranchProbabilityInfo.h
llvm::InstrProfValueProfileInst
This represents the llvm.instrprof.value.profile intrinsic.
Definition: IntrinsicInst.h:1221
Index
uint32_t Index
Definition: ELFObjHandler.cpp:83
llvm::TargetLibraryInfoWrapperPass
Definition: TargetLibraryInfo.h:465
uint64_t
llvm::GlobalValue::getVisibility
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:229
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:79
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
llvm::ConstantPointerNull::get
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1761
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
profDataReferencedByCode
static bool profDataReferencedByCode(const Module &M)
Definition: InstrProfiling.cpp:798
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::DenseMap
Definition: DenseMap.h:714
shouldRecordFunctionAddr
static bool shouldRecordFunctionAddr(Function *F)
Definition: InstrProfiling.cpp:802
llvm::isIRPGOFlagSet
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1143
llvm::LoopInfoBase::getLoopFor
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:970
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::Type::getInt64PtrTy
static PointerType * getInt64PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:305
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:441
llvm::appendToUsed
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
Definition: ModuleUtils.cpp:106
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:585
DIBuilder.h
llvm::LoopBase::getLoopPreheader
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:167
llvm::GlobalValue::setLinkage
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:460
ArrayRef.h
llvm::InstrProfOptions
Options for the frontend instrumentation based profiling pass.
Definition: Instrumentation.h:121
llvm::MDString::get
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:473
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:139
llvm::DiagnosticInfoPGOProfile
Diagnostic information for the PGO profiler.
Definition: DiagnosticInfo.h:325
llvm::LoopBase::hasDedicatedExits
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
Definition: LoopInfoImpl.h:92
llvm::getInstrProfRuntimeHookVarUseFuncName
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:154
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::InstrProfInstBase::getName
GlobalVariable * getName() const
Definition: IntrinsicInst.h:1183
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:83
lowering
Frontend instrumentation based coverage lowering
Definition: InstrProfiling.cpp:439
llvm::ArrayType::get
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:640
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
llvm::collectPGOFuncNameStrings
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:442
llvm::X86II::PD
@ PD
Definition: X86BaseInfo.h:787
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:650
Triple.h
llvm::getInstrProfRegFuncsName
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:124
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::LoopInfo
Definition: LoopInfo.h:1086
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1599
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:253
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
if
if(llvm_vc STREQUAL "") set(fake_version_inc "$
Definition: CMakeLists.txt:14
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:70
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:100
uint32_t
llvm::ConstantDataArray::getString
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:3086
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
SSAUpdater.h
BlockFrequencyInfo.h
llvm::AMDGPUISD::BFI
@ BFI
Definition: AMDGPUISelLowering.h:431
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:180
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:152
llvm::Init
Definition: Record.h:274
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
llvm::BasicBlock::front
const Instruction & front() const
Definition: BasicBlock.h:308
llvm::DoInstrProfNameCompression
cl::opt< bool > DoInstrProfNameCompression
llvm::LoopInfoBase::getLoopsInPreorder
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
Definition: LoopInfoImpl.h:578
Attributes.h
llvm::StringRef::size
constexpr LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:157
Constant.h
llvm::LLVMContext::diagnose
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Definition: LLVMContext.cpp:228
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:242
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:348
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
NC
#define NC
Definition: regutils.h:42
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
llvm::Error
Lightweight error class with error context and mandatory checking.
Definition: Error.h:155
GlobalVariable.h
getOrInsertValueProfilingCall
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
Definition: InstrProfiling.cpp:608
Casting.h
DiagnosticInfo.h
Function.h
enablesValueProfiling
static bool enablesValueProfiling(const Module &M)
Definition: InstrProfiling.cpp:792
llvm::TargetLibraryInfo
Provides information about what library functions are available for the current target.
Definition: TargetLibraryInfo.h:221
llvm::TargetStackID::Default
@ Default
Definition: TargetFrameLowering.h:28
llvm::InstrProfiling::run
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition: InstrProfiling.cpp:421
llvm::MCID::Add
@ Add
Definition: MCInstrDesc.h:183
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:474
llvm::InstrProfIncrementInst
This represents the llvm.instrprof.increment intrinsic.
Definition: IntrinsicInst.h:1198
llvm::getInstrProfSectionName
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:216
llvm::GlobalValue::ExternalLinkage
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:224
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Instructions.h
llvm::getInstrProfValueProfMemOpFuncName
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:79
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
needsRuntimeRegistrationOfSectionRange
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
Definition: InstrProfiling.cpp:839
SmallVector.h
Dominators.h
llvm::getCoverageUnusedNamesVarName
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:115
ModuleUtils.h
llvm::getInstrProfCounterBiasVarName
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:158
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:340
llvm::FunctionCallee
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
llvm::SmallVectorImpl< BasicBlock * >
InstrProfCorrelator.h
llvm::Type::getInt16Ty
static IntegerType * getInt16Ty(LLVMContext &C)
Definition: Type.cpp:240
DerivedTypes.h
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition: GlobalValue.h:273
llvm::InnerAnalysisManagerProxy
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:940
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1478
llvm::getInstrProfValueProfFuncName
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:73
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:389
llvm::appendToGlobalCtors
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:66
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::getInstrProfDataVarPrefix
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:87
llvm::cl::desc
Definition: CommandLine.h:412
llvm::GlobalObject::setAlignment
void setAlignment(MaybeAlign Align)
Definition: Globals.cpp:124
llvm::SSAUpdater
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:38
BasicBlockUtils.h
llvm::initializeInstrProfilingLegacyPassPass
void initializeInstrProfilingLegacyPassPass(PassRegistry &)
llvm::GlobalValue::LinkOnceODRLinkage
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
llvm::GlobalValue::setVisibility
void setVisibility(VisibilityTypes V)
Definition: GlobalValue.h:235
InitializePasses.h
llvm::DebugInfoCorrelate
cl::opt< bool > DebugInfoCorrelate("debug-info-correlate", cl::ZeroOrMore, cl::desc("Use debug info to correlate profiles."), cl::init(false))
Definition: PGOInstrumentation.cpp:295
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::getInstrProfRegFuncName
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:130
llvm::TargetLibraryAnalysis
Analysis pass providing the TargetLibraryInfo.
Definition: TargetLibraryInfo.h:440
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:364
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38