LLVM  14.0.0git
InstrProfiling.cpp
Go to the documentation of this file.
1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10 // It also builds the data structures and initialization code needed for
11 // updating execution counts and emitting the profile at runtime.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/ADT/Twine.h"
23 #include "llvm/Analysis/LoopInfo.h"
25 #include "llvm/IR/Attributes.h"
26 #include "llvm/IR/BasicBlock.h"
27 #include "llvm/IR/Constant.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DerivedTypes.h"
30 #include "llvm/IR/Dominators.h"
31 #include "llvm/IR/Function.h"
32 #include "llvm/IR/GlobalValue.h"
33 #include "llvm/IR/GlobalVariable.h"
34 #include "llvm/IR/IRBuilder.h"
35 #include "llvm/IR/Instruction.h"
36 #include "llvm/IR/Instructions.h"
37 #include "llvm/IR/IntrinsicInst.h"
38 #include "llvm/IR/Module.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/InitializePasses.h"
41 #include "llvm/Pass.h"
43 #include "llvm/Support/Casting.h"
45 #include "llvm/Support/Error.h"
50 #include <algorithm>
51 #include <cassert>
52 #include <cstddef>
53 #include <cstdint>
54 #include <string>
55 
56 using namespace llvm;
57 
58 #define DEBUG_TYPE "instrprof"
59 
60 namespace {
61 
62 cl::opt<bool> DoHashBasedCounterSplit(
63  "hash-based-counter-split",
64  cl::desc("Rename counter variable of a comdat function based on cfg hash"),
65  cl::init(true));
66 
67 cl::opt<bool> RuntimeCounterRelocation(
68  "runtime-counter-relocation",
69  cl::desc("Enable relocating counters at runtime."),
70  cl::init(false));
71 
72 cl::opt<bool> ValueProfileStaticAlloc(
73  "vp-static-alloc",
74  cl::desc("Do static counter allocation for value profiler"),
75  cl::init(true));
76 
77 cl::opt<double> NumCountersPerValueSite(
78  "vp-counters-per-site",
79  cl::desc("The average number of profile counters allocated "
80  "per value profiling site."),
81  // This is set to a very small value because in real programs, only
82  // a very small percentage of value sites have non-zero targets, e.g, 1/30.
83  // For those sites with non-zero profile, the average number of targets
84  // is usually smaller than 2.
85  cl::init(1.0));
86 
87 cl::opt<bool> AtomicCounterUpdateAll(
88  "instrprof-atomic-counter-update-all", cl::ZeroOrMore,
89  cl::desc("Make all profile counter updates atomic (for testing only)"),
90  cl::init(false));
91 
92 cl::opt<bool> AtomicCounterUpdatePromoted(
93  "atomic-counter-update-promoted", cl::ZeroOrMore,
94  cl::desc("Do counter update using atomic fetch add "
95  " for promoted counters only"),
96  cl::init(false));
97 
98 cl::opt<bool> AtomicFirstCounter(
99  "atomic-first-counter", cl::ZeroOrMore,
100  cl::desc("Use atomic fetch add for first counter in a function (usually "
101  "the entry counter)"),
102  cl::init(false));
103 
104 // If the option is not specified, the default behavior about whether
105 // counter promotion is done depends on how instrumentaiton lowering
106 // pipeline is setup, i.e., the default value of true of this option
107 // does not mean the promotion will be done by default. Explicitly
108 // setting this option can override the default behavior.
109 cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
110  cl::desc("Do counter register promotion"),
111  cl::init(false));
112 cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
113  cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20),
114  cl::desc("Max number counter promotions per loop to avoid"
115  " increasing register pressure too much"));
116 
117 // A debug option
119  MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
120  cl::desc("Max number of allowed counter promotions"));
121 
122 cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
123  cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3),
124  cl::desc("The max number of exiting blocks of a loop to allow "
125  " speculative counter promotion"));
126 
127 cl::opt<bool> SpeculativeCounterPromotionToLoop(
128  cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false),
129  cl::desc("When the option is false, if the target block is in a loop, "
130  "the promotion will be disallowed unless the promoted counter "
131  " update can be further/iteratively promoted into an acyclic "
132  " region."));
133 
134 cl::opt<bool> IterativeCounterPromotion(
135  cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
136  cl::desc("Allow counter promotion across the whole loop nest."));
137 
138 cl::opt<bool> SkipRetExitBlock(
139  cl::ZeroOrMore, "skip-ret-exit-block", cl::init(true),
140  cl::desc("Suppress counter promotion if exit blocks contain ret."));
141 
142 class InstrProfilingLegacyPass : public ModulePass {
143  InstrProfiling InstrProf;
144 
145 public:
146  static char ID;
147 
148  InstrProfilingLegacyPass() : ModulePass(ID) {}
149  InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false)
150  : ModulePass(ID), InstrProf(Options, IsCS) {
152  }
153 
154  StringRef getPassName() const override {
155  return "Frontend instrumentation-based coverage lowering";
156  }
157 
158  bool runOnModule(Module &M) override {
159  auto GetTLI = [this](Function &F) -> TargetLibraryInfo & {
160  return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
161  };
162  return InstrProf.run(M, GetTLI);
163  }
164 
165  void getAnalysisUsage(AnalysisUsage &AU) const override {
166  AU.setPreservesCFG();
168  }
169 };
170 
171 ///
172 /// A helper class to promote one counter RMW operation in the loop
173 /// into register update.
174 ///
175 /// RWM update for the counter will be sinked out of the loop after
176 /// the transformation.
177 ///
178 class PGOCounterPromoterHelper : public LoadAndStorePromoter {
179 public:
180  PGOCounterPromoterHelper(
182  BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
183  ArrayRef<Instruction *> InsertPts,
185  LoopInfo &LI)
186  : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
187  InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
188  assert(isa<LoadInst>(L));
189  assert(isa<StoreInst>(S));
190  SSA.AddAvailableValue(PH, Init);
191  }
192 
193  void doExtraRewritesBeforeFinalDeletion() override {
194  for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
195  BasicBlock *ExitBlock = ExitBlocks[i];
196  Instruction *InsertPos = InsertPts[i];
197  // Get LiveIn value into the ExitBlock. If there are multiple
198  // predecessors, the value is defined by a PHI node in this
199  // block.
200  Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
201  Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
202  Type *Ty = LiveInValue->getType();
203  IRBuilder<> Builder(InsertPos);
204  if (AtomicCounterUpdatePromoted)
205  // automic update currently can only be promoted across the current
206  // loop, not the whole loop nest.
207  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
208  MaybeAlign(),
210  else {
211  LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
212  auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
213  auto *NewStore = Builder.CreateStore(NewVal, Addr);
214 
215  // Now update the parent loop's candidate list:
216  if (IterativeCounterPromotion) {
217  auto *TargetLoop = LI.getLoopFor(ExitBlock);
218  if (TargetLoop)
219  LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
220  }
221  }
222  }
223  }
224 
225 private:
227  ArrayRef<BasicBlock *> ExitBlocks;
228  ArrayRef<Instruction *> InsertPts;
230  LoopInfo &LI;
231 };
232 
233 /// A helper class to do register promotion for all profile counter
234 /// updates in a loop.
235 ///
236 class PGOCounterPromoter {
237 public:
238  PGOCounterPromoter(
240  Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
241  : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop),
242  LI(LI), BFI(BFI) {
243 
244  // Skip collection of ExitBlocks and InsertPts for loops that will not be
245  // able to have counters promoted.
246  SmallVector<BasicBlock *, 8> LoopExitBlocks;
248 
249  L.getExitBlocks(LoopExitBlocks);
250  if (!isPromotionPossible(&L, LoopExitBlocks))
251  return;
252 
253  for (BasicBlock *ExitBlock : LoopExitBlocks) {
254  if (BlockSet.insert(ExitBlock).second) {
255  ExitBlocks.push_back(ExitBlock);
256  InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
257  }
258  }
259  }
260 
261  bool run(int64_t *NumPromoted) {
262  // Skip 'infinite' loops:
263  if (ExitBlocks.size() == 0)
264  return false;
265 
266  // Skip if any of the ExitBlocks contains a ret instruction.
267  // This is to prevent dumping of incomplete profile -- if the
268  // the loop is a long running loop and dump is called in the middle
269  // of the loop, the result profile is incomplete.
270  // FIXME: add other heuristics to detect long running loops.
271  if (SkipRetExitBlock) {
272  for (auto BB : ExitBlocks)
273  if (isa<ReturnInst>(BB->getTerminator()))
274  return false;
275  }
276 
277  unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
278  if (MaxProm == 0)
279  return false;
280 
281  unsigned Promoted = 0;
282  for (auto &Cand : LoopToCandidates[&L]) {
283 
285  SSAUpdater SSA(&NewPHIs);
286  Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
287 
288  // If BFI is set, we will use it to guide the promotions.
289  if (BFI) {
290  auto *BB = Cand.first->getParent();
291  auto InstrCount = BFI->getBlockProfileCount(BB);
292  if (!InstrCount)
293  continue;
294  auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
295  // If the average loop trip count is not greater than 1.5, we skip
296  // promotion.
297  if (PreheaderCount &&
298  (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2))
299  continue;
300  }
301 
302  PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
303  L.getLoopPreheader(), ExitBlocks,
304  InsertPts, LoopToCandidates, LI);
305  Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
306  Promoted++;
307  if (Promoted >= MaxProm)
308  break;
309 
310  (*NumPromoted)++;
311  if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
312  break;
313  }
314 
315  LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
316  << L.getLoopDepth() << ")\n");
317  return Promoted != 0;
318  }
319 
320 private:
321  bool allowSpeculativeCounterPromotion(Loop *LP) {
322  SmallVector<BasicBlock *, 8> ExitingBlocks;
323  L.getExitingBlocks(ExitingBlocks);
324  // Not considierered speculative.
325  if (ExitingBlocks.size() == 1)
326  return true;
327  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
328  return false;
329  return true;
330  }
331 
332  // Check whether the loop satisfies the basic conditions needed to perform
333  // Counter Promotions.
334  bool isPromotionPossible(Loop *LP,
335  const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
336  // We can't insert into a catchswitch.
337  if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
338  return isa<CatchSwitchInst>(Exit->getTerminator());
339  }))
340  return false;
341 
342  if (!LP->hasDedicatedExits())
343  return false;
344 
345  BasicBlock *PH = LP->getLoopPreheader();
346  if (!PH)
347  return false;
348 
349  return true;
350  }
351 
352  // Returns the max number of Counter Promotions for LP.
353  unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
354  SmallVector<BasicBlock *, 8> LoopExitBlocks;
355  LP->getExitBlocks(LoopExitBlocks);
356  if (!isPromotionPossible(LP, LoopExitBlocks))
357  return 0;
358 
359  SmallVector<BasicBlock *, 8> ExitingBlocks;
360  LP->getExitingBlocks(ExitingBlocks);
361 
362  // If BFI is set, we do more aggressive promotions based on BFI.
363  if (BFI)
364  return (unsigned)-1;
365 
366  // Not considierered speculative.
367  if (ExitingBlocks.size() == 1)
368  return MaxNumOfPromotionsPerLoop;
369 
370  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
371  return 0;
372 
373  // Whether the target block is in a loop does not matter:
374  if (SpeculativeCounterPromotionToLoop)
375  return MaxNumOfPromotionsPerLoop;
376 
377  // Now check the target block:
378  unsigned MaxProm = MaxNumOfPromotionsPerLoop;
379  for (auto *TargetBlock : LoopExitBlocks) {
380  auto *TargetLoop = LI.getLoopFor(TargetBlock);
381  if (!TargetLoop)
382  continue;
383  unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
384  unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
385  MaxProm =
386  std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
387  PendingCandsInTarget);
388  }
389  return MaxProm;
390  }
391 
393  SmallVector<BasicBlock *, 8> ExitBlocks;
395  Loop &L;
396  LoopInfo &LI;
398 };
399 
400 enum class ValueProfilingCallType {
401  // Individual values are tracked. Currently used for indiret call target
402  // profiling.
403  Default,
404 
405  // MemOp: the memop size value profiling.
406  MemOp
407 };
408 
409 } // end anonymous namespace
410 
414  auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
416  };
417  if (!run(M, GetTLI))
418  return PreservedAnalyses::all();
419 
420  return PreservedAnalyses::none();
421 }
422 
425  InstrProfilingLegacyPass, "instrprof",
426  "Frontend instrumentation-based coverage lowering.", false, false)
429  InstrProfilingLegacyPass, "instrprof",
430  "Frontend instrumentation-based coverage lowering.", false, false)
431 
432 ModulePass *
434  bool IsCS) {
435  return new InstrProfilingLegacyPass(Options, IsCS);
436 }
437 
439  InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
440  if (Inc)
441  return Inc;
442  return dyn_cast<InstrProfIncrementInst>(Instr);
443 }
444 
445 bool InstrProfiling::lowerIntrinsics(Function *F) {
446  bool MadeChange = false;
447  PromotionCandidates.clear();
448  for (BasicBlock &BB : *F) {
449  for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
451  if (Inc) {
452  lowerIncrement(Inc);
453  MadeChange = true;
454  } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
455  lowerValueProfileInst(Ind);
456  MadeChange = true;
457  }
458  }
459  }
460 
461  if (!MadeChange)
462  return false;
463 
464  promoteCounterLoadStores(F);
465  return true;
466 }
467 
468 bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
469  // Mach-O don't support weak external references.
470  if (TT.isOSBinFormatMachO())
471  return false;
472 
473  if (RuntimeCounterRelocation.getNumOccurrences() > 0)
474  return RuntimeCounterRelocation;
475 
476  // Fuchsia uses runtime counter relocation by default.
477  return TT.isOSFuchsia();
478 }
479 
480 bool InstrProfiling::isCounterPromotionEnabled() const {
481  if (DoCounterPromotion.getNumOccurrences() > 0)
482  return DoCounterPromotion;
483 
484  return Options.DoCounterPromotion;
485 }
486 
487 void InstrProfiling::promoteCounterLoadStores(Function *F) {
488  if (!isCounterPromotionEnabled())
489  return;
490 
491  DominatorTree DT(*F);
492  LoopInfo LI(DT);
493  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
494 
495  std::unique_ptr<BlockFrequencyInfo> BFI;
496  if (Options.UseBFIInPromotion) {
497  std::unique_ptr<BranchProbabilityInfo> BPI;
498  BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
499  BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
500  }
501 
502  for (const auto &LoadStore : PromotionCandidates) {
503  auto *CounterLoad = LoadStore.first;
504  auto *CounterStore = LoadStore.second;
505  BasicBlock *BB = CounterLoad->getParent();
506  Loop *ParentLoop = LI.getLoopFor(BB);
507  if (!ParentLoop)
508  continue;
509  LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
510  }
511 
512  SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
513 
514  // Do a post-order traversal of the loops so that counter updates can be
515  // iteratively hoisted outside the loop nest.
516  for (auto *Loop : llvm::reverse(Loops)) {
517  PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
518  Promoter.run(&TotalCountersPromoted);
519  }
520 }
521 
522 static bool needsRuntimeHookUnconditionally(const Triple &TT) {
523  // On Fuchsia, we only need runtime hook if any counters are present.
524  if (TT.isOSFuchsia())
525  return false;
526 
527  return true;
528 }
529 
530 /// Check if the module contains uses of any profiling intrinsics.
532  if (auto *F = M.getFunction(
533  Intrinsic::getName(llvm::Intrinsic::instrprof_increment)))
534  if (!F->use_empty())
535  return true;
536  if (auto *F = M.getFunction(
537  Intrinsic::getName(llvm::Intrinsic::instrprof_increment_step)))
538  if (!F->use_empty())
539  return true;
540  if (auto *F = M.getFunction(
541  Intrinsic::getName(llvm::Intrinsic::instrprof_value_profile)))
542  if (!F->use_empty())
543  return true;
544  return false;
545 }
546 
548  Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
549  this->M = &M;
550  this->GetTLI = std::move(GetTLI);
551  NamesVar = nullptr;
552  NamesSize = 0;
553  ProfileDataMap.clear();
554  CompilerUsedVars.clear();
555  UsedVars.clear();
556  TT = Triple(M.getTargetTriple());
557 
558  bool MadeChange = false;
559 
560  // Emit the runtime hook even if no counters are present.
562  MadeChange = emitRuntimeHook();
563 
564  // Improve compile time by avoiding linear scans when there is no work.
565  GlobalVariable *CoverageNamesVar =
566  M.getNamedGlobal(getCoverageUnusedNamesVarName());
567  if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
568  return MadeChange;
569 
570  // We did not know how many value sites there would be inside
571  // the instrumented function. This is counting the number of instrumented
572  // target value sites to enter it as field in the profile data variable.
573  for (Function &F : M) {
574  InstrProfIncrementInst *FirstProfIncInst = nullptr;
575  for (BasicBlock &BB : F)
576  for (auto I = BB.begin(), E = BB.end(); I != E; I++)
577  if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
578  computeNumValueSiteCounts(Ind);
579  else if (FirstProfIncInst == nullptr)
580  FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
581 
582  // Value profiling intrinsic lowering requires per-function profile data
583  // variable to be created first.
584  if (FirstProfIncInst != nullptr)
585  static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
586  }
587 
588  for (Function &F : M)
589  MadeChange |= lowerIntrinsics(&F);
590 
591  if (CoverageNamesVar) {
592  lowerCoverageData(CoverageNamesVar);
593  MadeChange = true;
594  }
595 
596  if (!MadeChange)
597  return false;
598 
599  emitVNodes();
600  emitNameData();
601  emitRuntimeHook();
602  emitRegistration();
603  emitUses();
604  emitInitialization();
605  return true;
606 }
607 
609  Module &M, const TargetLibraryInfo &TLI,
610  ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
611  LLVMContext &Ctx = M.getContext();
612  auto *ReturnTy = Type::getVoidTy(M.getContext());
613 
615  if (auto AK = TLI.getExtAttrForI32Param(false))
616  AL = AL.addParamAttribute(M.getContext(), 2, AK);
617 
619  CallType == ValueProfilingCallType::MemOp) &&
620  "Must be Default or MemOp");
621  Type *ParamTypes[] = {
622 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
624  };
625  auto *ValueProfilingCallTy =
626  FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
627  StringRef FuncName = CallType == ValueProfilingCallType::Default
630  return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
631 }
632 
633 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
634  GlobalVariable *Name = Ind->getName();
636  uint64_t Index = Ind->getIndex()->getZExtValue();
637  auto It = ProfileDataMap.find(Name);
638  if (It == ProfileDataMap.end()) {
639  PerFunctionProfileData PD;
640  PD.NumValueSites[ValueKind] = Index + 1;
641  ProfileDataMap[Name] = PD;
642  } else if (It->second.NumValueSites[ValueKind] <= Index)
643  It->second.NumValueSites[ValueKind] = Index + 1;
644 }
645 
646 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
647  GlobalVariable *Name = Ind->getName();
648  auto It = ProfileDataMap.find(Name);
649  assert(It != ProfileDataMap.end() && It->second.DataVar &&
650  "value profiling detected in function with no counter incerement");
651 
652  GlobalVariable *DataVar = It->second.DataVar;
654  uint64_t Index = Ind->getIndex()->getZExtValue();
655  for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
656  Index += It->second.NumValueSites[Kind];
657 
658  IRBuilder<> Builder(Ind);
659  bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
660  llvm::InstrProfValueKind::IPVK_MemOPSize);
661  CallInst *Call = nullptr;
662  auto *TLI = &GetTLI(*Ind->getFunction());
663 
664  // To support value profiling calls within Windows exception handlers, funclet
665  // information contained within operand bundles needs to be copied over to
666  // the library call. This is required for the IR to be processed by the
667  // WinEHPrepare pass.
669  Ind->getOperandBundlesAsDefs(OpBundles);
670  if (!IsMemOpSize) {
671  Value *Args[3] = {Ind->getTargetValue(),
672  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
673  Builder.getInt32(Index)};
674  Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
675  OpBundles);
676  } else {
677  Value *Args[3] = {Ind->getTargetValue(),
678  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
679  Builder.getInt32(Index)};
680  Call = Builder.CreateCall(
681  getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
682  Args, OpBundles);
683  }
684  if (auto AK = TLI->getExtAttrForI32Param(false))
685  Call->addParamAttr(2, AK);
686  Ind->replaceAllUsesWith(Call);
687  Ind->eraseFromParent();
688 }
689 
690 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
691  GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
692 
693  IRBuilder<> Builder(Inc);
694  uint64_t Index = Inc->getIndex()->getZExtValue();
695  Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(),
696  Counters, 0, Index);
697 
698  if (isRuntimeCounterRelocationEnabled()) {
699  Type *Int64Ty = Type::getInt64Ty(M->getContext());
700  Type *Int64PtrTy = Type::getInt64PtrTy(M->getContext());
701  Function *Fn = Inc->getParent()->getParent();
702  Instruction &I = Fn->getEntryBlock().front();
703  LoadInst *LI = dyn_cast<LoadInst>(&I);
704  if (!LI) {
706  GlobalVariable *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
707  if (!Bias) {
708  // Compiler must define this variable when runtime counter relocation
709  // is being used. Runtime has a weak external reference that is used
710  // to check whether that's the case or not.
711  Bias = new GlobalVariable(*M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
712  Constant::getNullValue(Int64Ty),
715  // A definition that's weak (linkonce_odr) without being in a COMDAT
716  // section wouldn't lead to link errors, but it would lead to a dead
717  // data word from every TU but one. Putting it in COMDAT ensures there
718  // will be exactly one data slot in the link.
719  if (TT.supportsCOMDAT())
720  Bias->setComdat(M->getOrInsertComdat(Bias->getName()));
721  }
722  LI = Builder.CreateLoad(Int64Ty, Bias);
723  }
724  auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), LI);
725  Addr = Builder.CreateIntToPtr(Add, Int64PtrTy);
726  }
727 
728  if (Options.Atomic || AtomicCounterUpdateAll ||
729  (Index == 0 && AtomicFirstCounter)) {
730  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
732  } else {
733  Value *IncStep = Inc->getStep();
734  Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
735  auto *Count = Builder.CreateAdd(Load, Inc->getStep());
736  auto *Store = Builder.CreateStore(Count, Addr);
737  if (isCounterPromotionEnabled())
738  PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
739  }
740  Inc->eraseFromParent();
741 }
742 
743 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
744  ConstantArray *Names =
745  cast<ConstantArray>(CoverageNamesVar->getInitializer());
746  for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
747  Constant *NC = Names->getOperand(I);
748  Value *V = NC->stripPointerCasts();
749  assert(isa<GlobalVariable>(V) && "Missing reference to function name");
750  GlobalVariable *Name = cast<GlobalVariable>(V);
751 
752  Name->setLinkage(GlobalValue::PrivateLinkage);
753  ReferencedNames.push_back(Name);
754  NC->dropAllReferences();
755  }
756  CoverageNamesVar->eraseFromParent();
757 }
758 
759 /// Get the name of a profiling variable for a particular function.
761  bool &Renamed) {
762  StringRef NamePrefix = getInstrProfNameVarPrefix();
763  StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
764  Function *F = Inc->getParent()->getParent();
765  Module *M = F->getParent();
766  if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
767  !canRenameComdatFunc(*F)) {
768  Renamed = false;
769  return (Prefix + Name).str();
770  }
771  Renamed = true;
772  uint64_t FuncHash = Inc->getHash()->getZExtValue();
773  SmallVector<char, 24> HashPostfix;
774  if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
775  return (Prefix + Name).str();
776  return (Prefix + Name + "." + Twine(FuncHash)).str();
777 }
778 
780  auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
781  if (!MD)
782  return 0;
783 
784  // If the flag is a ConstantAsMetadata, it should be an integer representable
785  // in 64-bits.
786  return cast<ConstantInt>(MD->getValue())->getZExtValue();
787 }
788 
789 static bool enablesValueProfiling(const Module &M) {
790  return isIRPGOFlagSet(&M) ||
791  getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
792 }
793 
794 // Conservatively returns true if data variables may be referenced by code.
795 static bool profDataReferencedByCode(const Module &M) {
796  return enablesValueProfiling(M);
797 }
798 
799 static inline bool shouldRecordFunctionAddr(Function *F) {
800  // Only record function addresses if IR PGO is enabled or if clang value
801  // profiling is enabled. Recording function addresses greatly increases object
802  // file size, because it prevents the inliner from deleting functions that
803  // have been inlined everywhere.
804  if (!profDataReferencedByCode(*F->getParent()))
805  return false;
806 
807  // Check the linkage
808  bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
809  if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
810  !HasAvailableExternallyLinkage)
811  return true;
812 
813  // A function marked 'alwaysinline' with available_externally linkage can't
814  // have its address taken. Doing so would create an undefined external ref to
815  // the function, which would fail to link.
816  if (HasAvailableExternallyLinkage &&
817  F->hasFnAttribute(Attribute::AlwaysInline))
818  return false;
819 
820  // Prohibit function address recording if the function is both internal and
821  // COMDAT. This avoids the profile data variable referencing internal symbols
822  // in COMDAT.
823  if (F->hasLocalLinkage() && F->hasComdat())
824  return false;
825 
826  // Check uses of this function for other than direct calls or invokes to it.
827  // Inline virtual functions have linkeOnceODR linkage. When a key method
828  // exists, the vtable will only be emitted in the TU where the key method
829  // is defined. In a TU where vtable is not available, the function won't
830  // be 'addresstaken'. If its address is not recorded here, the profile data
831  // with missing address may be picked by the linker leading to missing
832  // indirect call target info.
833  return F->hasAddressTaken() || F->hasLinkOnceLinkage();
834 }
835 
837  // Don't do this for Darwin. compiler-rt uses linker magic.
838  if (TT.isOSDarwin())
839  return false;
840  // Use linker script magic to get data/cnts/name start/end.
841  if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
842  TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS4CPU() ||
843  TT.isOSWindows())
844  return false;
845 
846  return true;
847 }
848 
850 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
851  GlobalVariable *NamePtr = Inc->getName();
852  auto It = ProfileDataMap.find(NamePtr);
853  PerFunctionProfileData PD;
854  if (It != ProfileDataMap.end()) {
855  if (It->second.RegionCounters)
856  return It->second.RegionCounters;
857  PD = It->second;
858  }
859 
860  // Match the linkage and visibility of the name global.
861  Function *Fn = Inc->getParent()->getParent();
863  GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
864 
865  // Move the name variable to the right section. Place them in a COMDAT group
866  // if the associated function is a COMDAT. This will make sure that only one
867  // copy of counters of the COMDAT function will be emitted after linking. Keep
868  // in mind that this pass may run before the inliner, so we need to create a
869  // new comdat group for the counters and profiling data. If we use the comdat
870  // of the parent function, that will result in relocations against discarded
871  // sections.
872  //
873  // If the data variable is referenced by code, counters and data have to be
874  // in different comdats for COFF because the Visual C++ linker will report
875  // duplicate symbol errors if there are multiple external symbols with the
876  // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
877  //
878  // For ELF, when not using COMDAT, put counters, data and values into a
879  // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
880  // allows -z start-stop-gc to discard the entire group when the function is
881  // discarded.
882  bool DataReferencedByCode = profDataReferencedByCode(*M);
883  bool NeedComdat = needsComdatForCounter(*Fn, *M);
884  bool Renamed;
885  std::string CntsVarName =
886  getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);
887  std::string DataVarName =
888  getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
889  auto MaybeSetComdat = [&](GlobalVariable *GV) {
890  bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
891  if (UseComdat) {
892  StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
893  ? GV->getName()
894  : CntsVarName;
895  Comdat *C = M->getOrInsertComdat(GroupName);
896  if (!NeedComdat)
897  C->setSelectionKind(Comdat::NoDeduplicate);
898  GV->setComdat(C);
899  }
900  };
901 
902  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
903  LLVMContext &Ctx = M->getContext();
904  ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
905 
906  // Create the counters variable.
907  auto *CounterPtr =
908  new GlobalVariable(*M, CounterTy, false, Linkage,
909  Constant::getNullValue(CounterTy), CntsVarName);
910  CounterPtr->setVisibility(Visibility);
911  CounterPtr->setSection(
912  getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
913  CounterPtr->setAlignment(Align(8));
914  MaybeSetComdat(CounterPtr);
915  CounterPtr->setLinkage(Linkage);
916 
917  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
918  // Allocate statically the array of pointers to value profile nodes for
919  // the current function.
920  Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
921  uint64_t NS = 0;
922  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
923  NS += PD.NumValueSites[Kind];
924  if (NS > 0 && ValueProfileStaticAlloc &&
926  ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
927  auto *ValuesVar = new GlobalVariable(
928  *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
929  getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
930  ValuesVar->setVisibility(Visibility);
931  ValuesVar->setSection(
932  getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
933  ValuesVar->setAlignment(Align(8));
934  MaybeSetComdat(ValuesVar);
935  ValuesPtrExpr =
937  }
938 
939  // Create data variable.
940  auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
941  auto *Int16Ty = Type::getInt16Ty(Ctx);
942  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
943  Type *DataTypes[] = {
944 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
946  };
947  auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
948 
949  Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
950  ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
951  : ConstantPointerNull::get(Int8PtrTy);
952 
953  Constant *Int16ArrayVals[IPVK_Last + 1];
954  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
955  Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
956 
957  // If the data variable is not referenced by code (if we don't emit
958  // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
959  // data variable live under linker GC, the data variable can be private. This
960  // optimization applies to ELF.
961  //
962  // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
963  // to be false.
964  //
965  // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
966  // that other copies must have the same CFG and cannot have value profiling.
967  // If no hash suffix, other profd copies may be referenced by code.
968  if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
969  (TT.isOSBinFormatELF() ||
970  (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
972  Visibility = GlobalValue::DefaultVisibility;
973  }
974  auto *Data =
975  new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName);
976  // Reference the counter variable with a label difference (link-time
977  // constant).
978  auto *RelativeCounterPtr =
979  ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
980  ConstantExpr::getPtrToInt(Data, IntPtrTy));
981 
982  Constant *DataVals[] = {
983 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
985  };
986  Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
987 
988  Data->setVisibility(Visibility);
989  Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
990  Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
991  MaybeSetComdat(Data);
992  Data->setLinkage(Linkage);
993 
994  PD.RegionCounters = CounterPtr;
995  PD.DataVar = Data;
996  ProfileDataMap[NamePtr] = PD;
997 
998  // Mark the data variable as used so that it isn't stripped out.
999  CompilerUsedVars.push_back(Data);
1000  // Now that the linkage set by the FE has been passed to the data and counter
1001  // variables, reset Name variable's linkage and visibility to private so that
1002  // it can be removed later by the compiler.
1004  // Collect the referenced names to be used by emitNameData.
1005  ReferencedNames.push_back(NamePtr);
1006 
1007  return CounterPtr;
1008 }
1009 
1010 void InstrProfiling::emitVNodes() {
1011  if (!ValueProfileStaticAlloc)
1012  return;
1013 
1014  // For now only support this on platforms that do
1015  // not require runtime registration to discover
1016  // named section start/end.
1018  return;
1019 
1020  size_t TotalNS = 0;
1021  for (auto &PD : ProfileDataMap) {
1022  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1023  TotalNS += PD.second.NumValueSites[Kind];
1024  }
1025 
1026  if (!TotalNS)
1027  return;
1028 
1029  uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1030 // Heuristic for small programs with very few total value sites.
1031 // The default value of vp-counters-per-site is chosen based on
1032 // the observation that large apps usually have a low percentage
1033 // of value sites that actually have any profile data, and thus
1034 // the average number of counters per site is low. For small
1035 // apps with very few sites, this may not be true. Bump up the
1036 // number of counters in this case.
1037 #define INSTR_PROF_MIN_VAL_COUNTS 10
1038  if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
1039  NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
1040 
1041  auto &Ctx = M->getContext();
1042  Type *VNodeTypes[] = {
1043 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1045  };
1046  auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
1047 
1048  ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
1049  auto *VNodesVar = new GlobalVariable(
1050  *M, VNodesTy, false, GlobalValue::PrivateLinkage,
1052  VNodesVar->setSection(
1053  getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
1054  // VNodesVar is used by runtime but not referenced via relocation by other
1055  // sections. Conservatively make it linker retained.
1056  UsedVars.push_back(VNodesVar);
1057 }
1058 
1059 void InstrProfiling::emitNameData() {
1060  std::string UncompressedData;
1061 
1062  if (ReferencedNames.empty())
1063  return;
1064 
1065  std::string CompressedNameStr;
1066  if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1069  }
1070 
1071  auto &Ctx = M->getContext();
1072  auto *NamesVal = ConstantDataArray::getString(
1073  Ctx, StringRef(CompressedNameStr), false);
1074  NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
1075  GlobalValue::PrivateLinkage, NamesVal,
1077  NamesSize = CompressedNameStr.size();
1078  NamesVar->setSection(
1079  getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1080  // On COFF, it's important to reduce the alignment down to 1 to prevent the
1081  // linker from inserting padding before the start of the names section or
1082  // between names entries.
1083  NamesVar->setAlignment(Align(1));
1084  // NamesVar is used by runtime but not referenced via relocation by other
1085  // sections. Conservatively make it linker retained.
1086  UsedVars.push_back(NamesVar);
1087 
1088  for (auto *NamePtr : ReferencedNames)
1089  NamePtr->eraseFromParent();
1090 }
1091 
1092 void InstrProfiling::emitRegistration() {
1094  return;
1095 
1096  // Construct the function.
1097  auto *VoidTy = Type::getVoidTy(M->getContext());
1098  auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
1099  auto *Int64Ty = Type::getInt64Ty(M->getContext());
1100  auto *RegisterFTy = FunctionType::get(VoidTy, false);
1101  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1103  RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1104  if (Options.NoRedZone)
1105  RegisterF->addFnAttr(Attribute::NoRedZone);
1106 
1107  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1108  auto *RuntimeRegisterF =
1111 
1112  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1113  for (Value *Data : CompilerUsedVars)
1114  if (!isa<Function>(Data))
1115  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1116  for (Value *Data : UsedVars)
1117  if (Data != NamesVar && !isa<Function>(Data))
1118  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1119 
1120  if (NamesVar) {
1121  Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1122  auto *NamesRegisterTy =
1123  FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
1124  auto *NamesRegisterF =
1127  IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
1128  IRB.getInt64(NamesSize)});
1129  }
1130 
1131  IRB.CreateRetVoid();
1132 }
1133 
1134 bool InstrProfiling::emitRuntimeHook() {
1135  // We expect the linker to be invoked with -u<hook_var> flag for Linux
1136  // in which case there is no need to emit the external variable.
1137  if (TT.isOSLinux())
1138  return false;
1139 
1140  // If the module's provided its own runtime, we don't need to do anything.
1141  if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1142  return false;
1143 
1144  // Declare an external variable that will pull in the runtime initialization.
1145  auto *Int32Ty = Type::getInt32Ty(M->getContext());
1146  auto *Var =
1148  nullptr, getInstrProfRuntimeHookVarName());
1149 
1150  if (TT.isOSBinFormatELF()) {
1151  // Mark the user variable as used so that it isn't stripped out.
1152  CompilerUsedVars.push_back(Var);
1153  } else {
1154  // Make a function that uses it.
1158  User->addFnAttr(Attribute::NoInline);
1159  if (Options.NoRedZone)
1160  User->addFnAttr(Attribute::NoRedZone);
1161  User->setVisibility(GlobalValue::HiddenVisibility);
1162  if (TT.supportsCOMDAT())
1163  User->setComdat(M->getOrInsertComdat(User->getName()));
1164 
1165  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1166  auto *Load = IRB.CreateLoad(Int32Ty, Var);
1167  IRB.CreateRet(Load);
1168 
1169  // Mark the function as used so that it isn't stripped out.
1170  CompilerUsedVars.push_back(User);
1171  }
1172  return true;
1173 }
1174 
1175 void InstrProfiling::emitUses() {
1176  // The metadata sections are parallel arrays. Optimizers (e.g.
1177  // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1178  // we conservatively retain all unconditionally in the compiler.
1179  //
1180  // On ELF and Mach-O, the linker can guarantee the associated sections will be
1181  // retained or discarded as a unit, so llvm.compiler.used is sufficient.
1182  // Similarly on COFF, if prof data is not referenced by code we use one comdat
1183  // and ensure this GC property as well. Otherwise, we have to conservatively
1184  // make all of the sections retained by the linker.
1185  if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
1187  appendToCompilerUsed(*M, CompilerUsedVars);
1188  else
1189  appendToUsed(*M, CompilerUsedVars);
1190 
1191  // We do not add proper references from used metadata sections to NamesVar and
1192  // VNodesVar, so we have to be conservative and place them in llvm.used
1193  // regardless of the target,
1194  appendToUsed(*M, UsedVars);
1195 }
1196 
1197 void InstrProfiling::emitInitialization() {
1198  // Create ProfileFileName variable. Don't don't this for the
1199  // context-sensitive instrumentation lowering: This lowering is after
1200  // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1201  // have already create the variable before LTO/ThinLTO linking.
1202  if (!IsCS)
1204  Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1205  if (!RegisterF)
1206  return;
1207 
1208  // Create the initialization function.
1209  auto *VoidTy = Type::getVoidTy(M->getContext());
1210  auto *F = Function::Create(FunctionType::get(VoidTy, false),
1213  F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1214  F->addFnAttr(Attribute::NoInline);
1215  if (Options.NoRedZone)
1216  F->addFnAttr(Attribute::NoRedZone);
1217 
1218  // Add the basic block and the necessary calls.
1219  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1220  IRB.CreateCall(RegisterF, {});
1221  IRB.CreateRetVoid();
1222 
1223  appendToGlobalCtors(*M, F, 0);
1224 }
llvm::createInstrProfilingLegacyPass
ModulePass * createInstrProfilingLegacyPass(const InstrProfOptions &Options=InstrProfOptions(), bool IsCS=false)
Insert frontend instrumentation based profiling. Parameter IsCS indicates if.
Definition: InstrProfiling.cpp:433
INSTR_PROF_MIN_VAL_COUNTS
#define INSTR_PROF_MIN_VAL_COUNTS
i
i
Definition: README.txt:29
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
Int32Ty
IntegerType * Int32Ty
Definition: NVVMIntrRange.cpp:67
llvm::GlobalVariable::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:385
instrumentation
xray instrumentation
Definition: XRayInstrumentation.cpp:267
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
llvm::CallBase::getOperandBundlesAsDefs
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition: Instructions.cpp:363
llvm::InstrProfIncrementInst::getStep
Value * getStep() const
Definition: IntrinsicInst.cpp:181
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(InstrProfilingLegacyPass, "instrprof", "Frontend instrumentation-based coverage lowering.", false, false) INITIALIZE_PASS_END(InstrProfilingLegacyPass
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::AArch64CC::AL
@ AL
Definition: AArch64BaseInfo.h:269
llvm::LoopBase::getExitBlocks
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
Definition: LoopInfoImpl.h:62
needsRuntimeHookUnconditionally
static bool needsRuntimeHookUnconditionally(const Triple &TT)
Definition: InstrProfiling.cpp:522
llvm::StructType::get
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:370
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:255
llvm::getInstrProfNamesVarName
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:100
llvm::ModulePass
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:238
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:107
IntrinsicInst.h
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:779
llvm::Triple::isOSBinFormatCOFF
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
Definition: Triple.h:637
llvm::cl::Prefix
@ Prefix
Definition: CommandLine.h:164
llvm::LoadAndStorePromoter
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater.
Definition: SSAUpdater.h:136
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition: GlobalValue.h:461
llvm::GlobalValue::HiddenVisibility
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
llvm::InstrProfOptions::UseBFIInPromotion
bool UseBFIInPromotion
Definition: Instrumentation.h:132
llvm::Function
Definition: Function.h:61
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:530
StringRef.h
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1327
llvm::NVPTX::LoadStore
LoadStore
Definition: NVPTX.h:99
Pass.h
Loops
Hexagon Hardware Loops
Definition: HexagonHardwareLoops.cpp:372
llvm::needsComdatForCounter
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1072
llvm::toString
std::string toString(Error E)
Write all error messages (if any) in E to a string.
Definition: Error.h:1020
llvm::Comdat::NoDeduplicate
@ NoDeduplicate
No deduplication is performed.
Definition: Comdat.h:37
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
ErrorHandling.h
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:717
llvm::Intrinsic::getName
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:879
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:40
llvm::ConstantExpr::getBitCast
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2208
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:325
Error.h
llvm::AtomicOrdering::SequentiallyConsistent
@ SequentiallyConsistent
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:45
FAM
FunctionAnalysisManager FAM
Definition: PassBuilderBindings.cpp:59
llvm::getInstrProfRuntimeHookVarName
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:148
llvm::MemOp
Definition: TargetLowering.h:112
InstrCount
static unsigned InstrCount
Definition: DFAPacketizer.cpp:53
llvm::PreservedAnalyses::none
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:158
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
containsProfilingIntrinsics
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
Definition: InstrProfiling.cpp:531
Module.h
llvm::reverse
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Definition: STLExtras.h:329
llvm::AttributeList
Definition: Attributes.h:398
llvm::TargetLibraryInfo::getExtAttrForI32Param
Attribute::AttrKind getExtAttrForI32Param(bool Signed=true) const
Returns extension attribute kind to be used for i32 parameters corresponding to C-level int or unsign...
Definition: TargetLibraryInfo.h:382
llvm::InstrProfOptions::Atomic
bool Atomic
Definition: Instrumentation.h:129
llvm::getInstrProfNamesRegFuncName
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:135
llvm::GlobalValue::UnnamedAddr::Global
@ Global
llvm::GlobalValue::DefaultVisibility
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:63
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
llvm::Triple::isOSLinux
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:595
llvm::GlobalValue::LinkageTypes
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
llvm::ArrayType
Class to represent array types.
Definition: DerivedTypes.h:357
llvm::createProfileFileNameVar
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1166
llvm::Data
@ Data
Definition: SIMachineScheduler.h:55
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:203
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::InstrProfiling
Instrumentation based profiling lowering pass.
Definition: InstrProfiling.h:35
F
#define F(x, y, z)
Definition: MD5.cpp:56
InstrProfData.inc
llvm::AtomicOrdering::Monotonic
@ Monotonic
getIntModuleFlagOrZero
static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag)
Definition: InstrProfiling.cpp:779
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::InstrProfValueProfileInst::getValueKind
ConstantInt * getValueKind() const
Definition: IntrinsicInst.h:1190
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ConstantExpr::getSub
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2676
llvm::StringRef::substr
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:611
llvm::ConstantPointerNull
A constant pointer value that points to null.
Definition: Constants.h:534
Instruction.h
CommandLine.h
llvm::AMDGPU::HSAMD::ValueKind
ValueKind
Value kinds.
Definition: AMDGPUMetadata.h:73
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition: BlockFrequencyInfo.h:37
llvm::Triple::isOSBinFormatELF
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:632
GlobalValue.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::GlobalValue::VisibilityTypes
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
Constants.h
InstrProfiling.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::getInstrProfValuesVarPrefix
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:93
llvm::GlobalObject::setSection
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:212
llvm::User
Definition: User.h:44
instrprof
instrprof
Definition: InstrProfiling.cpp:429
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
Twine.h
llvm::InstrProfValueProfileInst::getTargetValue
Value * getTargetValue() const
Definition: IntrinsicInst.h:1186
llvm::BranchProbabilityInfo
Analysis providing branch probability information.
Definition: BranchProbabilityInfo.h:115
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::ConstantExpr::getPtrToInt
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2180
llvm::canRenameComdatFunc
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1120
TargetLibraryInfo.h
llvm::getInstrProfCountersVarPrefix
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:90
llvm::Triple::supportsCOMDAT
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:815
false
Definition: StackSlotColoring.cpp:142
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:109
llvm::dwarf::toStringRef
StringRef toStringRef(const Optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
Definition: DWARFFormValue.h:186
llvm::getInstrProfInitFuncName
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:143
llvm::ConstantArray
ConstantArray - Constant Array Declarations.
Definition: Constants.h:409
llvm::Triple::isOSBinFormatMachO
bool isOSBinFormatMachO() const
Tests whether the environment is MachO.
Definition: Triple.h:645
llvm::Instruction
Definition: Instruction.h:45
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:146
InstrProf.h
llvm::Triple::getObjectFormat
ObjectFormatType getObjectFormat() const
getFormat - Get the object format for this triple.
Definition: Triple.h:337
llvm::appendToCompilerUsed
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
Definition: ModuleUtils.cpp:110
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
Options
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
Definition: PassBuilderBindings.cpp:48
llvm::LoopBase::getExitingBlocks
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
Definition: LoopInfoImpl.h:34
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:900
llvm::InstrProfValueProfileInst::getIndex
ConstantInt * getIndex() const
Definition: IntrinsicInst.h:1195
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
llvm::InstrProfOptions::DoCounterPromotion
bool DoCounterPromotion
Definition: Instrumentation.h:126
llvm::GlobalValue::InternalLinkage
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
llvm::Comdat
Definition: Comdat.h:31
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:153
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:742
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
Type.h
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
LoopInfo.h
llvm::Triple::isOSFuchsia
bool isOSFuchsia() const
Definition: Triple.h:514
llvm::getInstrProfVNodesVarName
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:96
llvm::cl::ZeroOrMore
@ ZeroOrMore
Definition: CommandLine.h:120
llvm::getInstrProfNameVarPrefix
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:84
BasicBlock.h
llvm::cl::opt< bool >
SSA
Memory SSA
Definition: MemorySSA.cpp:73
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition: GlobalVariable.h:136
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:78
BranchProbabilityInfo.h
llvm::InstrProfValueProfileInst
This represents the llvm.instrprof_value_profile intrinsic.
Definition: IntrinsicInst.h:1168
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
getVarName
static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
Definition: InstrProfiling.cpp:760
llvm::TargetLibraryInfoWrapperPass
Definition: TargetLibraryInfo.h:463
uint64_t
llvm::GlobalValue::getVisibility
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:229
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:80
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
llvm::ConstantPointerNull::get
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1757
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::InstrProfOptions::InstrProfileOutput
std::string InstrProfileOutput
Definition: Instrumentation.h:135
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
profDataReferencedByCode
static bool profDataReferencedByCode(const Module &M)
Definition: InstrProfiling.cpp:795
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::DenseMap
Definition: DenseMap.h:714
shouldRecordFunctionAddr
static bool shouldRecordFunctionAddr(Function *F)
Definition: InstrProfiling.cpp:799
llvm::isIRPGOFlagSet
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1098
llvm::LoopInfoBase::getLoopFor
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:967
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::GlobalObject::setComdat
void setComdat(Comdat *C)
Definition: GlobalObject.h:125
llvm::Type::getInt64PtrTy
static PointerType * getInt64PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:267
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::appendToUsed
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
Definition: ModuleUtils.cpp:106
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:572
llvm::LoopBase::getLoopPreheader
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:167
llvm::InstrProfIncrementInst::getIndex
ConstantInt * getIndex() const
Definition: IntrinsicInst.h:1150
llvm::GlobalValue::setLinkage
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:454
ArrayRef.h
llvm::InstrProfOptions
Options for the frontend instrumentation based profiling pass.
Definition: Instrumentation.h:121
llvm::X86II::PD
@ PD
Definition: X86BaseInfo.h:782
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:138
llvm::LoopBase::hasDedicatedExits
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
Definition: LoopInfoImpl.h:92
llvm::getInstrProfRuntimeHookVarUseFuncName
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:154
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:83
lowering
Frontend instrumentation based coverage lowering
Definition: InstrProfiling.cpp:430
llvm::ArrayType::get
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:602
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
llvm::InstrProfIncrementInst::getNumCounters
ConstantInt * getNumCounters() const
Definition: IntrinsicInst.h:1146
llvm::collectPGOFuncNameStrings
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:401
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:650
Triple.h
llvm::getInstrProfRegFuncsName
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:124
llvm::InstrProfIncrementInst::getHash
ConstantInt * getHash() const
Definition: IntrinsicInst.h:1142
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::LoopInfo
Definition: LoopInfo.h:1083
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1554
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:253
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:256
if
if(llvm_vc STREQUAL "") set(fake_version_inc "$
Definition: CMakeLists.txt:14
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:70
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:520
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:100
uint32_t
llvm::ConstantDataArray::getString
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:3041
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
SSAUpdater.h
BlockFrequencyInfo.h
llvm::AMDGPUISD::BFI
@ BFI
Definition: AMDGPUISelLowering.h:421
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:297
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:175
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:148
llvm::Init
Definition: Record.h:271
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
llvm::BasicBlock::front
const Instruction & front() const
Definition: BasicBlock.h:308
llvm::DoInstrProfNameCompression
cl::opt< bool > DoInstrProfNameCompression
Attributes.h
Constant.h
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:204
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:348
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
NC
#define NC
Definition: regutils.h:42
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
llvm::Error
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
GlobalVariable.h
getOrInsertValueProfilingCall
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
Definition: InstrProfiling.cpp:608
Casting.h
Function.h
enablesValueProfiling
static bool enablesValueProfiling(const Module &M)
Definition: InstrProfiling.cpp:789
llvm::TargetLibraryInfo
Provides information about what library functions are available for the current target.
Definition: TargetLibraryInfo.h:219
llvm::TargetStackID::Default
@ Default
Definition: TargetFrameLowering.h:28
llvm::InstrProfiling::run
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition: InstrProfiling.cpp:411
llvm::MCID::Add
@ Add
Definition: MCInstrDesc.h:183
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:476
llvm::InstrProfIncrementInst
This represents the llvm.instrprof_increment intrinsic.
Definition: IntrinsicInst.h:1128
llvm::getInstrProfSectionName
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:175
llvm::GlobalValue::ExternalLinkage
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
castToIncrementInst
static InstrProfIncrementInst * castToIncrementInst(Instruction *Instr)
Definition: InstrProfiling.cpp:438
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:186
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Instructions.h
llvm::getInstrProfValueProfMemOpFuncName
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:79
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
needsRuntimeRegistrationOfSectionRange
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
Definition: InstrProfiling.cpp:836
SmallVector.h
llvm::InstrProfOptions::NoRedZone
bool NoRedZone
Definition: Instrumentation.h:123
Dominators.h
llvm::getCoverageUnusedNamesVarName
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:115
ModuleUtils.h
llvm::getInstrProfCounterBiasVarName
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:158
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:340
llvm::InstrProfValueProfileInst::getName
GlobalVariable * getName() const
Definition: IntrinsicInst.h:1177
llvm::FunctionCallee
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
llvm::SmallVectorImpl< BasicBlock * >
llvm::Type::getInt16Ty
static IntegerType * getInt16Ty(LLVMContext &C)
Definition: Type.cpp:202
DerivedTypes.h
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition: GlobalValue.h:273
llvm::InnerAnalysisManagerProxy
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:936
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1475
llvm::getInstrProfValueProfFuncName
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:73
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:389
llvm::appendToGlobalCtors
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:66
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::getInstrProfDataVarPrefix
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:87
llvm::cl::desc
Definition: CommandLine.h:414
llvm::GlobalObject::setAlignment
void setAlignment(MaybeAlign Align)
Definition: Globals.cpp:117
llvm::InstrProfIncrementInst::getName
GlobalVariable * getName() const
Definition: IntrinsicInst.h:1137
llvm::SSAUpdater
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:38
llvm::StringRef::size
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:157
BasicBlockUtils.h
llvm::initializeInstrProfilingLegacyPassPass
void initializeInstrProfilingLegacyPassPass(PassRegistry &)
llvm::GlobalValue::LinkOnceODRLinkage
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
llvm::GlobalValue::setVisibility
void setVisibility(VisibilityTypes V)
Definition: GlobalValue.h:235
InitializePasses.h
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
llvm::getInstrProfRegFuncName
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:130
llvm::TargetLibraryAnalysis
Analysis pass providing the TargetLibraryInfo.
Definition: TargetLibraryInfo.h:438
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:364
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37