LLVM 17.0.0git
InstrProfiling.cpp
Go to the documentation of this file.
1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
16#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/Twine.h"
24#include "llvm/IR/Attributes.h"
25#include "llvm/IR/BasicBlock.h"
26#include "llvm/IR/Constant.h"
27#include "llvm/IR/Constants.h"
28#include "llvm/IR/DIBuilder.h"
31#include "llvm/IR/Dominators.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/GlobalValue.h"
35#include "llvm/IR/IRBuilder.h"
36#include "llvm/IR/Instruction.h"
39#include "llvm/IR/Module.h"
40#include "llvm/IR/Type.h"
42#include "llvm/Pass.h"
47#include "llvm/Support/Error.h"
52#include <algorithm>
53#include <cassert>
54#include <cstdint>
55#include <string>
56
57using namespace llvm;
58
59#define DEBUG_TYPE "instrprof"
60
61namespace llvm {
63 DebugInfoCorrelate("debug-info-correlate",
64 cl::desc("Use debug info to correlate profiles."),
65 cl::init(false));
66} // namespace llvm
67
68namespace {
69
70cl::opt<bool> DoHashBasedCounterSplit(
71 "hash-based-counter-split",
72 cl::desc("Rename counter variable of a comdat function based on cfg hash"),
73 cl::init(true));
74
76 RuntimeCounterRelocation("runtime-counter-relocation",
77 cl::desc("Enable relocating counters at runtime."),
78 cl::init(false));
79
80cl::opt<bool> ValueProfileStaticAlloc(
81 "vp-static-alloc",
82 cl::desc("Do static counter allocation for value profiler"),
83 cl::init(true));
84
85cl::opt<double> NumCountersPerValueSite(
86 "vp-counters-per-site",
87 cl::desc("The average number of profile counters allocated "
88 "per value profiling site."),
89 // This is set to a very small value because in real programs, only
90 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
91 // For those sites with non-zero profile, the average number of targets
92 // is usually smaller than 2.
93 cl::init(1.0));
94
95cl::opt<bool> AtomicCounterUpdateAll(
96 "instrprof-atomic-counter-update-all",
97 cl::desc("Make all profile counter updates atomic (for testing only)"),
98 cl::init(false));
99
100cl::opt<bool> AtomicCounterUpdatePromoted(
101 "atomic-counter-update-promoted",
102 cl::desc("Do counter update using atomic fetch add "
103 " for promoted counters only"),
104 cl::init(false));
105
106cl::opt<bool> AtomicFirstCounter(
107 "atomic-first-counter",
108 cl::desc("Use atomic fetch add for first counter in a function (usually "
109 "the entry counter)"),
110 cl::init(false));
111
112// If the option is not specified, the default behavior about whether
113// counter promotion is done depends on how instrumentaiton lowering
114// pipeline is setup, i.e., the default value of true of this option
115// does not mean the promotion will be done by default. Explicitly
116// setting this option can override the default behavior.
117cl::opt<bool> DoCounterPromotion("do-counter-promotion",
118 cl::desc("Do counter register promotion"),
119 cl::init(false));
120cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
121 "max-counter-promotions-per-loop", cl::init(20),
122 cl::desc("Max number counter promotions per loop to avoid"
123 " increasing register pressure too much"));
124
125// A debug option
127 MaxNumOfPromotions("max-counter-promotions", cl::init(-1),
128 cl::desc("Max number of allowed counter promotions"));
129
130cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
131 "speculative-counter-promotion-max-exiting", cl::init(3),
132 cl::desc("The max number of exiting blocks of a loop to allow "
133 " speculative counter promotion"));
134
135cl::opt<bool> SpeculativeCounterPromotionToLoop(
136 "speculative-counter-promotion-to-loop",
137 cl::desc("When the option is false, if the target block is in a loop, "
138 "the promotion will be disallowed unless the promoted counter "
139 " update can be further/iteratively promoted into an acyclic "
140 " region."));
141
142cl::opt<bool> IterativeCounterPromotion(
143 "iterative-counter-promotion", cl::init(true),
144 cl::desc("Allow counter promotion across the whole loop nest."));
145
146cl::opt<bool> SkipRetExitBlock(
147 "skip-ret-exit-block", cl::init(true),
148 cl::desc("Suppress counter promotion if exit blocks contain ret."));
149
150///
151/// A helper class to promote one counter RMW operation in the loop
152/// into register update.
153///
154/// RWM update for the counter will be sinked out of the loop after
155/// the transformation.
156///
157class PGOCounterPromoterHelper : public LoadAndStorePromoter {
158public:
159 PGOCounterPromoterHelper(
161 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
162 ArrayRef<Instruction *> InsertPts,
164 LoopInfo &LI)
165 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
166 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
167 assert(isa<LoadInst>(L));
168 assert(isa<StoreInst>(S));
169 SSA.AddAvailableValue(PH, Init);
170 }
171
172 void doExtraRewritesBeforeFinalDeletion() override {
173 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
174 BasicBlock *ExitBlock = ExitBlocks[i];
175 Instruction *InsertPos = InsertPts[i];
176 // Get LiveIn value into the ExitBlock. If there are multiple
177 // predecessors, the value is defined by a PHI node in this
178 // block.
179 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
180 Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
181 Type *Ty = LiveInValue->getType();
182 IRBuilder<> Builder(InsertPos);
183 if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Addr)) {
184 // If isRuntimeCounterRelocationEnabled() is true then the address of
185 // the store instruction is computed with two instructions in
186 // InstrProfiling::getCounterAddress(). We need to copy those
187 // instructions to this block to compute Addr correctly.
188 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
189 // %Addr = inttoptr i64 %BiasAdd to i64*
190 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
191 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
192 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
193 Addr = Builder.CreateIntToPtr(BiasInst, Ty->getPointerTo());
194 }
195 if (AtomicCounterUpdatePromoted)
196 // automic update currently can only be promoted across the current
197 // loop, not the whole loop nest.
198 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
199 MaybeAlign(),
200 AtomicOrdering::SequentiallyConsistent);
201 else {
202 LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
203 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
204 auto *NewStore = Builder.CreateStore(NewVal, Addr);
205
206 // Now update the parent loop's candidate list:
207 if (IterativeCounterPromotion) {
208 auto *TargetLoop = LI.getLoopFor(ExitBlock);
209 if (TargetLoop)
210 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
211 }
212 }
213 }
214 }
215
216private:
218 ArrayRef<BasicBlock *> ExitBlocks;
219 ArrayRef<Instruction *> InsertPts;
221 LoopInfo &LI;
222};
223
224/// A helper class to do register promotion for all profile counter
225/// updates in a loop.
226///
227class PGOCounterPromoter {
228public:
229 PGOCounterPromoter(
231 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
232 : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
233
234 // Skip collection of ExitBlocks and InsertPts for loops that will not be
235 // able to have counters promoted.
236 SmallVector<BasicBlock *, 8> LoopExitBlocks;
238
239 L.getExitBlocks(LoopExitBlocks);
240 if (!isPromotionPossible(&L, LoopExitBlocks))
241 return;
242
243 for (BasicBlock *ExitBlock : LoopExitBlocks) {
244 if (BlockSet.insert(ExitBlock).second) {
245 ExitBlocks.push_back(ExitBlock);
246 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
247 }
248 }
249 }
250
251 bool run(int64_t *NumPromoted) {
252 // Skip 'infinite' loops:
253 if (ExitBlocks.size() == 0)
254 return false;
255
256 // Skip if any of the ExitBlocks contains a ret instruction.
257 // This is to prevent dumping of incomplete profile -- if the
258 // the loop is a long running loop and dump is called in the middle
259 // of the loop, the result profile is incomplete.
260 // FIXME: add other heuristics to detect long running loops.
261 if (SkipRetExitBlock) {
262 for (auto *BB : ExitBlocks)
263 if (isa<ReturnInst>(BB->getTerminator()))
264 return false;
265 }
266
267 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
268 if (MaxProm == 0)
269 return false;
270
271 unsigned Promoted = 0;
272 for (auto &Cand : LoopToCandidates[&L]) {
273
275 SSAUpdater SSA(&NewPHIs);
276 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
277
278 // If BFI is set, we will use it to guide the promotions.
279 if (BFI) {
280 auto *BB = Cand.first->getParent();
281 auto InstrCount = BFI->getBlockProfileCount(BB);
282 if (!InstrCount)
283 continue;
284 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
285 // If the average loop trip count is not greater than 1.5, we skip
286 // promotion.
287 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
288 continue;
289 }
290
291 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
292 L.getLoopPreheader(), ExitBlocks,
293 InsertPts, LoopToCandidates, LI);
294 Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
295 Promoted++;
296 if (Promoted >= MaxProm)
297 break;
298
299 (*NumPromoted)++;
300 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
301 break;
302 }
303
304 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
305 << L.getLoopDepth() << ")\n");
306 return Promoted != 0;
307 }
308
309private:
310 bool allowSpeculativeCounterPromotion(Loop *LP) {
311 SmallVector<BasicBlock *, 8> ExitingBlocks;
312 L.getExitingBlocks(ExitingBlocks);
313 // Not considierered speculative.
314 if (ExitingBlocks.size() == 1)
315 return true;
316 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
317 return false;
318 return true;
319 }
320
321 // Check whether the loop satisfies the basic conditions needed to perform
322 // Counter Promotions.
323 bool
324 isPromotionPossible(Loop *LP,
325 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
326 // We can't insert into a catchswitch.
327 if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
328 return isa<CatchSwitchInst>(Exit->getTerminator());
329 }))
330 return false;
331
332 if (!LP->hasDedicatedExits())
333 return false;
334
335 BasicBlock *PH = LP->getLoopPreheader();
336 if (!PH)
337 return false;
338
339 return true;
340 }
341
342 // Returns the max number of Counter Promotions for LP.
343 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
344 SmallVector<BasicBlock *, 8> LoopExitBlocks;
345 LP->getExitBlocks(LoopExitBlocks);
346 if (!isPromotionPossible(LP, LoopExitBlocks))
347 return 0;
348
349 SmallVector<BasicBlock *, 8> ExitingBlocks;
350 LP->getExitingBlocks(ExitingBlocks);
351
352 // If BFI is set, we do more aggressive promotions based on BFI.
353 if (BFI)
354 return (unsigned)-1;
355
356 // Not considierered speculative.
357 if (ExitingBlocks.size() == 1)
358 return MaxNumOfPromotionsPerLoop;
359
360 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
361 return 0;
362
363 // Whether the target block is in a loop does not matter:
364 if (SpeculativeCounterPromotionToLoop)
365 return MaxNumOfPromotionsPerLoop;
366
367 // Now check the target block:
368 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
369 for (auto *TargetBlock : LoopExitBlocks) {
370 auto *TargetLoop = LI.getLoopFor(TargetBlock);
371 if (!TargetLoop)
372 continue;
373 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
374 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
375 MaxProm =
376 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
377 PendingCandsInTarget);
378 }
379 return MaxProm;
380 }
381
385 Loop &L;
386 LoopInfo &LI;
388};
389
390enum class ValueProfilingCallType {
391 // Individual values are tracked. Currently used for indiret call target
392 // profiling.
393 Default,
394
395 // MemOp: the memop size value profiling.
396 MemOp
397};
398
399} // end anonymous namespace
400
404 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
406 };
407 if (!run(M, GetTLI))
408 return PreservedAnalyses::all();
409
411}
412
413bool InstrProfiling::lowerIntrinsics(Function *F) {
414 bool MadeChange = false;
415 PromotionCandidates.clear();
416 for (BasicBlock &BB : *F) {
417 for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
418 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
419 lowerIncrement(IPIS);
420 MadeChange = true;
421 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
422 lowerIncrement(IPI);
423 MadeChange = true;
424 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
425 lowerCover(IPC);
426 MadeChange = true;
427 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
428 lowerValueProfileInst(IPVP);
429 MadeChange = true;
430 }
431 }
432 }
433
434 if (!MadeChange)
435 return false;
436
437 promoteCounterLoadStores(F);
438 return true;
439}
440
441bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
442 // Mach-O don't support weak external references.
443 if (TT.isOSBinFormatMachO())
444 return false;
445
446 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
447 return RuntimeCounterRelocation;
448
449 // Fuchsia uses runtime counter relocation by default.
450 return TT.isOSFuchsia();
451}
452
453bool InstrProfiling::isCounterPromotionEnabled() const {
454 if (DoCounterPromotion.getNumOccurrences() > 0)
455 return DoCounterPromotion;
456
457 return Options.DoCounterPromotion;
458}
459
460void InstrProfiling::promoteCounterLoadStores(Function *F) {
461 if (!isCounterPromotionEnabled())
462 return;
463
464 DominatorTree DT(*F);
465 LoopInfo LI(DT);
466 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
467
468 std::unique_ptr<BlockFrequencyInfo> BFI;
469 if (Options.UseBFIInPromotion) {
470 std::unique_ptr<BranchProbabilityInfo> BPI;
471 BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
472 BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
473 }
474
475 for (const auto &LoadStore : PromotionCandidates) {
476 auto *CounterLoad = LoadStore.first;
477 auto *CounterStore = LoadStore.second;
478 BasicBlock *BB = CounterLoad->getParent();
479 Loop *ParentLoop = LI.getLoopFor(BB);
480 if (!ParentLoop)
481 continue;
482 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
483 }
484
486
487 // Do a post-order traversal of the loops so that counter updates can be
488 // iteratively hoisted outside the loop nest.
489 for (auto *Loop : llvm::reverse(Loops)) {
490 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
491 Promoter.run(&TotalCountersPromoted);
492 }
493}
494
496 // On Fuchsia, we only need runtime hook if any counters are present.
497 if (TT.isOSFuchsia())
498 return false;
499
500 return true;
501}
502
503/// Check if the module contains uses of any profiling intrinsics.
505 auto containsIntrinsic = [&](int ID) {
506 if (auto *F = M.getFunction(Intrinsic::getName(ID)))
507 return !F->use_empty();
508 return false;
509 };
510 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
511 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
512 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
513 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
514}
515
517 Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
518 this->M = &M;
519 this->GetTLI = std::move(GetTLI);
520 NamesVar = nullptr;
521 NamesSize = 0;
522 ProfileDataMap.clear();
523 CompilerUsedVars.clear();
524 UsedVars.clear();
525 TT = Triple(M.getTargetTriple());
526
527 bool MadeChange = false;
528 bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
529 if (NeedsRuntimeHook)
530 MadeChange = emitRuntimeHook();
531
532 bool ContainsProfiling = containsProfilingIntrinsics(M);
533 GlobalVariable *CoverageNamesVar =
534 M.getNamedGlobal(getCoverageUnusedNamesVarName());
535 // Improve compile time by avoiding linear scans when there is no work.
536 if (!ContainsProfiling && !CoverageNamesVar)
537 return MadeChange;
538
539 // We did not know how many value sites there would be inside
540 // the instrumented function. This is counting the number of instrumented
541 // target value sites to enter it as field in the profile data variable.
542 for (Function &F : M) {
543 InstrProfIncrementInst *FirstProfIncInst = nullptr;
544 for (BasicBlock &BB : F)
545 for (auto I = BB.begin(), E = BB.end(); I != E; I++)
546 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
547 computeNumValueSiteCounts(Ind);
548 else if (FirstProfIncInst == nullptr)
549 FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
550
551 // Value profiling intrinsic lowering requires per-function profile data
552 // variable to be created first.
553 if (FirstProfIncInst != nullptr)
554 static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
555 }
556
557 for (Function &F : M)
558 MadeChange |= lowerIntrinsics(&F);
559
560 if (CoverageNamesVar) {
561 lowerCoverageData(CoverageNamesVar);
562 MadeChange = true;
563 }
564
565 if (!MadeChange)
566 return false;
567
568 emitVNodes();
569 emitNameData();
570
571 // Emit runtime hook for the cases where the target does not unconditionally
572 // require pulling in profile runtime, and coverage is enabled on code that is
573 // not eliminated by the front-end, e.g. unused functions with internal
574 // linkage.
575 if (!NeedsRuntimeHook && ContainsProfiling)
576 emitRuntimeHook();
577
578 emitRegistration();
579 emitUses();
580 emitInitialization();
581 return true;
582}
583
585 Module &M, const TargetLibraryInfo &TLI,
586 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
587 LLVMContext &Ctx = M.getContext();
588 auto *ReturnTy = Type::getVoidTy(M.getContext());
589
590 AttributeList AL;
591 if (auto AK = TLI.getExtAttrForI32Param(false))
592 AL = AL.addParamAttribute(M.getContext(), 2, AK);
593
594 assert((CallType == ValueProfilingCallType::Default ||
595 CallType == ValueProfilingCallType::MemOp) &&
596 "Must be Default or MemOp");
597 Type *ParamTypes[] = {
598#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
600 };
601 auto *ValueProfilingCallTy =
602 FunctionType::get(ReturnTy, ArrayRef(ParamTypes), false);
603 StringRef FuncName = CallType == ValueProfilingCallType::Default
606 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
607}
608
609void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
610 GlobalVariable *Name = Ind->getName();
613 auto &PD = ProfileDataMap[Name];
614 PD.NumValueSites[ValueKind] =
615 std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));
616}
617
618void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
619 // TODO: Value profiling heavily depends on the data section which is omitted
620 // in lightweight mode. We need to move the value profile pointer to the
621 // Counter struct to get this working.
622 assert(
624 "Value profiling is not yet supported with lightweight instrumentation");
625 GlobalVariable *Name = Ind->getName();
626 auto It = ProfileDataMap.find(Name);
627 assert(It != ProfileDataMap.end() && It->second.DataVar &&
628 "value profiling detected in function with no counter incerement");
629
630 GlobalVariable *DataVar = It->second.DataVar;
633 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
634 Index += It->second.NumValueSites[Kind];
635
636 IRBuilder<> Builder(Ind);
637 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
638 llvm::InstrProfValueKind::IPVK_MemOPSize);
639 CallInst *Call = nullptr;
640 auto *TLI = &GetTLI(*Ind->getFunction());
641
642 // To support value profiling calls within Windows exception handlers, funclet
643 // information contained within operand bundles needs to be copied over to
644 // the library call. This is required for the IR to be processed by the
645 // WinEHPrepare pass.
647 Ind->getOperandBundlesAsDefs(OpBundles);
648 if (!IsMemOpSize) {
649 Value *Args[3] = {Ind->getTargetValue(),
650 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
651 Builder.getInt32(Index)};
652 Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
653 OpBundles);
654 } else {
655 Value *Args[3] = {Ind->getTargetValue(),
656 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
657 Builder.getInt32(Index)};
658 Call = Builder.CreateCall(
659 getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
660 Args, OpBundles);
661 }
662 if (auto AK = TLI->getExtAttrForI32Param(false))
663 Call->addParamAttr(2, AK);
664 Ind->replaceAllUsesWith(Call);
665 Ind->eraseFromParent();
666}
667
668Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) {
669 auto *Counters = getOrCreateRegionCounters(I);
671
672 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
673 Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());
674
675 if (!isRuntimeCounterRelocationEnabled())
676 return Addr;
677
678 Type *Int64Ty = Type::getInt64Ty(M->getContext());
679 Function *Fn = I->getParent()->getParent();
680 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
681 if (!BiasLI) {
682 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
683 auto *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
684 if (!Bias) {
685 // Compiler must define this variable when runtime counter relocation
686 // is being used. Runtime has a weak external reference that is used
687 // to check whether that's the case or not.
688 Bias = new GlobalVariable(
689 *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
691 Bias->setVisibility(GlobalVariable::HiddenVisibility);
692 // A definition that's weak (linkonce_odr) without being in a COMDAT
693 // section wouldn't lead to link errors, but it would lead to a dead
694 // data word from every TU but one. Putting it in COMDAT ensures there
695 // will be exactly one data slot in the link.
696 if (TT.supportsCOMDAT())
697 Bias->setComdat(M->getOrInsertComdat(Bias->getName()));
698 }
699 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
700 }
701 auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
702 return Builder.CreateIntToPtr(Add, Addr->getType());
703}
704
705void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) {
706 auto *Addr = getCounterAddress(CoverInstruction);
707 IRBuilder<> Builder(CoverInstruction);
708 // We store zero to represent that this block is covered.
709 Builder.CreateStore(Builder.getInt8(0), Addr);
710 CoverInstruction->eraseFromParent();
711}
712
713void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
714 auto *Addr = getCounterAddress(Inc);
715
716 IRBuilder<> Builder(Inc);
717 if (Options.Atomic || AtomicCounterUpdateAll ||
718 (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
719 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
721 } else {
722 Value *IncStep = Inc->getStep();
723 Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
724 auto *Count = Builder.CreateAdd(Load, Inc->getStep());
725 auto *Store = Builder.CreateStore(Count, Addr);
726 if (isCounterPromotionEnabled())
727 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
728 }
729 Inc->eraseFromParent();
730}
731
732void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
734 cast<ConstantArray>(CoverageNamesVar->getInitializer());
735 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
736 Constant *NC = Names->getOperand(I);
737 Value *V = NC->stripPointerCasts();
738 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
739 GlobalVariable *Name = cast<GlobalVariable>(V);
740
742 ReferencedNames.push_back(Name);
743 if (isa<ConstantExpr>(NC))
744 NC->dropAllReferences();
745 }
746 CoverageNamesVar->eraseFromParent();
747}
748
749/// Get the name of a profiling variable for a particular function.
750static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
751 bool &Renamed) {
753 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
754 Function *F = Inc->getParent()->getParent();
755 Module *M = F->getParent();
756 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
758 Renamed = false;
759 return (Prefix + Name).str();
760 }
761 Renamed = true;
762 uint64_t FuncHash = Inc->getHash()->getZExtValue();
763 SmallVector<char, 24> HashPostfix;
764 if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
765 return (Prefix + Name).str();
766 return (Prefix + Name + "." + Twine(FuncHash)).str();
767}
768
770 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
771 if (!MD)
772 return 0;
773
774 // If the flag is a ConstantAsMetadata, it should be an integer representable
775 // in 64-bits.
776 return cast<ConstantInt>(MD->getValue())->getZExtValue();
777}
778
779static bool enablesValueProfiling(const Module &M) {
780 return isIRPGOFlagSet(&M) ||
781 getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
782}
783
784// Conservatively returns true if data variables may be referenced by code.
785static bool profDataReferencedByCode(const Module &M) {
786 return enablesValueProfiling(M);
787}
788
789static inline bool shouldRecordFunctionAddr(Function *F) {
790 // Only record function addresses if IR PGO is enabled or if clang value
791 // profiling is enabled. Recording function addresses greatly increases object
792 // file size, because it prevents the inliner from deleting functions that
793 // have been inlined everywhere.
794 if (!profDataReferencedByCode(*F->getParent()))
795 return false;
796
797 // Check the linkage
798 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
799 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
800 !HasAvailableExternallyLinkage)
801 return true;
802
803 // A function marked 'alwaysinline' with available_externally linkage can't
804 // have its address taken. Doing so would create an undefined external ref to
805 // the function, which would fail to link.
806 if (HasAvailableExternallyLinkage &&
807 F->hasFnAttribute(Attribute::AlwaysInline))
808 return false;
809
810 // Prohibit function address recording if the function is both internal and
811 // COMDAT. This avoids the profile data variable referencing internal symbols
812 // in COMDAT.
813 if (F->hasLocalLinkage() && F->hasComdat())
814 return false;
815
816 // Check uses of this function for other than direct calls or invokes to it.
817 // Inline virtual functions have linkeOnceODR linkage. When a key method
818 // exists, the vtable will only be emitted in the TU where the key method
819 // is defined. In a TU where vtable is not available, the function won't
820 // be 'addresstaken'. If its address is not recorded here, the profile data
821 // with missing address may be picked by the linker leading to missing
822 // indirect call target info.
823 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
824}
825
826static inline bool shouldUsePublicSymbol(Function *Fn) {
827 // It isn't legal to make an alias of this function at all
828 if (Fn->isDeclarationForLinker())
829 return true;
830
831 // Symbols with local linkage can just use the symbol directly without
832 // introducing relocations
833 if (Fn->hasLocalLinkage())
834 return true;
835
836 // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
837 // unfavorable interaction between the new alias and the alias renaming done
838 // in LowerTypeTests under ThinLTO. For comdat functions that would normally
839 // be deduplicated, but the renaming scheme ends up preventing renaming, since
840 // it creates unique names for each alias, resulting in duplicated symbols. In
841 // the future, we should update the CFI related passes to migrate these
842 // aliases to the same module as the jump-table they refer to will be defined.
843 if (Fn->hasMetadata(LLVMContext::MD_type))
844 return true;
845
846 // For comdat functions, an alias would need the same linkage as the original
847 // function and hidden visibility. There is no point in adding an alias with
848 // identical linkage an visibility to avoid introducing symbolic relocations.
849 if (Fn->hasComdat() &&
851 return true;
852
853 // its OK to use an alias
854 return false;
855}
856
858 auto *Int8PtrTy = Type::getInt8PtrTy(Fn->getContext());
859 // Store a nullptr in __llvm_profd, if we shouldn't use a real address
861 return ConstantPointerNull::get(Int8PtrTy);
862
863 // If we can't use an alias, we must use the public symbol, even though this
864 // may require a symbolic relocation.
865 if (shouldUsePublicSymbol(Fn))
866 return ConstantExpr::getBitCast(Fn, Int8PtrTy);
867
868 // When possible use a private alias to avoid symbolic relocations.
870 Fn->getName() + ".local", Fn);
871
872 // When the instrumented function is a COMDAT function, we cannot use a
873 // private alias. If we did, we would create reference to a local label in
874 // this function's section. If this version of the function isn't selected by
875 // the linker, then the metadata would introduce a reference to a discarded
876 // section. So, for COMDAT functions, we need to adjust the linkage of the
877 // alias. Using hidden visibility avoids a dynamic relocation and an entry in
878 // the dynamic symbol table.
879 //
880 // Note that this handles COMDAT functions with visibility other than Hidden,
881 // since that case is covered in shouldUsePublicSymbol()
882 if (Fn->hasComdat()) {
883 GA->setLinkage(Fn->getLinkage());
885 }
886
887 // appendToCompilerUsed(*Fn->getParent(), {GA});
888
889 return ConstantExpr::getBitCast(GA, Int8PtrTy);
890}
891
893 // Don't do this for Darwin. compiler-rt uses linker magic.
894 if (TT.isOSDarwin())
895 return false;
896 // Use linker script magic to get data/cnts/name start/end.
897 if (TT.isOSAIX() || TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
898 TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS() || TT.isOSWindows())
899 return false;
900
901 return true;
902}
903
905InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name,
907 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
908 auto &Ctx = M->getContext();
909 GlobalVariable *GV;
910 if (isa<InstrProfCoverInst>(Inc)) {
911 auto *CounterTy = Type::getInt8Ty(Ctx);
912 auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
913 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
914 std::vector<Constant *> InitialValues(NumCounters,
915 Constant::getAllOnesValue(CounterTy));
916 GV = new GlobalVariable(*M, CounterArrTy, false, Linkage,
917 ConstantArray::get(CounterArrTy, InitialValues),
918 Name);
919 GV->setAlignment(Align(1));
920 } else {
921 auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
922 GV = new GlobalVariable(*M, CounterTy, false, Linkage,
923 Constant::getNullValue(CounterTy), Name);
924 GV->setAlignment(Align(8));
925 }
926 return GV;
927}
928
930InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
931 GlobalVariable *NamePtr = Inc->getName();
932 auto &PD = ProfileDataMap[NamePtr];
933 if (PD.RegionCounters)
934 return PD.RegionCounters;
935
936 // Match the linkage and visibility of the name global.
937 Function *Fn = Inc->getParent()->getParent();
939 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
940
941 // Use internal rather than private linkage so the counter variable shows up
942 // in the symbol table when using debug info for correlation.
946
947 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
948 // symbols in the same csect won't be discarded. When there are duplicate weak
949 // symbols, we can NOT guarantee that the relocations get resolved to the
950 // intended weak symbol, so we can not ensure the correctness of the relative
951 // CounterPtr, so we have to use private linkage for counter and data symbols.
952 if (TT.isOSBinFormatXCOFF()) {
955 }
956 // Move the name variable to the right section. Place them in a COMDAT group
957 // if the associated function is a COMDAT. This will make sure that only one
958 // copy of counters of the COMDAT function will be emitted after linking. Keep
959 // in mind that this pass may run before the inliner, so we need to create a
960 // new comdat group for the counters and profiling data. If we use the comdat
961 // of the parent function, that will result in relocations against discarded
962 // sections.
963 //
964 // If the data variable is referenced by code, counters and data have to be
965 // in different comdats for COFF because the Visual C++ linker will report
966 // duplicate symbol errors if there are multiple external symbols with the
967 // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
968 //
969 // For ELF, when not using COMDAT, put counters, data and values into a
970 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
971 // allows -z start-stop-gc to discard the entire group when the function is
972 // discarded.
973 bool DataReferencedByCode = profDataReferencedByCode(*M);
974 bool NeedComdat = needsComdatForCounter(*Fn, *M);
975 bool Renamed;
976 std::string CntsVarName =
978 std::string DataVarName =
979 getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
980 auto MaybeSetComdat = [&](GlobalVariable *GV) {
981 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
982 if (UseComdat) {
983 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
984 ? GV->getName()
985 : CntsVarName;
986 Comdat *C = M->getOrInsertComdat(GroupName);
987 if (!NeedComdat)
988 C->setSelectionKind(Comdat::NoDeduplicate);
989 GV->setComdat(C);
990 // COFF doesn't allow the comdat group leader to have private linkage, so
991 // upgrade private linkage to internal linkage to produce a symbol table
992 // entry.
993 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
995 }
996 };
997
998 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
999 LLVMContext &Ctx = M->getContext();
1000
1001 auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage);
1002 CounterPtr->setVisibility(Visibility);
1003 CounterPtr->setSection(
1004 getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
1005 CounterPtr->setLinkage(Linkage);
1006 MaybeSetComdat(CounterPtr);
1007 PD.RegionCounters = CounterPtr;
1008 if (DebugInfoCorrelate) {
1009 if (auto *SP = Fn->getSubprogram()) {
1010 DIBuilder DB(*M, true, SP->getUnit());
1011 Metadata *FunctionNameAnnotation[] = {
1014 };
1015 Metadata *CFGHashAnnotation[] = {
1018 };
1019 Metadata *NumCountersAnnotation[] = {
1022 };
1023 auto Annotations = DB.getOrCreateArray({
1024 MDNode::get(Ctx, FunctionNameAnnotation),
1025 MDNode::get(Ctx, CFGHashAnnotation),
1026 MDNode::get(Ctx, NumCountersAnnotation),
1027 });
1028 auto *DICounter = DB.createGlobalVariableExpression(
1029 SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
1030 /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
1031 CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
1032 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1033 Annotations);
1034 CounterPtr->addDebugInfo(DICounter);
1035 DB.finalize();
1036 } else {
1037 std::string Msg = ("Missing debug info for function " + Fn->getName() +
1038 "; required for profile correlation.")
1039 .str();
1040 Ctx.diagnose(
1041 DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
1042 }
1043 }
1044
1045 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
1046 // Allocate statically the array of pointers to value profile nodes for
1047 // the current function.
1048 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
1049 uint64_t NS = 0;
1050 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1051 NS += PD.NumValueSites[Kind];
1052 if (NS > 0 && ValueProfileStaticAlloc &&
1054 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
1055 auto *ValuesVar = new GlobalVariable(
1056 *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
1057 getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
1058 ValuesVar->setVisibility(Visibility);
1059 ValuesVar->setSection(
1060 getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
1061 ValuesVar->setAlignment(Align(8));
1062 MaybeSetComdat(ValuesVar);
1063 ValuesPtrExpr =
1065 }
1066
1067 if (DebugInfoCorrelate) {
1068 // Mark the counter variable as used so that it isn't optimized out.
1069 CompilerUsedVars.push_back(PD.RegionCounters);
1070 return PD.RegionCounters;
1071 }
1072
1073 // Create data variable.
1074 auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
1075 auto *Int16Ty = Type::getInt16Ty(Ctx);
1076 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
1077 Type *DataTypes[] = {
1078#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1080 };
1081 auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));
1082
1083 Constant *FunctionAddr = getFuncAddrForProfData(Fn);
1084
1085 Constant *Int16ArrayVals[IPVK_Last + 1];
1086 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1087 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
1088
1089 // If the data variable is not referenced by code (if we don't emit
1090 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1091 // data variable live under linker GC, the data variable can be private. This
1092 // optimization applies to ELF.
1093 //
1094 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1095 // to be false.
1096 //
1097 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1098 // that other copies must have the same CFG and cannot have value profiling.
1099 // If no hash suffix, other profd copies may be referenced by code.
1100 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1101 (TT.isOSBinFormatELF() ||
1102 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1104 Visibility = GlobalValue::DefaultVisibility;
1105 }
1106 auto *Data =
1107 new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName);
1108 // Reference the counter variable with a label difference (link-time
1109 // constant).
1110 auto *RelativeCounterPtr =
1111 ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
1112 ConstantExpr::getPtrToInt(Data, IntPtrTy));
1113
1114 Constant *DataVals[] = {
1115#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1117 };
1118 Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1119
1120 Data->setVisibility(Visibility);
1121 Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
1122 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
1123 MaybeSetComdat(Data);
1124
1125 PD.DataVar = Data;
1126
1127 // Mark the data variable as used so that it isn't stripped out.
1128 CompilerUsedVars.push_back(Data);
1129 // Now that the linkage set by the FE has been passed to the data and counter
1130 // variables, reset Name variable's linkage and visibility to private so that
1131 // it can be removed later by the compiler.
1133 // Collect the referenced names to be used by emitNameData.
1134 ReferencedNames.push_back(NamePtr);
1135
1136 return PD.RegionCounters;
1137}
1138
1139void InstrProfiling::emitVNodes() {
1140 if (!ValueProfileStaticAlloc)
1141 return;
1142
1143 // For now only support this on platforms that do
1144 // not require runtime registration to discover
1145 // named section start/end.
1147 return;
1148
1149 size_t TotalNS = 0;
1150 for (auto &PD : ProfileDataMap) {
1151 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1152 TotalNS += PD.second.NumValueSites[Kind];
1153 }
1154
1155 if (!TotalNS)
1156 return;
1157
1158 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1159// Heuristic for small programs with very few total value sites.
1160// The default value of vp-counters-per-site is chosen based on
1161// the observation that large apps usually have a low percentage
1162// of value sites that actually have any profile data, and thus
1163// the average number of counters per site is low. For small
1164// apps with very few sites, this may not be true. Bump up the
1165// number of counters in this case.
1166#define INSTR_PROF_MIN_VAL_COUNTS 10
1167 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
1168 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
1169
1170 auto &Ctx = M->getContext();
1171 Type *VNodeTypes[] = {
1172#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1174 };
1175 auto *VNodeTy = StructType::get(Ctx, ArrayRef(VNodeTypes));
1176
1177 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
1178 auto *VNodesVar = new GlobalVariable(
1179 *M, VNodesTy, false, GlobalValue::PrivateLinkage,
1181 VNodesVar->setSection(
1182 getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
1183 VNodesVar->setAlignment(M->getDataLayout().getABITypeAlign(VNodesTy));
1184 // VNodesVar is used by runtime but not referenced via relocation by other
1185 // sections. Conservatively make it linker retained.
1186 UsedVars.push_back(VNodesVar);
1187}
1188
1189void InstrProfiling::emitNameData() {
1190 std::string UncompressedData;
1191
1192 if (ReferencedNames.empty())
1193 return;
1194
1195 std::string CompressedNameStr;
1196 if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1198 report_fatal_error(Twine(toString(std::move(E))), false);
1199 }
1200
1201 auto &Ctx = M->getContext();
1202 auto *NamesVal =
1203 ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);
1204 NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
1207 NamesSize = CompressedNameStr.size();
1208 NamesVar->setSection(
1209 getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1210 // On COFF, it's important to reduce the alignment down to 1 to prevent the
1211 // linker from inserting padding before the start of the names section or
1212 // between names entries.
1213 NamesVar->setAlignment(Align(1));
1214 // NamesVar is used by runtime but not referenced via relocation by other
1215 // sections. Conservatively make it linker retained.
1216 UsedVars.push_back(NamesVar);
1217
1218 for (auto *NamePtr : ReferencedNames)
1219 NamePtr->eraseFromParent();
1220}
1221
1222void InstrProfiling::emitRegistration() {
1224 return;
1225
1226 // Construct the function.
1227 auto *VoidTy = Type::getVoidTy(M->getContext());
1228 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
1229 auto *Int64Ty = Type::getInt64Ty(M->getContext());
1230 auto *RegisterFTy = FunctionType::get(VoidTy, false);
1231 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1233 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1234 if (Options.NoRedZone)
1235 RegisterF->addFnAttr(Attribute::NoRedZone);
1236
1237 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1238 auto *RuntimeRegisterF =
1241
1242 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1243 for (Value *Data : CompilerUsedVars)
1244 if (!isa<Function>(Data))
1245 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1246 for (Value *Data : UsedVars)
1247 if (Data != NamesVar && !isa<Function>(Data))
1248 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1249
1250 if (NamesVar) {
1251 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1252 auto *NamesRegisterTy =
1253 FunctionType::get(VoidTy, ArrayRef(ParamTypes), false);
1254 auto *NamesRegisterF =
1257 IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
1258 IRB.getInt64(NamesSize)});
1259 }
1260
1261 IRB.CreateRetVoid();
1262}
1263
1264bool InstrProfiling::emitRuntimeHook() {
1265 // We expect the linker to be invoked with -u<hook_var> flag for Linux
1266 // in which case there is no need to emit the external variable.
1267 if (TT.isOSLinux() || TT.isOSAIX())
1268 return false;
1269
1270 // If the module's provided its own runtime, we don't need to do anything.
1271 if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1272 return false;
1273
1274 // Declare an external variable that will pull in the runtime initialization.
1275 auto *Int32Ty = Type::getInt32Ty(M->getContext());
1276 auto *Var =
1279 Var->setVisibility(GlobalValue::HiddenVisibility);
1280
1281 if (TT.isOSBinFormatELF() && !TT.isPS()) {
1282 // Mark the user variable as used so that it isn't stripped out.
1283 CompilerUsedVars.push_back(Var);
1284 } else {
1285 // Make a function that uses it.
1289 User->addFnAttr(Attribute::NoInline);
1290 if (Options.NoRedZone)
1291 User->addFnAttr(Attribute::NoRedZone);
1292 User->setVisibility(GlobalValue::HiddenVisibility);
1293 if (TT.supportsCOMDAT())
1294 User->setComdat(M->getOrInsertComdat(User->getName()));
1295
1296 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1297 auto *Load = IRB.CreateLoad(Int32Ty, Var);
1298 IRB.CreateRet(Load);
1299
1300 // Mark the function as used so that it isn't stripped out.
1301 CompilerUsedVars.push_back(User);
1302 }
1303 return true;
1304}
1305
1306void InstrProfiling::emitUses() {
1307 // The metadata sections are parallel arrays. Optimizers (e.g.
1308 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1309 // we conservatively retain all unconditionally in the compiler.
1310 //
1311 // On ELF and Mach-O, the linker can guarantee the associated sections will be
1312 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
1313 // Similarly on COFF, if prof data is not referenced by code we use one comdat
1314 // and ensure this GC property as well. Otherwise, we have to conservatively
1315 // make all of the sections retained by the linker.
1316 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
1318 appendToCompilerUsed(*M, CompilerUsedVars);
1319 else
1320 appendToUsed(*M, CompilerUsedVars);
1321
1322 // We do not add proper references from used metadata sections to NamesVar and
1323 // VNodesVar, so we have to be conservative and place them in llvm.used
1324 // regardless of the target,
1325 appendToUsed(*M, UsedVars);
1326}
1327
1328void InstrProfiling::emitInitialization() {
1329 // Create ProfileFileName variable. Don't don't this for the
1330 // context-sensitive instrumentation lowering: This lowering is after
1331 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1332 // have already create the variable before LTO/ThinLTO linking.
1333 if (!IsCS)
1335 Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1336 if (!RegisterF)
1337 return;
1338
1339 // Create the initialization function.
1340 auto *VoidTy = Type::getVoidTy(M->getContext());
1341 auto *F = Function::Create(FunctionType::get(VoidTy, false),
1344 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1345 F->addFnAttr(Attribute::NoInline);
1346 if (Options.NoRedZone)
1347 F->addFnAttr(Attribute::NoRedZone);
1348
1349 // Add the basic block and the necessary calls.
1350 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1351 IRB.CreateCall(RegisterF, {});
1352 IRB.CreateRetVoid();
1353
1354 appendToGlobalCtors(*M, F, 0);
1355}
assume Assume Builder
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static unsigned InstrCount
#define LLVM_DEBUG(X)
Definition: Debug.h:101
@ Default
Definition: DwarfDebug.cpp:86
uint64_t Addr
std::string Name
Hexagon Hardware Loops
static bool enablesValueProfiling(const Module &M)
static bool shouldRecordFunctionAddr(Function *F)
static bool needsRuntimeHookUnconditionally(const Triple &TT)
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
#define INSTR_PROF_MIN_VAL_COUNTS
static Constant * getFuncAddrForProfData(Function *Fn)
static bool shouldUsePublicSymbol(Function *Fn)
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag)
static bool profDataReferencedByCode(const Module &M)
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Memory SSA
Definition: MemorySSA.cpp:71
Module.h This file contains the declarations for the Module class.
IntegerType * Int32Ty
if(VerifyEach)
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
@ Names
Definition: TextStubV5.cpp:106
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
Annotations lets you mark points and ranges inside source code, for tests:
Definition: Annotations.h:53
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:658
@ Add
*p = old + v
Definition: Instructions.h:734
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
iterator end()
Definition: BasicBlock.h:316
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:314
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:245
const Instruction & front() const
Definition: BasicBlock.h:326
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:105
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:127
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
This class represents a function call, abstracting a target machine's calling convention.
@ NoDeduplicate
No deduplication is performed.
Definition: Comdat.h:39
ConstantArray - Constant Array Declarations.
Definition: Constants.h:413
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1242
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:419
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2946
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2621
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2192
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2220
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:145
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1698
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1307
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:403
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:356
bool isZeroValue() const
Return true if the value is negative zero or null value.
Definition: Constants.cpp:62
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
Lightweight error class with error context and mandatory checking.
Definition: Error.h:156
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:136
const BasicBlock & getEntryBlock() const
Definition: Function.h:740
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1625
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:319
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
Definition: Globals.cpp:520
bool hasMetadata() const
Return true if this value has any metadata attached to it.
Definition: Value.h:585
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
Definition: Globals.cpp:130
void setComdat(Comdat *C)
Definition: Globals.cpp:198
bool hasComdat() const
Definition: GlobalObject.h:127
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:252
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:244
LinkageTypes getLinkage() const
Definition: GlobalValue.h:541
bool hasLocalLinkage() const
Definition: GlobalValue.h:523
bool hasPrivateLinkage() const
Definition: GlobalValue.h:522
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:532
bool isDeclarationForLinker() const
Definition: GlobalValue.h:614
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:63
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:468
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2558
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:933
static const char * FunctionNameAttributeName
static const char * CFGHashAttributeName
static const char * NumCountersAttributeName
This represents the llvm.instrprof.cover intrinsic.
This represents the llvm.instrprof.increment intrinsic.
A base class for all instrprof intrinsics.
ConstantInt * getNumCounters() const
GlobalVariable * getName() const
ConstantInt * getHash() const
ConstantInt * getIndex() const
This represents the llvm.instrprof.value.profile intrinsic.
ConstantInt * getIndex() const
ConstantInt * getValueKind() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
const BasicBlock * getParent() const
Definition: Instruction.h:90
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:74
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:82
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater.
Definition: SSAUpdater.h:147
virtual void doExtraRewritesBeforeFinalDeletion()
This hook is invoked after all the stores are found and inserted as available values.
Definition: SSAUpdater.h:172
An instruction for reading from memory.
Definition: Instructions.h:177
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
Definition: LoopInfoImpl.h:64
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
Definition: LoopInfoImpl.h:33
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:183
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
Definition: LoopInfoImpl.h:112
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
Definition: LoopInfoImpl.h:587
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:992
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:547
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1399
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:497
Root of the metadata hierarchy.
Definition: Metadata.h:61
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:39
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:558
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:426
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
Definition: Triple.h:382
bool isOSBinFormatMachO() const
Tests whether the environment is MachO.
Definition: Triple.h:688
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:976
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
Definition: Triple.h:680
bool isOSBinFormatXCOFF() const
Tests whether the OS uses the XCOFF binary format.
Definition: Triple.h:698
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:638
bool isOSAIX() const
Tests whether the OS is AIX.
Definition: Triple.h:670
bool isPS() const
Tests whether the target is the PS4 or PS5 platform.
Definition: Triple.h:722
bool isOSFuchsia() const
Definition: Triple.h:549
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:675
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
ValueKind
Value kinds.
const CustomOperand< const MCSubtargetInfo & > Msg[]
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:979
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:90
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:154
cl::opt< bool > DoInstrProfNameCompression
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:748
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:93
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:121
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:213
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:149
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:99
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1826
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:164
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:511
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:160
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:130
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:96
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:477
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:85
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:141
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
@ Add
Sum of integers.
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1135
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1183
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1206
@ DS_Warning
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:71
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:439
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:79
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:136
cl::opt< bool > DebugInfoCorrelate("debug-info-correlate", cl::desc("Use debug info to correlate profiles."), cl::init(false))
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:106
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1161
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:102
#define NC
Definition: regutils.h:42
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
std::string InstrProfileOutput
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117