LLVM 18.0.0git
MemProfiler.cpp
Go to the documentation of this file.
1//===- MemProfiler.cpp - memory allocation and access profiler ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of MemProfiler. Memory accesses are instrumented
10// to increment the access count held in a shadow memory location, or
11// alternatively to call into the runtime. Memory intrinsic calls (memmove,
12// memcpy, memset) are changed to call the memory profiling runtime version
13// instead.
14//
15//===----------------------------------------------------------------------===//
16
19#include "llvm/ADT/Statistic.h"
20#include "llvm/ADT/StringRef.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/GlobalValue.h"
29#include "llvm/IR/IRBuilder.h"
30#include "llvm/IR/Instruction.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/Type.h"
34#include "llvm/IR/Value.h"
37#include "llvm/Support/BLAKE3.h"
39#include "llvm/Support/Debug.h"
45#include <map>
46#include <set>
47
48using namespace llvm;
49using namespace llvm::memprof;
50
51#define DEBUG_TYPE "memprof"
52
53namespace llvm {
57} // namespace llvm
58
59constexpr int LLVM_MEM_PROFILER_VERSION = 1;
60
61// Size of memory mapped to a single shadow location.
63
64// Scale from granularity down to shadow size.
66
67constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
69// On Emscripten, the system needs more than one priorities for constructors.
71constexpr char MemProfInitName[] = "__memprof_init";
73 "__memprof_version_mismatch_check_v";
74
76 "__memprof_shadow_memory_dynamic_address";
77
78constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
79
80// Command-line flags.
81
83 "memprof-guard-against-version-mismatch",
84 cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
85 cl::init(true));
86
87// This flag may need to be replaced with -f[no-]memprof-reads.
88static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
89 cl::desc("instrument read instructions"),
90 cl::Hidden, cl::init(true));
91
92static cl::opt<bool>
93 ClInstrumentWrites("memprof-instrument-writes",
94 cl::desc("instrument write instructions"), cl::Hidden,
95 cl::init(true));
96
98 "memprof-instrument-atomics",
99 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
100 cl::init(true));
101
103 "memprof-use-callbacks",
104 cl::desc("Use callbacks instead of inline instrumentation sequences."),
105 cl::Hidden, cl::init(false));
106
108 ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
109 cl::desc("Prefix for memory access callbacks"),
110 cl::Hidden, cl::init("__memprof_"));
111
112// These flags allow to change the shadow mapping.
113// The shadow mapping looks like
114// Shadow = ((Mem & mask) >> scale) + offset
115
116static cl::opt<int> ClMappingScale("memprof-mapping-scale",
117 cl::desc("scale of memprof shadow mapping"),
119
120static cl::opt<int>
121 ClMappingGranularity("memprof-mapping-granularity",
122 cl::desc("granularity of memprof shadow mapping"),
124
125static cl::opt<bool> ClStack("memprof-instrument-stack",
126 cl::desc("Instrument scalar stack variables"),
127 cl::Hidden, cl::init(false));
128
129// Debug flags.
130
131static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
132 cl::init(0));
133
134static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
135 cl::desc("Debug func"));
136
137static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
138 cl::Hidden, cl::init(-1));
139
140static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
141 cl::Hidden, cl::init(-1));
142
143STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
144STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
145STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
146STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
147STATISTIC(NumOfMemProfMissing, "Number of functions without memory profile.");
148
149namespace {
150
151/// This struct defines the shadow mapping using the rule:
152/// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
153struct ShadowMapping {
154 ShadowMapping() {
155 Scale = ClMappingScale;
156 Granularity = ClMappingGranularity;
157 Mask = ~(Granularity - 1);
158 }
159
160 int Scale;
161 int Granularity;
162 uint64_t Mask; // Computed as ~(Granularity-1)
163};
164
165static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
168}
169
170struct InterestingMemoryAccess {
171 Value *Addr = nullptr;
172 bool IsWrite;
173 Type *AccessTy;
175 Value *MaybeMask = nullptr;
176};
177
178/// Instrument the code in module to profile memory accesses.
179class MemProfiler {
180public:
181 MemProfiler(Module &M) {
182 C = &(M.getContext());
183 LongSize = M.getDataLayout().getPointerSizeInBits();
184 IntptrTy = Type::getIntNTy(*C, LongSize);
185 }
186
187 /// If it is an interesting memory access, populate information
188 /// about the access and return a InterestingMemoryAccess struct.
189 /// Otherwise return std::nullopt.
190 std::optional<InterestingMemoryAccess>
191 isInterestingMemoryAccess(Instruction *I) const;
192
193 void instrumentMop(Instruction *I, const DataLayout &DL,
194 InterestingMemoryAccess &Access);
195 void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
196 Value *Addr, uint32_t TypeSize, bool IsWrite);
197 void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
198 Instruction *I, Value *Addr, Type *AccessTy,
199 bool IsWrite);
200 void instrumentMemIntrinsic(MemIntrinsic *MI);
201 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
202 bool instrumentFunction(Function &F);
203 bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
204 bool insertDynamicShadowAtFunctionEntry(Function &F);
205
206private:
207 void initializeCallbacks(Module &M);
208
209 LLVMContext *C;
210 int LongSize;
211 Type *IntptrTy;
212 ShadowMapping Mapping;
213
214 // These arrays is indexed by AccessIsWrite
215 FunctionCallee MemProfMemoryAccessCallback[2];
216 FunctionCallee MemProfMemoryAccessCallbackSized[2];
217
218 FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
219 Value *DynamicShadowOffset = nullptr;
220};
221
222class ModuleMemProfiler {
223public:
224 ModuleMemProfiler(Module &M) { TargetTriple = Triple(M.getTargetTriple()); }
225
226 bool instrumentModule(Module &);
227
228private:
229 Triple TargetTriple;
230 ShadowMapping Mapping;
231 Function *MemProfCtorFunction = nullptr;
232};
233
234} // end anonymous namespace
235
237
240 Module &M = *F.getParent();
241 MemProfiler Profiler(M);
242 if (Profiler.instrumentFunction(F))
244 return PreservedAnalyses::all();
245}
246
248
251 ModuleMemProfiler Profiler(M);
252 if (Profiler.instrumentModule(M))
254 return PreservedAnalyses::all();
255}
256
257Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
258 // (Shadow & mask) >> scale
259 Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
260 Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
261 // (Shadow >> scale) | offset
262 assert(DynamicShadowOffset);
263 return IRB.CreateAdd(Shadow, DynamicShadowOffset);
264}
265
266// Instrument memset/memmove/memcpy
267void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
268 IRBuilder<> IRB(MI);
269 if (isa<MemTransferInst>(MI)) {
270 IRB.CreateCall(
271 isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
272 {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
273 IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
274 IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
275 } else if (isa<MemSetInst>(MI)) {
276 IRB.CreateCall(
277 MemProfMemset,
278 {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
279 IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
280 IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
281 }
282 MI->eraseFromParent();
283}
284
285std::optional<InterestingMemoryAccess>
286MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
287 // Do not instrument the load fetching the dynamic shadow address.
288 if (DynamicShadowOffset == I)
289 return std::nullopt;
290
291 InterestingMemoryAccess Access;
292
293 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
295 return std::nullopt;
296 Access.IsWrite = false;
297 Access.AccessTy = LI->getType();
298 Access.Addr = LI->getPointerOperand();
299 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
301 return std::nullopt;
302 Access.IsWrite = true;
303 Access.AccessTy = SI->getValueOperand()->getType();
304 Access.Addr = SI->getPointerOperand();
305 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
307 return std::nullopt;
308 Access.IsWrite = true;
309 Access.AccessTy = RMW->getValOperand()->getType();
310 Access.Addr = RMW->getPointerOperand();
311 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
313 return std::nullopt;
314 Access.IsWrite = true;
315 Access.AccessTy = XCHG->getCompareOperand()->getType();
316 Access.Addr = XCHG->getPointerOperand();
317 } else if (auto *CI = dyn_cast<CallInst>(I)) {
318 auto *F = CI->getCalledFunction();
319 if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
320 F->getIntrinsicID() == Intrinsic::masked_store)) {
321 unsigned OpOffset = 0;
322 if (F->getIntrinsicID() == Intrinsic::masked_store) {
324 return std::nullopt;
325 // Masked store has an initial operand for the value.
326 OpOffset = 1;
327 Access.AccessTy = CI->getArgOperand(0)->getType();
328 Access.IsWrite = true;
329 } else {
331 return std::nullopt;
332 Access.AccessTy = CI->getType();
333 Access.IsWrite = false;
334 }
335
336 auto *BasePtr = CI->getOperand(0 + OpOffset);
337 Access.MaybeMask = CI->getOperand(2 + OpOffset);
338 Access.Addr = BasePtr;
339 }
340 }
341
342 if (!Access.Addr)
343 return std::nullopt;
344
345 // Do not instrument accesses from different address spaces; we cannot deal
346 // with them.
347 Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
348 if (PtrTy->getPointerAddressSpace() != 0)
349 return std::nullopt;
350
351 // Ignore swifterror addresses.
352 // swifterror memory addresses are mem2reg promoted by instruction
353 // selection. As such they cannot have regular uses like an instrumentation
354 // function and it makes no sense to track them as memory.
355 if (Access.Addr->isSwiftError())
356 return std::nullopt;
357
358 // Peel off GEPs and BitCasts.
359 auto *Addr = Access.Addr->stripInBoundsOffsets();
360
361 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
362 // Do not instrument PGO counter updates.
363 if (GV->hasSection()) {
364 StringRef SectionName = GV->getSection();
365 // Check if the global is in the PGO counters section.
366 auto OF = Triple(I->getModule()->getTargetTriple()).getObjectFormat();
367 if (SectionName.endswith(
368 getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
369 return std::nullopt;
370 }
371
372 // Do not instrument accesses to LLVM internal variables.
373 if (GV->getName().startswith("__llvm"))
374 return std::nullopt;
375 }
376
377 const DataLayout &DL = I->getModule()->getDataLayout();
378 Access.TypeSize = DL.getTypeStoreSizeInBits(Access.AccessTy);
379 return Access;
380}
381
382void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
384 Type *AccessTy, bool IsWrite) {
385 auto *VTy = cast<FixedVectorType>(AccessTy);
386 uint64_t ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType());
387 unsigned Num = VTy->getNumElements();
388 auto *Zero = ConstantInt::get(IntptrTy, 0);
389 for (unsigned Idx = 0; Idx < Num; ++Idx) {
390 Value *InstrumentedAddress = nullptr;
391 Instruction *InsertBefore = I;
392 if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
393 // dyn_cast as we might get UndefValue
394 if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
395 if (Masked->isZero())
396 // Mask is constant false, so no instrumentation needed.
397 continue;
398 // If we have a true or undef value, fall through to instrumentAddress.
399 // with InsertBefore == I
400 }
401 } else {
402 IRBuilder<> IRB(I);
403 Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
404 Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
405 InsertBefore = ThenTerm;
406 }
407
408 IRBuilder<> IRB(InsertBefore);
409 InstrumentedAddress =
410 IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
411 instrumentAddress(I, InsertBefore, InstrumentedAddress, ElemTypeSize,
412 IsWrite);
413 }
414}
415
416void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
417 InterestingMemoryAccess &Access) {
418 // Skip instrumentation of stack accesses unless requested.
419 if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {
420 if (Access.IsWrite)
421 ++NumSkippedStackWrites;
422 else
423 ++NumSkippedStackReads;
424 return;
425 }
426
427 if (Access.IsWrite)
428 NumInstrumentedWrites++;
429 else
430 NumInstrumentedReads++;
431
432 if (Access.MaybeMask) {
433 instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
434 Access.AccessTy, Access.IsWrite);
435 } else {
436 // Since the access counts will be accumulated across the entire allocation,
437 // we only update the shadow access count for the first location and thus
438 // don't need to worry about alignment and type size.
439 instrumentAddress(I, I, Access.Addr, Access.TypeSize, Access.IsWrite);
440 }
441}
442
443void MemProfiler::instrumentAddress(Instruction *OrigIns,
444 Instruction *InsertBefore, Value *Addr,
445 uint32_t TypeSize, bool IsWrite) {
446 IRBuilder<> IRB(InsertBefore);
447 Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
448
449 if (ClUseCalls) {
450 IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
451 return;
452 }
453
454 // Create an inline sequence to compute shadow location, and increment the
455 // value by one.
456 Type *ShadowTy = Type::getInt64Ty(*C);
457 Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
458 Value *ShadowPtr = memToShadow(AddrLong, IRB);
459 Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
460 Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
462 ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
463 IRB.CreateStore(ShadowValue, ShadowAddr);
464}
465
466// Create the variable for the profile file name.
468 const MDString *MemProfFilename =
469 dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
470 if (!MemProfFilename)
471 return;
472 assert(!MemProfFilename->getString().empty() &&
473 "Unexpected MemProfProfileFilename metadata with empty string");
474 Constant *ProfileNameConst = ConstantDataArray::getString(
475 M.getContext(), MemProfFilename->getString(), true);
476 GlobalVariable *ProfileNameVar = new GlobalVariable(
477 M, ProfileNameConst->getType(), /*isConstant=*/true,
479 Triple TT(M.getTargetTriple());
480 if (TT.supportsCOMDAT()) {
482 ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
483 }
484}
485
486bool ModuleMemProfiler::instrumentModule(Module &M) {
487 // Create a module constructor.
488 std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
489 std::string VersionCheckName =
491 : "";
492 std::tie(MemProfCtorFunction, std::ignore) =
494 MemProfInitName, /*InitArgTypes=*/{},
495 /*InitArgs=*/{}, VersionCheckName);
496
497 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
498 appendToGlobalCtors(M, MemProfCtorFunction, Priority);
499
501
502 return true;
503}
504
505void MemProfiler::initializeCallbacks(Module &M) {
506 IRBuilder<> IRB(*C);
507
508 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
509 const std::string TypeStr = AccessIsWrite ? "store" : "load";
510
511 SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
512 SmallVector<Type *, 2> Args1{1, IntptrTy};
513 MemProfMemoryAccessCallbackSized[AccessIsWrite] =
514 M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr + "N",
515 FunctionType::get(IRB.getVoidTy(), Args2, false));
516
517 MemProfMemoryAccessCallback[AccessIsWrite] =
518 M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr,
519 FunctionType::get(IRB.getVoidTy(), Args1, false));
520 }
521 MemProfMemmove = M.getOrInsertFunction(
523 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
524 MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
525 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
526 IRB.getInt8PtrTy(), IntptrTy);
527 MemProfMemset = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset",
528 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
529 IRB.getInt32Ty(), IntptrTy);
530}
531
532bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
533 // For each NSObject descendant having a +load method, this method is invoked
534 // by the ObjC runtime before any of the static constructors is called.
535 // Therefore we need to instrument such methods with a call to __memprof_init
536 // at the beginning in order to initialize our runtime before any access to
537 // the shadow memory.
538 // We cannot just ignore these methods, because they may call other
539 // instrumented functions.
540 if (F.getName().find(" load]") != std::string::npos) {
541 FunctionCallee MemProfInitFunction =
543 IRBuilder<> IRB(&F.front(), F.front().begin());
544 IRB.CreateCall(MemProfInitFunction, {});
545 return true;
546 }
547 return false;
548}
549
550bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
551 IRBuilder<> IRB(&F.front().front());
552 Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
554 if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
555 cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
556 DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
557 return true;
558}
559
560bool MemProfiler::instrumentFunction(Function &F) {
561 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
562 return false;
563 if (ClDebugFunc == F.getName())
564 return false;
565 if (F.getName().startswith("__memprof_"))
566 return false;
567
568 bool FunctionModified = false;
569
570 // If needed, insert __memprof_init.
571 // This function needs to be called even if the function body is not
572 // instrumented.
573 if (maybeInsertMemProfInitAtFunctionEntry(F))
574 FunctionModified = true;
575
576 LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
577
578 initializeCallbacks(*F.getParent());
579
581
582 // Fill the set of memory operations to instrument.
583 for (auto &BB : F) {
584 for (auto &Inst : BB) {
585 if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
586 ToInstrument.push_back(&Inst);
587 }
588 }
589
590 if (ToInstrument.empty()) {
591 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
592 << " " << F << "\n");
593
594 return FunctionModified;
595 }
596
597 FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
598
599 int NumInstrumented = 0;
600 for (auto *Inst : ToInstrument) {
601 if (ClDebugMin < 0 || ClDebugMax < 0 ||
602 (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
603 std::optional<InterestingMemoryAccess> Access =
604 isInterestingMemoryAccess(Inst);
605 if (Access)
606 instrumentMop(Inst, F.getParent()->getDataLayout(), *Access);
607 else
608 instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
609 }
610 NumInstrumented++;
611 }
612
613 if (NumInstrumented > 0)
614 FunctionModified = true;
615
616 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
617 << F << "\n");
618
619 return FunctionModified;
620}
621
623 std::vector<uint64_t> &InlinedCallStack,
624 LLVMContext &Ctx) {
625 I.setMetadata(LLVMContext::MD_callsite,
626 buildCallstackMetadata(InlinedCallStack, Ctx));
627}
628
630 uint32_t Column) {
631 llvm::HashBuilder<llvm::TruncatedBLAKE3<8>, llvm::support::endianness::little>
633 HashBuilder.add(Function, LineOffset, Column);
635 uint64_t Id;
636 std::memcpy(&Id, Hash.data(), sizeof(Hash));
637 return Id;
638}
639
642}
643
644static void addCallStack(CallStackTrie &AllocTrie,
645 const AllocationInfo *AllocInfo) {
646 SmallVector<uint64_t> StackIds;
647 for (const auto &StackFrame : AllocInfo->CallStack)
648 StackIds.push_back(computeStackId(StackFrame));
649 auto AllocType = getAllocType(AllocInfo->Info.getTotalLifetimeAccessDensity(),
650 AllocInfo->Info.getAllocCount(),
651 AllocInfo->Info.getTotalLifetime());
652 AllocTrie.addCallStack(AllocType, StackIds);
653}
654
655// Helper to compare the InlinedCallStack computed from an instruction's debug
656// info to a list of Frames from profile data (either the allocation data or a
657// callsite). For callsites, the StartIndex to use in the Frame array may be
658// non-zero.
659static bool
661 ArrayRef<uint64_t> InlinedCallStack,
662 unsigned StartIndex = 0) {
663 auto StackFrame = ProfileCallStack.begin() + StartIndex;
664 auto InlCallStackIter = InlinedCallStack.begin();
665 for (; StackFrame != ProfileCallStack.end() &&
666 InlCallStackIter != InlinedCallStack.end();
667 ++StackFrame, ++InlCallStackIter) {
668 uint64_t StackId = computeStackId(*StackFrame);
669 if (StackId != *InlCallStackIter)
670 return false;
671 }
672 // Return true if we found and matched all stack ids from the call
673 // instruction.
674 return InlCallStackIter == InlinedCallStack.end();
675}
676
677static void readMemprof(Module &M, Function &F,
679 const TargetLibraryInfo &TLI) {
680 auto &Ctx = M.getContext();
681
682 auto FuncName = getIRPGOFuncName(F);
683 auto FuncGUID = Function::getGUID(FuncName);
684 std::optional<memprof::MemProfRecord> MemProfRec;
685 auto Err = MemProfReader->getMemProfRecord(FuncGUID).moveInto(MemProfRec);
686 if (Err) {
687 // If we don't find getIRPGOFuncName(), try getPGOFuncName() to handle
688 // profiles built by older compilers
689 Err = handleErrors(std::move(Err), [&](const InstrProfError &IE) -> Error {
690 if (IE.get() != instrprof_error::unknown_function)
691 return make_error<InstrProfError>(IE);
692 auto FuncName = getPGOFuncName(F);
693 auto FuncGUID = Function::getGUID(FuncName);
694 if (auto Err =
695 MemProfReader->getMemProfRecord(FuncGUID).moveInto(MemProfRec))
696 return Err;
697 return Error::success();
698 });
699 }
700 if (Err) {
701 handleAllErrors(std::move(Err), [&](const InstrProfError &IPE) {
702 auto Err = IPE.get();
703 bool SkipWarning = false;
704 LLVM_DEBUG(dbgs() << "Error in reading profile for Func " << FuncName
705 << ": ");
707 NumOfMemProfMissing++;
708 SkipWarning = !PGOWarnMissing;
709 LLVM_DEBUG(dbgs() << "unknown function");
710 } else if (Err == instrprof_error::hash_mismatch) {
711 SkipWarning =
714 (F.hasComdat() ||
716 LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")");
717 }
718
719 if (SkipWarning)
720 return;
721
722 std::string Msg = (IPE.message() + Twine(" ") + F.getName().str() +
723 Twine(" Hash = ") + std::to_string(FuncGUID))
724 .str();
725
726 Ctx.diagnose(
727 DiagnosticInfoPGOProfile(M.getName().data(), Msg, DS_Warning));
728 });
729 return;
730 }
731
732 // Build maps of the location hash to all profile data with that leaf location
733 // (allocation info and the callsites).
734 std::map<uint64_t, std::set<const AllocationInfo *>> LocHashToAllocInfo;
735 // For the callsites we need to record the index of the associated frame in
736 // the frame array (see comments below where the map entries are added).
737 std::map<uint64_t, std::set<std::pair<const SmallVector<Frame> *, unsigned>>>
738 LocHashToCallSites;
739 for (auto &AI : MemProfRec->AllocSites) {
740 // Associate the allocation info with the leaf frame. The later matching
741 // code will match any inlined call sequences in the IR with a longer prefix
742 // of call stack frames.
743 uint64_t StackId = computeStackId(AI.CallStack[0]);
744 LocHashToAllocInfo[StackId].insert(&AI);
745 }
746 for (auto &CS : MemProfRec->CallSites) {
747 // Need to record all frames from leaf up to and including this function,
748 // as any of these may or may not have been inlined at this point.
749 unsigned Idx = 0;
750 for (auto &StackFrame : CS) {
751 uint64_t StackId = computeStackId(StackFrame);
752 LocHashToCallSites[StackId].insert(std::make_pair(&CS, Idx++));
753 // Once we find this function, we can stop recording.
754 if (StackFrame.Function == FuncGUID)
755 break;
756 }
757 assert(Idx <= CS.size() && CS[Idx - 1].Function == FuncGUID);
758 }
759
760 auto GetOffset = [](const DILocation *DIL) {
761 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &
762 0xffff;
763 };
764
765 // Now walk the instructions, looking up the associated profile data using
766 // dbug locations.
767 for (auto &BB : F) {
768 for (auto &I : BB) {
769 if (I.isDebugOrPseudoInst())
770 continue;
771 // We are only interested in calls (allocation or interior call stack
772 // context calls).
773 auto *CI = dyn_cast<CallBase>(&I);
774 if (!CI)
775 continue;
776 auto *CalledFunction = CI->getCalledFunction();
777 if (CalledFunction && CalledFunction->isIntrinsic())
778 continue;
779 // List of call stack ids computed from the location hashes on debug
780 // locations (leaf to inlined at root).
781 std::vector<uint64_t> InlinedCallStack;
782 // Was the leaf location found in one of the profile maps?
783 bool LeafFound = false;
784 // If leaf was found in a map, iterators pointing to its location in both
785 // of the maps. It might exist in neither, one, or both (the latter case
786 // can happen because we don't currently have discriminators to
787 // distinguish the case when a single line/col maps to both an allocation
788 // and another callsite).
789 std::map<uint64_t, std::set<const AllocationInfo *>>::iterator
790 AllocInfoIter;
791 std::map<uint64_t, std::set<std::pair<const SmallVector<Frame> *,
792 unsigned>>>::iterator CallSitesIter;
793 for (const DILocation *DIL = I.getDebugLoc(); DIL != nullptr;
794 DIL = DIL->getInlinedAt()) {
795 // Use C++ linkage name if possible. Need to compile with
796 // -fdebug-info-for-profiling to get linkage name.
797 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
798 if (Name.empty())
799 Name = DIL->getScope()->getSubprogram()->getName();
800 auto CalleeGUID = Function::getGUID(Name);
801 auto StackId =
802 computeStackId(CalleeGUID, GetOffset(DIL), DIL->getColumn());
803 // LeafFound will only be false on the first iteration, since we either
804 // set it true or break out of the loop below.
805 if (!LeafFound) {
806 AllocInfoIter = LocHashToAllocInfo.find(StackId);
807 CallSitesIter = LocHashToCallSites.find(StackId);
808 // Check if the leaf is in one of the maps. If not, no need to look
809 // further at this call.
810 if (AllocInfoIter == LocHashToAllocInfo.end() &&
811 CallSitesIter == LocHashToCallSites.end())
812 break;
813 LeafFound = true;
814 }
815 InlinedCallStack.push_back(StackId);
816 }
817 // If leaf not in either of the maps, skip inst.
818 if (!LeafFound)
819 continue;
820
821 // First add !memprof metadata from allocation info, if we found the
822 // instruction's leaf location in that map, and if the rest of the
823 // instruction's locations match the prefix Frame locations on an
824 // allocation context with the same leaf.
825 if (AllocInfoIter != LocHashToAllocInfo.end()) {
826 // Only consider allocations via new, to reduce unnecessary metadata,
827 // since those are the only allocations that will be targeted initially.
828 if (!isNewLikeFn(CI, &TLI))
829 continue;
830 // We may match this instruction's location list to multiple MIB
831 // contexts. Add them to a Trie specialized for trimming the contexts to
832 // the minimal needed to disambiguate contexts with unique behavior.
833 CallStackTrie AllocTrie;
834 for (auto *AllocInfo : AllocInfoIter->second) {
835 // Check the full inlined call stack against this one.
836 // If we found and thus matched all frames on the call, include
837 // this MIB.
839 InlinedCallStack))
840 addCallStack(AllocTrie, AllocInfo);
841 }
842 // We might not have matched any to the full inlined call stack.
843 // But if we did, create and attach metadata, or a function attribute if
844 // all contexts have identical profiled behavior.
845 if (!AllocTrie.empty()) {
846 // MemprofMDAttached will be false if a function attribute was
847 // attached.
848 bool MemprofMDAttached = AllocTrie.buildAndAttachMIBMetadata(CI);
849 assert(MemprofMDAttached == I.hasMetadata(LLVMContext::MD_memprof));
850 if (MemprofMDAttached) {
851 // Add callsite metadata for the instruction's location list so that
852 // it simpler later on to identify which part of the MIB contexts
853 // are from this particular instruction (including during inlining,
854 // when the callsite metdata will be updated appropriately).
855 // FIXME: can this be changed to strip out the matching stack
856 // context ids from the MIB contexts and not add any callsite
857 // metadata here to save space?
858 addCallsiteMetadata(I, InlinedCallStack, Ctx);
859 }
860 }
861 continue;
862 }
863
864 // Otherwise, add callsite metadata. If we reach here then we found the
865 // instruction's leaf location in the callsites map and not the allocation
866 // map.
867 assert(CallSitesIter != LocHashToCallSites.end());
868 for (auto CallStackIdx : CallSitesIter->second) {
869 // If we found and thus matched all frames on the call, create and
870 // attach call stack metadata.
872 *CallStackIdx.first, InlinedCallStack, CallStackIdx.second)) {
873 addCallsiteMetadata(I, InlinedCallStack, Ctx);
874 // Only need to find one with a matching call stack and add a single
875 // callsite metadata.
876 break;
877 }
878 }
879 }
880 }
881}
882
883MemProfUsePass::MemProfUsePass(std::string MemoryProfileFile,
885 : MemoryProfileFileName(MemoryProfileFile), FS(FS) {
886 if (!FS)
887 this->FS = vfs::getRealFileSystem();
888}
889
891 LLVM_DEBUG(dbgs() << "Read in memory profile:");
892 auto &Ctx = M.getContext();
893 auto ReaderOrErr = IndexedInstrProfReader::create(MemoryProfileFileName, *FS);
894 if (Error E = ReaderOrErr.takeError()) {
895 handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) {
896 Ctx.diagnose(
897 DiagnosticInfoPGOProfile(MemoryProfileFileName.data(), EI.message()));
898 });
899 return PreservedAnalyses::all();
900 }
901
902 std::unique_ptr<IndexedInstrProfReader> MemProfReader =
903 std::move(ReaderOrErr.get());
904 if (!MemProfReader) {
905 Ctx.diagnose(DiagnosticInfoPGOProfile(
906 MemoryProfileFileName.data(), StringRef("Cannot get MemProfReader")));
907 return PreservedAnalyses::all();
908 }
909
910 if (!MemProfReader->hasMemoryProfile()) {
911 Ctx.diagnose(DiagnosticInfoPGOProfile(MemoryProfileFileName.data(),
912 "Not a memory profile"));
913 return PreservedAnalyses::all();
914 }
915
916 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
917
918 for (auto &F : M) {
919 if (F.isDeclaration())
920 continue;
921
923 readMemprof(M, F, MemProfReader.get(), TLI);
924 }
925
927}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< int > ClDebugMin("asan-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
static cl::opt< std::string > ClMemoryAccessCallbackPrefix("asan-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__asan_"))
static cl::opt< bool > ClInsertVersionCheck("asan-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
static cl::opt< bool > ClInstrumentWrites("asan-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
static cl::opt< int > ClDebugMax("asan-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
static cl::opt< bool > ClStack("asan-stack", cl::desc("Handle stack memory"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClInstrumentAtomics("asan-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
static cl::opt< int > ClMappingScale("asan-mapping-scale", cl::desc("scale of asan shadow mapping"), cl::Hidden, cl::init(0))
static cl::opt< std::string > ClDebugFunc("asan-debug-func", cl::Hidden, cl::desc("Debug func"))
static cl::opt< bool > ClInstrumentReads("asan-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
std::string Name
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
constexpr char MemProfVersionCheckNamePrefix[]
Definition: MemProfiler.cpp:72
static cl::opt< int > ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority
Definition: MemProfiler.cpp:70
static cl::opt< std::string > ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func"))
constexpr char MemProfShadowMemoryDynamicAddress[]
Definition: MemProfiler.cpp:75
static void addCallStack(CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo)
constexpr uint64_t MemProfCtorAndDtorPriority
Definition: MemProfiler.cpp:68
constexpr int LLVM_MEM_PROFILER_VERSION
Definition: MemProfiler.cpp:59
static cl::opt< bool > ClUseCalls("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false))
static cl::opt< bool > ClInstrumentAtomics("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClInsertVersionCheck("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
constexpr char MemProfInitName[]
Definition: MemProfiler.cpp:71
constexpr char MemProfFilenameVar[]
Definition: MemProfiler.cpp:78
static uint64_t computeStackId(GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column)
static cl::opt< bool > ClStack("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false))
constexpr uint64_t DefaultShadowGranularity
Definition: MemProfiler.cpp:62
constexpr uint64_t DefaultShadowScale
Definition: MemProfiler.cpp:65
static cl::opt< std::string > ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_"))
constexpr char MemProfModuleCtorName[]
Definition: MemProfiler.cpp:67
static cl::opt< bool > ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
static cl::opt< int > ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
static void readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI)
static cl::opt< bool > ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
static cl::opt< int > ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0))
static cl::opt< int > ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale))
static cl::opt< int > ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowGranularity))
static void addCallsiteMetadata(Instruction &I, std::vector< uint64_t > &InlinedCallStack, LLVMContext &Ctx)
static bool stackFrameIncludesInlinedCallStack(ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack, unsigned StartIndex=0)
AllocType
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
Defines the virtual file system interface vfs::FileSystem.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
iterator begin() const
Definition: ArrayRef.h:153
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:513
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:718
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2890
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
This is an important base class in LLVM.
Definition: Constant.h:41
Debug location.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
Diagnostic information for the PGO profiler.
Base class for error info classes.
Definition: Error.h:45
virtual std::string message() const
Return the error message as a string.
Definition: Error.h:53
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
void setComdat(Comdat *C)
Definition: Globals.cpp:196
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:532
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:591
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:52
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:49
HashResultTy< HasherT_ > final()
Forward to HasherT::final() if available.
Definition: HashBuilder.h:66
Implementation of the HashBuilder interface.
Definition: HashBuilder.h:94
std::enable_if_t< hashbuilder_detail::IsHashableData< T >::value, HashBuilderImpl & > add(T Value)
Implement hashing for hashable data types, e.g. integral or enum values.
Definition: HashBuilder.h:109
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2422
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2135
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2084
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1428
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:512
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1786
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
Definition: IRBuilder.h:560
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1466
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1799
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1318
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2158
Type * getVoidTy()
Fetch the type representing void.
Definition: IRBuilder.h:550
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2374
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1862
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2628
Reader for the indexed binary instrprof format.
static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")
Factory method to create an indexed reader.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:933
instrprof_error get() const
Definition: InstrProf.h:382
std::string message() const override
Return the error message as a string.
Definition: InstrProf.cpp:237
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:177
A single uniqued string.
Definition: Metadata.h:611
StringRef getString() const
Definition: Metadata.cpp:509
This is the common base class for memset/memcpy/memmove.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
MemProfUsePass(std::string MemoryProfileFile, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
bool empty() const
Definition: SmallVector.h:94
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
An instruction for storing to memory.
Definition: Instructions.h:301
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
Definition: Triple.h:381
bool isOSEmscripten() const
Tests whether the OS is Emscripten.
Definition: Triple.h:658
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
Class to build a trie of call stack contexts for a particular profiled allocation call,...
void addCallStack(AllocationType AllocType, ArrayRef< uint64_t > StackIds)
Add a call stack context with the given allocation type to the Trie.
bool buildAndAttachMIBMetadata(CallBase *CI)
Build and attach the minimal necessary MIB metadata.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
MDNode * buildCallstackMetadata(ArrayRef< uint64_t > CallStack, LLVMContext &Ctx)
Build callstack metadata from the provided list of call stack ids.
AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity, uint64_t AllocCount, uint64_t TotalLifetime)
Return the allocation type for a given set of memory profile values.
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Return the modified name for function F suitable to be used the key for profile lookup.
Definition: InstrProf.cpp:342
std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
Definition: InstrProf.cpp:313
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
Definition: Error.h:970
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
Definition: Error.h:947
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
cl::opt< bool > PGOWarnMissing
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:218
std::array< uint8_t, NumBytes > BLAKE3Result
The constant LLVM_BLAKE3_OUT_LEN provides the default output length, 32 bytes, which is recommended f...
Definition: BLAKE3.h:35
FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName, ArrayRef< Type * > InitArgTypes, bool Weak=false)
std::pair< Function *, FunctionCallee > createSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, StringRef VersionCheckName=StringRef(), bool Weak=false)
Creates sanitizer constructor function, and calls sanitizer's init function from it.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
HashBuilderImpl< HasherT,(Endianness==support::endianness::native ? support::endian::system_endianness() :Endianness)> HashBuilder
Interface to help hash various types through a hasher type.
Definition: HashBuilder.h:401
cl::opt< bool > NoPGOWarnMismatch
Definition: MemProfiler.cpp:55
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1306
@ DS_Warning
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:73
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
bool isNewLikeFn(const Value *V, const TargetLibraryInfo *TLI)
Tests if a value is a call or invoke to a library function that allocates memory via new.
cl::opt< bool > NoPGOWarnMismatchComdatWeak
Summary of memprof metadata on allocations.
GlobalValue::GUID Function
Definition: MemProf.h:145
uint32_t LineOffset
Definition: MemProf.h:150