LLVM  15.0.0git
MemProfiler.cpp
Go to the documentation of this file.
1 //===- MemProfiler.cpp - memory allocation and access profiler ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of MemProfiler. Memory accesses are instrumented
10 // to increment the access count held in a shadow memory location, or
11 // alternatively to call into the runtime. Memory intrinsic calls (memmove,
12 // memcpy, memset) are changed to call the memory profiling runtime version
13 // instead.
14 //
15 //===----------------------------------------------------------------------===//
16 
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/ADT/Triple.h"
23 #include "llvm/IR/Constant.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/IR/Function.h"
26 #include "llvm/IR/GlobalValue.h"
27 #include "llvm/IR/IRBuilder.h"
28 #include "llvm/IR/Instruction.h"
29 #include "llvm/IR/IntrinsicInst.h"
30 #include "llvm/IR/Module.h"
31 #include "llvm/IR/Type.h"
32 #include "llvm/IR/Value.h"
33 #include "llvm/InitializePasses.h"
34 #include "llvm/Pass.h"
37 #include "llvm/Support/Debug.h"
40 
41 using namespace llvm;
42 
43 #define DEBUG_TYPE "memprof"
44 
45 constexpr int LLVM_MEM_PROFILER_VERSION = 1;
46 
47 // Size of memory mapped to a single shadow location.
49 
50 // Scale from granularity down to shadow size.
52 
53 constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
55 // On Emscripten, the system needs more than one priorities for constructors.
57 constexpr char MemProfInitName[] = "__memprof_init";
59  "__memprof_version_mismatch_check_v";
60 
62  "__memprof_shadow_memory_dynamic_address";
63 
64 constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
65 
66 // Command-line flags.
67 
69  "memprof-guard-against-version-mismatch",
70  cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
71  cl::init(true));
72 
73 // This flag may need to be replaced with -f[no-]memprof-reads.
74 static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
75  cl::desc("instrument read instructions"),
76  cl::Hidden, cl::init(true));
77 
78 static cl::opt<bool>
79  ClInstrumentWrites("memprof-instrument-writes",
80  cl::desc("instrument write instructions"), cl::Hidden,
81  cl::init(true));
82 
84  "memprof-instrument-atomics",
85  cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
86  cl::init(true));
87 
89  "memprof-use-callbacks",
90  cl::desc("Use callbacks instead of inline instrumentation sequences."),
91  cl::Hidden, cl::init(false));
92 
94  ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
95  cl::desc("Prefix for memory access callbacks"),
96  cl::Hidden, cl::init("__memprof_"));
97 
98 // These flags allow to change the shadow mapping.
99 // The shadow mapping looks like
100 // Shadow = ((Mem & mask) >> scale) + offset
101 
102 static cl::opt<int> ClMappingScale("memprof-mapping-scale",
103  cl::desc("scale of memprof shadow mapping"),
105 
106 static cl::opt<int>
107  ClMappingGranularity("memprof-mapping-granularity",
108  cl::desc("granularity of memprof shadow mapping"),
110 
111 static cl::opt<bool> ClStack("memprof-instrument-stack",
112  cl::desc("Instrument scalar stack variables"),
113  cl::Hidden, cl::init(false));
114 
115 // Debug flags.
116 
117 static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
118  cl::init(0));
119 
120 static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
121  cl::desc("Debug func"));
122 
123 static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
124  cl::Hidden, cl::init(-1));
125 
126 static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
127  cl::Hidden, cl::init(-1));
128 
129 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
130 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
131 STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
132 STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
133 
134 namespace {
135 
136 /// This struct defines the shadow mapping using the rule:
137 /// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
138 struct ShadowMapping {
139  ShadowMapping() {
140  Scale = ClMappingScale;
141  Granularity = ClMappingGranularity;
142  Mask = ~(Granularity - 1);
143  }
144 
145  int Scale;
146  int Granularity;
147  uint64_t Mask; // Computed as ~(Granularity-1)
148 };
149 
150 static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
153 }
154 
155 struct InterestingMemoryAccess {
156  Value *Addr = nullptr;
157  bool IsWrite;
158  unsigned Alignment;
159  Type *AccessTy;
161  Value *MaybeMask = nullptr;
162 };
163 
164 /// Instrument the code in module to profile memory accesses.
165 class MemProfiler {
166 public:
167  MemProfiler(Module &M) {
168  C = &(M.getContext());
169  LongSize = M.getDataLayout().getPointerSizeInBits();
170  IntptrTy = Type::getIntNTy(*C, LongSize);
171  }
172 
173  /// If it is an interesting memory access, populate information
174  /// about the access and return a InterestingMemoryAccess struct.
175  /// Otherwise return None.
177  isInterestingMemoryAccess(Instruction *I) const;
178 
179  void instrumentMop(Instruction *I, const DataLayout &DL,
180  InterestingMemoryAccess &Access);
181  void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
182  Value *Addr, uint32_t TypeSize, bool IsWrite);
184  Instruction *I, Value *Addr,
185  unsigned Alignment, Type *AccessTy,
186  bool IsWrite);
187  void instrumentMemIntrinsic(MemIntrinsic *MI);
188  Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
189  bool instrumentFunction(Function &F);
190  bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
191  bool insertDynamicShadowAtFunctionEntry(Function &F);
192 
193 private:
194  void initializeCallbacks(Module &M);
195 
196  LLVMContext *C;
197  int LongSize;
198  Type *IntptrTy;
199  ShadowMapping Mapping;
200 
201  // These arrays is indexed by AccessIsWrite
202  FunctionCallee MemProfMemoryAccessCallback[2];
203  FunctionCallee MemProfMemoryAccessCallbackSized[2];
204 
205  FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
206  Value *DynamicShadowOffset = nullptr;
207 };
208 
209 class MemProfilerLegacyPass : public FunctionPass {
210 public:
211  static char ID;
212 
213  explicit MemProfilerLegacyPass() : FunctionPass(ID) {
215  }
216 
217  StringRef getPassName() const override { return "MemProfilerFunctionPass"; }
218 
219  bool runOnFunction(Function &F) override {
220  MemProfiler Profiler(*F.getParent());
221  return Profiler.instrumentFunction(F);
222  }
223 };
224 
225 class ModuleMemProfiler {
226 public:
227  ModuleMemProfiler(Module &M) { TargetTriple = Triple(M.getTargetTriple()); }
228 
229  bool instrumentModule(Module &);
230 
231 private:
232  Triple TargetTriple;
233  ShadowMapping Mapping;
234  Function *MemProfCtorFunction = nullptr;
235 };
236 
237 class ModuleMemProfilerLegacyPass : public ModulePass {
238 public:
239  static char ID;
240 
241  explicit ModuleMemProfilerLegacyPass() : ModulePass(ID) {
243  }
244 
245  StringRef getPassName() const override { return "ModuleMemProfiler"; }
246 
247  void getAnalysisUsage(AnalysisUsage &AU) const override {}
248 
249  bool runOnModule(Module &M) override {
250  ModuleMemProfiler MemProfiler(M);
251  return MemProfiler.instrumentModule(M);
252  }
253 };
254 
255 } // end anonymous namespace
256 
258 
261  Module &M = *F.getParent();
262  MemProfiler Profiler(M);
263  if (Profiler.instrumentFunction(F))
264  return PreservedAnalyses::none();
265  return PreservedAnalyses::all();
266 }
267 
269 
272  ModuleMemProfiler Profiler(M);
273  if (Profiler.instrumentModule(M))
274  return PreservedAnalyses::none();
275  return PreservedAnalyses::all();
276 }
277 
279 
280 INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof",
281  "MemProfiler: profile memory allocations and accesses.",
282  false, false)
283 INITIALIZE_PASS_END(MemProfilerLegacyPass, "memprof",
284  "MemProfiler: profile memory allocations and accesses.",
286 
288  return new MemProfilerLegacyPass();
289 }
290 
292 
293 INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module",
294  "MemProfiler: profile memory allocations and accesses."
295  "ModulePass",
296  false, false)
297 
299  return new ModuleMemProfilerLegacyPass();
300 }
301 
302 Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
303  // (Shadow & mask) >> scale
304  Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
305  Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
306  // (Shadow >> scale) | offset
307  assert(DynamicShadowOffset);
308  return IRB.CreateAdd(Shadow, DynamicShadowOffset);
309 }
310 
311 // Instrument memset/memmove/memcpy
312 void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
313  IRBuilder<> IRB(MI);
314  if (isa<MemTransferInst>(MI)) {
315  IRB.CreateCall(
316  isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
317  {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
318  IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
319  IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
320  } else if (isa<MemSetInst>(MI)) {
321  IRB.CreateCall(
322  MemProfMemset,
323  {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
324  IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
325  IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
326  }
327  MI->eraseFromParent();
328 }
329 
331 MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
332  // Do not instrument the load fetching the dynamic shadow address.
333  if (DynamicShadowOffset == I)
334  return None;
335 
336  InterestingMemoryAccess Access;
337 
338  if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
339  if (!ClInstrumentReads)
340  return None;
341  Access.IsWrite = false;
342  Access.AccessTy = LI->getType();
343  Access.Alignment = LI->getAlignment();
344  Access.Addr = LI->getPointerOperand();
345  } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
346  if (!ClInstrumentWrites)
347  return None;
348  Access.IsWrite = true;
349  Access.AccessTy = SI->getValueOperand()->getType();
350  Access.Alignment = SI->getAlignment();
351  Access.Addr = SI->getPointerOperand();
352  } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
353  if (!ClInstrumentAtomics)
354  return None;
355  Access.IsWrite = true;
356  Access.AccessTy = RMW->getValOperand()->getType();
357  Access.Alignment = 0;
358  Access.Addr = RMW->getPointerOperand();
359  } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
360  if (!ClInstrumentAtomics)
361  return None;
362  Access.IsWrite = true;
363  Access.AccessTy = XCHG->getCompareOperand()->getType();
364  Access.Alignment = 0;
365  Access.Addr = XCHG->getPointerOperand();
366  } else if (auto *CI = dyn_cast<CallInst>(I)) {
367  auto *F = CI->getCalledFunction();
368  if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
369  F->getIntrinsicID() == Intrinsic::masked_store)) {
370  unsigned OpOffset = 0;
371  if (F->getIntrinsicID() == Intrinsic::masked_store) {
372  if (!ClInstrumentWrites)
373  return None;
374  // Masked store has an initial operand for the value.
375  OpOffset = 1;
376  Access.AccessTy = CI->getArgOperand(0)->getType();
377  Access.IsWrite = true;
378  } else {
379  if (!ClInstrumentReads)
380  return None;
381  Access.AccessTy = CI->getType();
382  Access.IsWrite = false;
383  }
384 
385  auto *BasePtr = CI->getOperand(0 + OpOffset);
386  if (auto *AlignmentConstant =
387  dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
388  Access.Alignment = (unsigned)AlignmentConstant->getZExtValue();
389  else
390  Access.Alignment = 1; // No alignment guarantees. We probably got Undef
391  Access.MaybeMask = CI->getOperand(2 + OpOffset);
392  Access.Addr = BasePtr;
393  }
394  }
395 
396  if (!Access.Addr)
397  return None;
398 
399  // Do not instrument acesses from different address spaces; we cannot deal
400  // with them.
401  Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
402  if (PtrTy->getPointerAddressSpace() != 0)
403  return None;
404 
405  // Ignore swifterror addresses.
406  // swifterror memory addresses are mem2reg promoted by instruction
407  // selection. As such they cannot have regular uses like an instrumentation
408  // function and it makes no sense to track them as memory.
409  if (Access.Addr->isSwiftError())
410  return None;
411 
412  // Peel off GEPs and BitCasts.
413  auto *Addr = Access.Addr->stripInBoundsOffsets();
414 
415  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
416  // Do not instrument PGO counter updates.
417  if (GV->hasSection()) {
418  StringRef SectionName = GV->getSection();
419  // Check if the global is in the PGO counters section.
420  auto OF = Triple(I->getModule()->getTargetTriple()).getObjectFormat();
421  if (SectionName.endswith(
422  getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
423  return None;
424  }
425 
426  // Do not instrument accesses to LLVM internal variables.
427  if (GV->getName().startswith("__llvm"))
428  return None;
429  }
430 
431  const DataLayout &DL = I->getModule()->getDataLayout();
432  Access.TypeSize = DL.getTypeStoreSizeInBits(Access.AccessTy);
433  return Access;
434 }
435 
437  Instruction *I, Value *Addr,
438  unsigned Alignment,
439  Type *AccessTy, bool IsWrite) {
440  auto *VTy = cast<FixedVectorType>(AccessTy);
441  uint64_t ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType());
442  unsigned Num = VTy->getNumElements();
443  auto *Zero = ConstantInt::get(IntptrTy, 0);
444  for (unsigned Idx = 0; Idx < Num; ++Idx) {
445  Value *InstrumentedAddress = nullptr;
446  Instruction *InsertBefore = I;
447  if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
448  // dyn_cast as we might get UndefValue
449  if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
450  if (Masked->isZero())
451  // Mask is constant false, so no instrumentation needed.
452  continue;
453  // If we have a true or undef value, fall through to instrumentAddress.
454  // with InsertBefore == I
455  }
456  } else {
457  IRBuilder<> IRB(I);
458  Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
459  Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
460  InsertBefore = ThenTerm;
461  }
462 
463  IRBuilder<> IRB(InsertBefore);
464  InstrumentedAddress =
465  IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
466  instrumentAddress(I, InsertBefore, InstrumentedAddress, ElemTypeSize,
467  IsWrite);
468  }
469 }
470 
471 void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
472  InterestingMemoryAccess &Access) {
473  // Skip instrumentation of stack accesses unless requested.
474  if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {
475  if (Access.IsWrite)
476  ++NumSkippedStackWrites;
477  else
478  ++NumSkippedStackReads;
479  return;
480  }
481 
482  if (Access.IsWrite)
483  NumInstrumentedWrites++;
484  else
485  NumInstrumentedReads++;
486 
487  if (Access.MaybeMask) {
488  instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
489  Access.Alignment, Access.AccessTy,
490  Access.IsWrite);
491  } else {
492  // Since the access counts will be accumulated across the entire allocation,
493  // we only update the shadow access count for the first location and thus
494  // don't need to worry about alignment and type size.
495  instrumentAddress(I, I, Access.Addr, Access.TypeSize, Access.IsWrite);
496  }
497 }
498 
499 void MemProfiler::instrumentAddress(Instruction *OrigIns,
500  Instruction *InsertBefore, Value *Addr,
501  uint32_t TypeSize, bool IsWrite) {
502  IRBuilder<> IRB(InsertBefore);
503  Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
504 
505  if (ClUseCalls) {
506  IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
507  return;
508  }
509 
510  // Create an inline sequence to compute shadow location, and increment the
511  // value by one.
512  Type *ShadowTy = Type::getInt64Ty(*C);
513  Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
514  Value *ShadowPtr = memToShadow(AddrLong, IRB);
515  Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
516  Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
518  ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
519  IRB.CreateStore(ShadowValue, ShadowAddr);
520 }
521 
522 // Create the variable for the profile file name.
524  const MDString *MemProfFilename =
525  dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
526  if (!MemProfFilename)
527  return;
528  assert(!MemProfFilename->getString().empty() &&
529  "Unexpected MemProfProfileFilename metadata with empty string");
530  Constant *ProfileNameConst = ConstantDataArray::getString(
531  M.getContext(), MemProfFilename->getString(), true);
532  GlobalVariable *ProfileNameVar = new GlobalVariable(
533  M, ProfileNameConst->getType(), /*isConstant=*/true,
535  Triple TT(M.getTargetTriple());
536  if (TT.supportsCOMDAT()) {
537  ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
538  ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
539  }
540 }
541 
542 bool ModuleMemProfiler::instrumentModule(Module &M) {
543  // Create a module constructor.
544  std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
545  std::string VersionCheckName =
547  : "";
548  std::tie(MemProfCtorFunction, std::ignore) =
550  MemProfInitName, /*InitArgTypes=*/{},
551  /*InitArgs=*/{}, VersionCheckName);
552 
553  const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
554  appendToGlobalCtors(M, MemProfCtorFunction, Priority);
555 
557 
558  return true;
559 }
560 
561 void MemProfiler::initializeCallbacks(Module &M) {
562  IRBuilder<> IRB(*C);
563 
564  for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
565  const std::string TypeStr = AccessIsWrite ? "store" : "load";
566 
567  SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
568  SmallVector<Type *, 2> Args1{1, IntptrTy};
569  MemProfMemoryAccessCallbackSized[AccessIsWrite] =
570  M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr + "N",
571  FunctionType::get(IRB.getVoidTy(), Args2, false));
572 
573  MemProfMemoryAccessCallback[AccessIsWrite] =
574  M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr,
575  FunctionType::get(IRB.getVoidTy(), Args1, false));
576  }
577  MemProfMemmove = M.getOrInsertFunction(
578  ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
579  IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
580  MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
581  IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
582  IRB.getInt8PtrTy(), IntptrTy);
583  MemProfMemset = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset",
584  IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
585  IRB.getInt32Ty(), IntptrTy);
586 }
587 
588 bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
589  // For each NSObject descendant having a +load method, this method is invoked
590  // by the ObjC runtime before any of the static constructors is called.
591  // Therefore we need to instrument such methods with a call to __memprof_init
592  // at the beginning in order to initialize our runtime before any access to
593  // the shadow memory.
594  // We cannot just ignore these methods, because they may call other
595  // instrumented functions.
596  if (F.getName().find(" load]") != std::string::npos) {
597  FunctionCallee MemProfInitFunction =
599  IRBuilder<> IRB(&F.front(), F.front().begin());
600  IRB.CreateCall(MemProfInitFunction, {});
601  return true;
602  }
603  return false;
604 }
605 
606 bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
607  IRBuilder<> IRB(&F.front().front());
608  Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
610  if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
611  cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
612  DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
613  return true;
614 }
615 
616 bool MemProfiler::instrumentFunction(Function &F) {
617  if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
618  return false;
619  if (ClDebugFunc == F.getName())
620  return false;
621  if (F.getName().startswith("__memprof_"))
622  return false;
623 
624  bool FunctionModified = false;
625 
626  // If needed, insert __memprof_init.
627  // This function needs to be called even if the function body is not
628  // instrumented.
629  if (maybeInsertMemProfInitAtFunctionEntry(F))
630  FunctionModified = true;
631 
632  LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
633 
634  initializeCallbacks(*F.getParent());
635 
636  SmallVector<Instruction *, 16> ToInstrument;
637 
638  // Fill the set of memory operations to instrument.
639  for (auto &BB : F) {
640  for (auto &Inst : BB) {
641  if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
642  ToInstrument.push_back(&Inst);
643  }
644  }
645 
646  if (ToInstrument.empty()) {
647  LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
648  << " " << F << "\n");
649 
650  return FunctionModified;
651  }
652 
653  FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
654 
655  int NumInstrumented = 0;
656  for (auto *Inst : ToInstrument) {
657  if (ClDebugMin < 0 || ClDebugMax < 0 ||
658  (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
660  isInterestingMemoryAccess(Inst);
661  if (Access)
662  instrumentMop(Inst, F.getParent()->getDataLayout(), *Access);
663  else
664  instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
665  }
666  NumInstrumented++;
667  }
668 
669  if (NumInstrumented > 0)
670  FunctionModified = true;
671 
672  LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
673  << F << "\n");
674 
675  return FunctionModified;
676 }
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
llvm::IRBuilderBase::CreateIntCast
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2021
llvm::initializeMemProfilerLegacyPassPass
void initializeMemProfilerLegacyPassPass(PassRegistry &)
llvm::IRBuilderBase::getInt32Ty
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:501
llvm::IRBuilderBase::CreateStore
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1662
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:104
llvm::GlobalObject::setComdat
void setComdat(Comdat *C)
Definition: Globals.cpp:190
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
ClDebugFunc
static cl::opt< std::string > ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func"))
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
ClInstrumentWrites
static cl::opt< bool > ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
llvm::ModulePass
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:248
IntrinsicInst.h
llvm::Function
Definition: Function.h:60
StringRef.h
Pass.h
llvm::PointerType::get
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Definition: Type.cpp:727
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
Statistic.h
llvm::Type::getPointerAddressSpace
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:729
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:361
ValueTracking.h
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:139
DefaultShadowGranularity
constexpr uint64_t DefaultShadowGranularity
Definition: MemProfiler.cpp:48
llvm::PreservedAnalyses::none
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Module.h
DefaultShadowScale
constexpr uint64_t DefaultShadowScale
Definition: MemProfiler.cpp:51
llvm::MemIntrinsic
This is the common base class for memset/memcpy/memmove.
Definition: IntrinsicInst.h:962
LLVM_MEM_PROFILER_VERSION
constexpr int LLVM_MEM_PROFILER_VERSION
Definition: MemProfiler.cpp:45
llvm::Optional
Definition: APInt.h:33
Vector
So we should use XX3Form_Rcr to implement intrinsic Convert DP outs ins xscvdpsp No builtin are required Round &Convert QP DP(dword[1] is set to zero) No builtin are required Round to Quad Precision because you need to assign rounding mode in instruction Provide builtin(set f128:$vT,(int_ppc_vsx_xsrqpi f128:$vB))(set f128 yields< n x< ty > >< result > yields< ty >< result > No builtin are required Load Store Vector
Definition: README_P9.txt:497
ClDebugMax
static cl::opt< int > ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
llvm::createProfileFileNameVar
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1182
and
We currently generate a but we really shouldn eax ecx xorl edx divl ecx eax divl ecx movl eax ret A similar code sequence works for division We currently compile i32 v2 eax eax jo LBB1_2 and
Definition: README.txt:1271
llvm::Triple::isOSEmscripten
bool isOSEmscripten() const
Tests whether the OS is Emscripten.
Definition: Triple.h:631
llvm::IRBuilderBase::CreateIntToPtr
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1947
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::createMemProfilerFunctionPass
FunctionPass * createMemProfilerFunctionPass()
Definition: MemProfiler.cpp:287
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
Instruction.h
CommandLine.h
GlobalValue.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::ModuleMemProfilerPass::ModuleMemProfilerPass
ModuleMemProfilerPass()
llvm::IRBuilderBase::CreateGEP
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1725
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::IRBuilderBase::CreateLoad
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1649
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
MemProfInitName
constexpr char MemProfInitName[]
Definition: MemProfiler.cpp:57
false
Definition: StackSlotColoring.cpp:141
llvm::MemProfilerPass::MemProfilerPass
MemProfilerPass()
llvm::Instruction
Definition: Instruction.h:42
InstrProf.h
llvm::Triple::getObjectFormat
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
Definition: Triple.h:363
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::codeview::EncodedFramePtrReg::BasePtr
@ BasePtr
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:919
llvm::getUnderlyingObject
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
Definition: ValueTracking.cpp:4343
ClInsertVersionCheck
static cl::opt< bool > ClInsertVersionCheck("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
ClDebug
static cl::opt< int > ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0))
ClDebugMin
static cl::opt< int > ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
llvm::None
const NoneType None
Definition: None.h:24
instrumentMaskedLoadOrStore
static void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, const DataLayout &DL, Type *IntptrTy, Value *Mask, Instruction *I, Value *Addr, MaybeAlign Alignment, unsigned Granularity, Type *OpType, bool IsWrite, Value *SizeArgument, bool UseCalls, uint32_t Exp)
Definition: AddressSanitizer.cpp:1463
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
Type.h
llvm::IRBuilderBase::CreateAnd
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1350
llvm::IRBuilderBase::CreatePointerCast
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1998
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof", "MemProfiler: profile memory allocations and accesses.", false, false) INITIALIZE_PASS_END(MemProfilerLegacyPass
llvm::SPIRV::Decoration::Alignment
@ Alignment
llvm::cl::opt< bool >
MemProfModuleCtorName
constexpr char MemProfModuleCtorName[]
Definition: MemProfiler.cpp:53
MemProfVersionCheckNamePrefix
constexpr char MemProfVersionCheckNamePrefix[]
Definition: MemProfiler.cpp:58
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:305
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::StringRef::empty
constexpr LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:153
llvm::createSanitizerCtorAndInitFunctions
std::pair< Function *, FunctionCallee > createSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, StringRef VersionCheckName=StringRef())
Creates sanitizer constructor function, and calls sanitizer's init function from it.
Definition: ModuleUtils.cpp:135
uint64_t
ClStack
static cl::opt< bool > ClStack("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false))
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:78
llvm::GlobalValue::WeakAnyLinkage
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:52
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
llvm::declareSanitizerInitFunction
FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName, ArrayRef< Type * > InitArgTypes)
Definition: ModuleUtils.cpp:114
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::PICLevel::NotPIC
@ NotPIC
Definition: CodeGen.h:33
llvm::IRBuilderBase::getInt8PtrTy
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
Definition: IRBuilder.h:549
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
ClUseCalls
static cl::opt< bool > ClUseCalls("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false))
MemProfEmscriptenCtorAndDtorPriority
constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority
Definition: MemProfiler.cpp:56
llvm::GlobalValue::setLinkage
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:459
llvm::IRBuilderBase::CreateAdd
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1201
ClInstrumentReads
static cl::opt< bool > ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::IRBuilderBase::CreateExtractElement
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2276
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Triple.h
DataLayout.h
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
uint32_t
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::ConstantDataArray::getString
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:3091
llvm::GlobalValue::AvailableExternallyLinkage
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:49
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:176
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:727
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
Constant.h
profile
sample profile
Definition: SampleProfile.cpp:1822
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:240
llvm::createModuleMemProfilerLegacyPassPass
ModulePass * createModuleMemProfilerLegacyPassPass()
llvm::Type::getIntNTy
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition: Type.cpp:243
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
llvm::initializeModuleMemProfilerLegacyPassPass
void initializeModuleMemProfilerLegacyPassPass(PassRegistry &)
llvm::SectionName
Definition: DWARFSection.h:21
MemProfFilenameVar
constexpr char MemProfFilenameVar[]
Definition: MemProfiler.cpp:64
llvm::TypeSize
Definition: TypeSize.h:421
llvm::MemProfilerPass::run
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: MemProfiler.cpp:259
Function.h
llvm::ModuleMemProfilerPass::run
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition: MemProfiler.cpp:270
INITIALIZE_PASS
INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module", "MemProfiler: profile memory allocations and accesses." "ModulePass", false, false) ModulePass *llvm
Definition: MemProfiler.cpp:293
ClMappingGranularity
static cl::opt< int > ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowGranularity))
MemProfCtorAndDtorPriority
constexpr uint64_t MemProfCtorAndDtorPriority
Definition: MemProfiler.cpp:54
MemProfShadowMemoryDynamicAddress
constexpr char MemProfShadowMemoryDynamicAddress[]
Definition: MemProfiler.cpp:61
ClMappingScale
static cl::opt< int > ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale))
llvm::getInstrProfSectionName
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:214
llvm::GlobalValue::ExternalLinkage
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
memprof
memprof
Definition: MemProfiler.cpp:283
SmallVector.h
llvm::IRBuilderBase::CreateLShr
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1310
ModuleUtils.h
ClInstrumentAtomics
static cl::opt< bool > ClInstrumentAtomics("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
llvm::to_string
std::string to_string(const T &Value)
Definition: ScopedPrinter.h:86
MemProfiler.h
llvm::FunctionCallee
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
ClMemoryAccessCallbackPrefix
static cl::opt< std::string > ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_"))
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::IRBuilderBase::getVoidTy
Type * getVoidTy()
Fetch the type representing void.
Definition: IRBuilder.h:539
llvm::SplitBlockAndInsertIfThen
Instruction * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights, DominatorTree *DT, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
Definition: BasicBlockUtils.cpp:1442
llvm::appendToGlobalCtors
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:65
llvm::cl::desc
Definition: CommandLine.h:405
llvm::MDString::getString
StringRef getString() const
Definition: Metadata.cpp:509
BasicBlockUtils.h
llvm::MDString
A single uniqued string.
Definition: Metadata.h:612
Value.h
InitializePasses.h
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::AtomicCmpXchgInst
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:522
Debug.h
llvm::IRBuilderBase::CreateCall
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2229
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37