LLVM  15.0.0git
MemorySanitizer.cpp
Go to the documentation of this file.
1 //===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file is a part of MemorySanitizer, a detector of uninitialized
11 /// reads.
12 ///
13 /// The algorithm of the tool is similar to Memcheck
14 /// (http://goo.gl/QKbem). We associate a few shadow bits with every
15 /// byte of the application memory, poison the shadow of the malloc-ed
16 /// or alloca-ed memory, load the shadow bits on every memory read,
17 /// propagate the shadow bits through some of the arithmetic
18 /// instruction (including MOV), store the shadow bits on every memory
19 /// write, report a bug on some other instructions (e.g. JMP) if the
20 /// associated shadow is poisoned.
21 ///
22 /// But there are differences too. The first and the major one:
23 /// compiler instrumentation instead of binary instrumentation. This
24 /// gives us much better register allocation, possible compiler
25 /// optimizations and a fast start-up. But this brings the major issue
26 /// as well: msan needs to see all program events, including system
27 /// calls and reads/writes in system libraries, so we either need to
28 /// compile *everything* with msan or use a binary translation
29 /// component (e.g. DynamoRIO) to instrument pre-built libraries.
30 /// Another difference from Memcheck is that we use 8 shadow bits per
31 /// byte of application memory and use a direct shadow mapping. This
32 /// greatly simplifies the instrumentation code and avoids races on
33 /// shadow updates (Memcheck is single-threaded so races are not a
34 /// concern there. Memcheck uses 2 shadow bits per byte with a slow
35 /// path storage that uses 8 bits per byte).
36 ///
37 /// The default value of shadow is 0, which means "clean" (not poisoned).
38 ///
39 /// Every module initializer should call __msan_init to ensure that the
40 /// shadow memory is ready. On error, __msan_warning is called. Since
41 /// parameters and return values may be passed via registers, we have a
42 /// specialized thread-local shadow for return values
43 /// (__msan_retval_tls) and parameters (__msan_param_tls).
44 ///
45 /// Origin tracking.
46 ///
47 /// MemorySanitizer can track origins (allocation points) of all uninitialized
48 /// values. This behavior is controlled with a flag (msan-track-origins) and is
49 /// disabled by default.
50 ///
51 /// Origins are 4-byte values created and interpreted by the runtime library.
52 /// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53 /// of application memory. Propagation of origins is basically a bunch of
54 /// "select" instructions that pick the origin of a dirty argument, if an
55 /// instruction has one.
56 ///
57 /// Every 4 aligned, consecutive bytes of application memory have one origin
58 /// value associated with them. If these bytes contain uninitialized data
59 /// coming from 2 different allocations, the last store wins. Because of this,
60 /// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61 /// practice.
62 ///
63 /// Origins are meaningless for fully initialized values, so MemorySanitizer
64 /// avoids storing origin to memory when a fully initialized value is stored.
65 /// This way it avoids needless overwriting origin of the 4-byte region on
66 /// a short (i.e. 1 byte) clean store, and it is also good for performance.
67 ///
68 /// Atomic handling.
69 ///
70 /// Ideally, every atomic store of application value should update the
71 /// corresponding shadow location in an atomic way. Unfortunately, atomic store
72 /// of two disjoint locations can not be done without severe slowdown.
73 ///
74 /// Therefore, we implement an approximation that may err on the safe side.
75 /// In this implementation, every atomically accessed location in the program
76 /// may only change from (partially) uninitialized to fully initialized, but
77 /// not the other way around. We load the shadow _after_ the application load,
78 /// and we store the shadow _before_ the app store. Also, we always store clean
79 /// shadow (if the application store is atomic). This way, if the store-load
80 /// pair constitutes a happens-before arc, shadow store and load are correctly
81 /// ordered such that the load will get either the value that was stored, or
82 /// some later value (which is always clean).
83 ///
84 /// This does not work very well with Compare-And-Swap (CAS) and
85 /// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86 /// must store the new shadow before the app operation, and load the shadow
87 /// after the app operation. Computers don't work this way. Current
88 /// implementation ignores the load aspect of CAS/RMW, always returning a clean
89 /// value. It implements the store part as a simple atomic store by storing a
90 /// clean shadow.
91 ///
92 /// Instrumenting inline assembly.
93 ///
94 /// For inline assembly code LLVM has little idea about which memory locations
95 /// become initialized depending on the arguments. It can be possible to figure
96 /// out which arguments are meant to point to inputs and outputs, but the
97 /// actual semantics can be only visible at runtime. In the Linux kernel it's
98 /// also possible that the arguments only indicate the offset for a base taken
99 /// from a segment register, so it's dangerous to treat any asm() arguments as
100 /// pointers. We take a conservative approach generating calls to
101 /// __msan_instrument_asm_store(ptr, size)
102 /// , which defer the memory unpoisoning to the runtime library.
103 /// The latter can perform more complex address checks to figure out whether
104 /// it's safe to touch the shadow memory.
105 /// Like with atomic operations, we call __msan_instrument_asm_store() before
106 /// the assembly call, so that changes to the shadow memory will be seen by
107 /// other threads together with main memory initialization.
108 ///
109 /// KernelMemorySanitizer (KMSAN) implementation.
110 ///
111 /// The major differences between KMSAN and MSan instrumentation are:
112 /// - KMSAN always tracks the origins and implies msan-keep-going=true;
113 /// - KMSAN allocates shadow and origin memory for each page separately, so
114 /// there are no explicit accesses to shadow and origin in the
115 /// instrumentation.
116 /// Shadow and origin values for a particular X-byte memory location
117 /// (X=1,2,4,8) are accessed through pointers obtained via the
118 /// __msan_metadata_ptr_for_load_X(ptr)
119 /// __msan_metadata_ptr_for_store_X(ptr)
120 /// functions. The corresponding functions check that the X-byte accesses
121 /// are possible and returns the pointers to shadow and origin memory.
122 /// Arbitrary sized accesses are handled with:
123 /// __msan_metadata_ptr_for_load_n(ptr, size)
124 /// __msan_metadata_ptr_for_store_n(ptr, size);
125 /// - TLS variables are stored in a single per-task struct. A call to a
126 /// function __msan_get_context_state() returning a pointer to that struct
127 /// is inserted into every instrumented function before the entry block;
128 /// - __msan_warning() takes a 32-bit origin parameter;
129 /// - local variables are poisoned with __msan_poison_alloca() upon function
130 /// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
131 /// function;
132 /// - the pass doesn't declare any global variables or add global constructors
133 /// to the translation unit.
134 ///
135 /// Also, KMSAN currently ignores uninitialized memory passed into inline asm
136 /// calls, making sure we're on the safe side wrt. possible false positives.
137 ///
138 /// KernelMemorySanitizer only supports X86_64 at the moment.
139 ///
140 //
141 // FIXME: This sanitizer does not yet handle scalable vectors
142 //
143 //===----------------------------------------------------------------------===//
144 
146 #include "llvm/ADT/APInt.h"
147 #include "llvm/ADT/ArrayRef.h"
149 #include "llvm/ADT/SmallSet.h"
150 #include "llvm/ADT/SmallString.h"
151 #include "llvm/ADT/SmallVector.h"
152 #include "llvm/ADT/StringExtras.h"
153 #include "llvm/ADT/StringRef.h"
154 #include "llvm/ADT/Triple.h"
157 #include "llvm/IR/Argument.h"
158 #include "llvm/IR/Attributes.h"
159 #include "llvm/IR/BasicBlock.h"
160 #include "llvm/IR/CallingConv.h"
161 #include "llvm/IR/Constant.h"
162 #include "llvm/IR/Constants.h"
163 #include "llvm/IR/DataLayout.h"
164 #include "llvm/IR/DerivedTypes.h"
165 #include "llvm/IR/Function.h"
166 #include "llvm/IR/GlobalValue.h"
167 #include "llvm/IR/GlobalVariable.h"
168 #include "llvm/IR/IRBuilder.h"
169 #include "llvm/IR/InlineAsm.h"
170 #include "llvm/IR/InstVisitor.h"
171 #include "llvm/IR/InstrTypes.h"
172 #include "llvm/IR/Instruction.h"
173 #include "llvm/IR/Instructions.h"
174 #include "llvm/IR/IntrinsicInst.h"
175 #include "llvm/IR/Intrinsics.h"
176 #include "llvm/IR/IntrinsicsX86.h"
177 #include "llvm/IR/MDBuilder.h"
178 #include "llvm/IR/Module.h"
179 #include "llvm/IR/Type.h"
180 #include "llvm/IR/Value.h"
181 #include "llvm/IR/ValueMap.h"
182 #include "llvm/Support/Alignment.h"
184 #include "llvm/Support/Casting.h"
186 #include "llvm/Support/Debug.h"
188 #include "llvm/Support/MathExtras.h"
193 #include <algorithm>
194 #include <cassert>
195 #include <cstddef>
196 #include <cstdint>
197 #include <memory>
198 #include <string>
199 #include <tuple>
200 
201 using namespace llvm;
202 
203 #define DEBUG_TYPE "msan"
204 
205 static const unsigned kOriginSize = 4;
206 static const Align kMinOriginAlignment = Align(4);
207 static const Align kShadowTLSAlignment = Align(8);
208 
209 // These constants must be kept in sync with the ones in msan.h.
210 static const unsigned kParamTLSSize = 800;
211 static const unsigned kRetvalTLSSize = 800;
212 
213 // Accesses sizes are powers of two: 1, 2, 4, 8.
214 static const size_t kNumberOfAccessSizes = 4;
215 
216 /// Track origins of uninitialized values.
217 ///
218 /// Adds a section to MemorySanitizer report that points to the allocation
219 /// (stack or heap) the uninitialized bits came from originally.
220 static cl::opt<int> ClTrackOrigins("msan-track-origins",
221  cl::desc("Track origins (allocation sites) of poisoned memory"),
222  cl::Hidden, cl::init(0));
223 
224 static cl::opt<bool> ClKeepGoing("msan-keep-going",
225  cl::desc("keep going after reporting a UMR"),
226  cl::Hidden, cl::init(false));
227 
228 static cl::opt<bool> ClPoisonStack("msan-poison-stack",
229  cl::desc("poison uninitialized stack variables"),
230  cl::Hidden, cl::init(true));
231 
232 static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call",
233  cl::desc("poison uninitialized stack variables with a call"),
234  cl::Hidden, cl::init(false));
235 
236 static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern",
237  cl::desc("poison uninitialized stack variables with the given pattern"),
238  cl::Hidden, cl::init(0xff));
239 
240 static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
241  cl::desc("poison undef temps"),
242  cl::Hidden, cl::init(true));
243 
244 static cl::opt<bool> ClHandleICmp("msan-handle-icmp",
245  cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
246  cl::Hidden, cl::init(true));
247 
248 static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
249  cl::desc("exact handling of relational integer ICmp"),
250  cl::Hidden, cl::init(false));
251 
253  "msan-handle-lifetime-intrinsics",
254  cl::desc(
255  "when possible, poison scoped variables at the beginning of the scope "
256  "(slower, but more precise)"),
257  cl::Hidden, cl::init(true));
258 
259 // When compiling the Linux kernel, we sometimes see false positives related to
260 // MSan being unable to understand that inline assembly calls may initialize
261 // local variables.
262 // This flag makes the compiler conservatively unpoison every memory location
263 // passed into an assembly call. Note that this may cause false positives.
264 // Because it's impossible to figure out the array sizes, we can only unpoison
265 // the first sizeof(type) bytes for each type* pointer.
266 // The instrumentation is only enabled in KMSAN builds, and only if
267 // -msan-handle-asm-conservative is on. This is done because we may want to
268 // quickly disable assembly instrumentation when it breaks.
270  "msan-handle-asm-conservative",
271  cl::desc("conservative handling of inline assembly"), cl::Hidden,
272  cl::init(true));
273 
274 // This flag controls whether we check the shadow of the address
275 // operand of load or store. Such bugs are very rare, since load from
276 // a garbage address typically results in SEGV, but still happen
277 // (e.g. only lower bits of address are garbage, or the access happens
278 // early at program startup where malloc-ed memory is more likely to
279 // be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
280 static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address",
281  cl::desc("report accesses through a pointer which has poisoned shadow"),
282  cl::Hidden, cl::init(true));
283 
285  "msan-eager-checks",
286  cl::desc("check arguments and return values at function call boundaries"),
287  cl::Hidden, cl::init(false));
288 
289 static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions",
290  cl::desc("print out instructions with default strict semantics"),
291  cl::Hidden, cl::init(false));
292 
294  "msan-instrumentation-with-call-threshold",
295  cl::desc(
296  "If the function being instrumented requires more than "
297  "this number of checks and origin stores, use callbacks instead of "
298  "inline checks (-1 means never use callbacks)."),
299  cl::Hidden, cl::init(3500));
300 
301 static cl::opt<bool>
302  ClEnableKmsan("msan-kernel",
303  cl::desc("Enable KernelMemorySanitizer instrumentation"),
304  cl::Hidden, cl::init(false));
305 
306 static cl::opt<bool>
307  ClDisableChecks("msan-disable-checks",
308  cl::desc("Apply no_sanitize to the whole file"), cl::Hidden,
309  cl::init(false));
310 
311 // This is an experiment to enable handling of cases where shadow is a non-zero
312 // compile-time constant. For some unexplainable reason they were silently
313 // ignored in the instrumentation.
314 static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
315  cl::desc("Insert checks for constant shadow values"),
316  cl::Hidden, cl::init(false));
317 
318 // This is off by default because of a bug in gold:
319 // https://sourceware.org/bugzilla/show_bug.cgi?id=19002
320 static cl::opt<bool> ClWithComdat("msan-with-comdat",
321  cl::desc("Place MSan constructors in comdat sections"),
322  cl::Hidden, cl::init(false));
323 
324 // These options allow to specify custom memory map parameters
325 // See MemoryMapParams for details.
326 static cl::opt<uint64_t> ClAndMask("msan-and-mask",
327  cl::desc("Define custom MSan AndMask"),
328  cl::Hidden, cl::init(0));
329 
330 static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
331  cl::desc("Define custom MSan XorMask"),
332  cl::Hidden, cl::init(0));
333 
334 static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
335  cl::desc("Define custom MSan ShadowBase"),
336  cl::Hidden, cl::init(0));
337 
338 static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
339  cl::desc("Define custom MSan OriginBase"),
340  cl::Hidden, cl::init(0));
341 
342 const char kMsanModuleCtorName[] = "msan.module_ctor";
343 const char kMsanInitName[] = "__msan_init";
344 
345 namespace {
346 
347 // Memory map parameters used in application-to-shadow address calculation.
348 // Offset = (Addr & ~AndMask) ^ XorMask
349 // Shadow = ShadowBase + Offset
350 // Origin = OriginBase + Offset
351 struct MemoryMapParams {
352  uint64_t AndMask;
353  uint64_t XorMask;
354  uint64_t ShadowBase;
355  uint64_t OriginBase;
356 };
357 
358 struct PlatformMemoryMapParams {
359  const MemoryMapParams *bits32;
360  const MemoryMapParams *bits64;
361 };
362 
363 } // end anonymous namespace
364 
365 // i386 Linux
366 static const MemoryMapParams Linux_I386_MemoryMapParams = {
367  0x000080000000, // AndMask
368  0, // XorMask (not used)
369  0, // ShadowBase (not used)
370  0x000040000000, // OriginBase
371 };
372 
373 // x86_64 Linux
374 static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
375 #ifdef MSAN_LINUX_X86_64_OLD_MAPPING
376  0x400000000000, // AndMask
377  0, // XorMask (not used)
378  0, // ShadowBase (not used)
379  0x200000000000, // OriginBase
380 #else
381  0, // AndMask (not used)
382  0x500000000000, // XorMask
383  0, // ShadowBase (not used)
384  0x100000000000, // OriginBase
385 #endif
386 };
387 
388 // mips64 Linux
389 static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
390  0, // AndMask (not used)
391  0x008000000000, // XorMask
392  0, // ShadowBase (not used)
393  0x002000000000, // OriginBase
394 };
395 
396 // ppc64 Linux
397 static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
398  0xE00000000000, // AndMask
399  0x100000000000, // XorMask
400  0x080000000000, // ShadowBase
401  0x1C0000000000, // OriginBase
402 };
403 
404 // s390x Linux
405 static const MemoryMapParams Linux_S390X_MemoryMapParams = {
406  0xC00000000000, // AndMask
407  0, // XorMask (not used)
408  0x080000000000, // ShadowBase
409  0x1C0000000000, // OriginBase
410 };
411 
412 // aarch64 Linux
413 static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
414  0, // AndMask (not used)
415  0x06000000000, // XorMask
416  0, // ShadowBase (not used)
417  0x01000000000, // OriginBase
418 };
419 
420 // i386 FreeBSD
421 static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
422  0x000180000000, // AndMask
423  0x000040000000, // XorMask
424  0x000020000000, // ShadowBase
425  0x000700000000, // OriginBase
426 };
427 
428 // x86_64 FreeBSD
429 static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
430  0xc00000000000, // AndMask
431  0x200000000000, // XorMask
432  0x100000000000, // ShadowBase
433  0x380000000000, // OriginBase
434 };
435 
436 // x86_64 NetBSD
437 static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
438  0, // AndMask
439  0x500000000000, // XorMask
440  0, // ShadowBase
441  0x100000000000, // OriginBase
442 };
443 
444 static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
447 };
448 
449 static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
450  nullptr,
452 };
453 
454 static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
455  nullptr,
457 };
458 
459 static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = {
460  nullptr,
462 };
463 
464 static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
465  nullptr,
467 };
468 
469 static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
472 };
473 
474 static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
475  nullptr,
477 };
478 
479 namespace {
480 
481 /// Instrument functions of a module to detect uninitialized reads.
482 ///
483 /// Instantiating MemorySanitizer inserts the msan runtime library API function
484 /// declarations into the module if they don't exist already. Instantiating
485 /// ensures the __msan_init function is in the list of global constructors for
486 /// the module.
487 class MemorySanitizer {
488 public:
489  MemorySanitizer(Module &M, MemorySanitizerOptions Options)
490  : CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins),
491  Recover(Options.Recover), EagerChecks(Options.EagerChecks) {
492  initializeModule(M);
493  }
494 
495  // MSan cannot be moved or copied because of MapParams.
496  MemorySanitizer(MemorySanitizer &&) = delete;
497  MemorySanitizer &operator=(MemorySanitizer &&) = delete;
498  MemorySanitizer(const MemorySanitizer &) = delete;
499  MemorySanitizer &operator=(const MemorySanitizer &) = delete;
500 
501  bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
502 
503 private:
504  friend struct MemorySanitizerVisitor;
505  friend struct VarArgAMD64Helper;
506  friend struct VarArgMIPS64Helper;
507  friend struct VarArgAArch64Helper;
508  friend struct VarArgPowerPC64Helper;
509  friend struct VarArgSystemZHelper;
510 
511  void initializeModule(Module &M);
512  void initializeCallbacks(Module &M);
513  void createKernelApi(Module &M);
514  void createUserspaceApi(Module &M);
515 
516  /// True if we're compiling the Linux kernel.
517  bool CompileKernel;
518  /// Track origins (allocation points) of uninitialized values.
519  int TrackOrigins;
520  bool Recover;
521  bool EagerChecks;
522 
523  LLVMContext *C;
524  Type *IntptrTy;
525  Type *OriginTy;
526 
527  // XxxTLS variables represent the per-thread state in MSan and per-task state
528  // in KMSAN.
529  // For the userspace these point to thread-local globals. In the kernel land
530  // they point to the members of a per-task struct obtained via a call to
531  // __msan_get_context_state().
532 
533  /// Thread-local shadow storage for function parameters.
534  Value *ParamTLS;
535 
536  /// Thread-local origin storage for function parameters.
537  Value *ParamOriginTLS;
538 
539  /// Thread-local shadow storage for function return value.
540  Value *RetvalTLS;
541 
542  /// Thread-local origin storage for function return value.
543  Value *RetvalOriginTLS;
544 
545  /// Thread-local shadow storage for in-register va_arg function
546  /// parameters (x86_64-specific).
547  Value *VAArgTLS;
548 
549  /// Thread-local shadow storage for in-register va_arg function
550  /// parameters (x86_64-specific).
551  Value *VAArgOriginTLS;
552 
553  /// Thread-local shadow storage for va_arg overflow area
554  /// (x86_64-specific).
555  Value *VAArgOverflowSizeTLS;
556 
557  /// Are the instrumentation callbacks set up?
558  bool CallbacksInitialized = false;
559 
560  /// The run-time callback to print a warning.
561  FunctionCallee WarningFn;
562 
563  // These arrays are indexed by log2(AccessSize).
564  FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
565  FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
566 
567  /// Run-time helper that generates a new origin value for a stack
568  /// allocation.
569  FunctionCallee MsanSetAllocaOrigin4Fn;
570 
571  /// Run-time helper that poisons stack on function entry.
572  FunctionCallee MsanPoisonStackFn;
573 
574  /// Run-time helper that records a store (or any event) of an
575  /// uninitialized value and returns an updated origin id encoding this info.
576  FunctionCallee MsanChainOriginFn;
577 
578  /// Run-time helper that paints an origin over a region.
579  FunctionCallee MsanSetOriginFn;
580 
581  /// MSan runtime replacements for memmove, memcpy and memset.
582  FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
583 
584  /// KMSAN callback for task-local function argument shadow.
585  StructType *MsanContextStateTy;
586  FunctionCallee MsanGetContextStateFn;
587 
588  /// Functions for poisoning/unpoisoning local variables
589  FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
590 
591  /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin
592  /// pointers.
593  FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
594  FunctionCallee MsanMetadataPtrForLoad_1_8[4];
595  FunctionCallee MsanMetadataPtrForStore_1_8[4];
596  FunctionCallee MsanInstrumentAsmStoreFn;
597 
598  /// Helper to choose between different MsanMetadataPtrXxx().
599  FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
600 
601  /// Memory map parameters used in application-to-shadow calculation.
602  const MemoryMapParams *MapParams;
603 
604  /// Custom memory map parameters used when -msan-shadow-base or
605  // -msan-origin-base is provided.
606  MemoryMapParams CustomMapParams;
607 
608  MDNode *ColdCallWeights;
609 
610  /// Branch weights for origin store.
611  MDNode *OriginStoreWeights;
612 };
613 
614 void insertModuleCtor(Module &M) {
617  /*InitArgTypes=*/{},
618  /*InitArgs=*/{},
619  // This callback is invoked when the functions are created the first
620  // time. Hook them into the global ctors list in that case:
621  [&](Function *Ctor, FunctionCallee) {
622  if (!ClWithComdat) {
623  appendToGlobalCtors(M, Ctor, 0);
624  return;
625  }
626  Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
627  Ctor->setComdat(MsanCtorComdat);
628  appendToGlobalCtors(M, Ctor, 0, Ctor);
629  });
630 }
631 
632 template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) {
633  return (Opt.getNumOccurrences() > 0) ? Opt : Default;
634 }
635 
636 } // end anonymous namespace
637 
639  bool EagerChecks)
640  : Kernel(getOptOrDefault(ClEnableKmsan, K)),
641  TrackOrigins(getOptOrDefault(ClTrackOrigins, Kernel ? 2 : TO)),
642  Recover(getOptOrDefault(ClKeepGoing, Kernel || R)),
643  EagerChecks(getOptOrDefault(ClEagerChecks, EagerChecks)) {}
644 
647  MemorySanitizer Msan(*F.getParent(), Options);
648  if (Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
649  return PreservedAnalyses::none();
650  return PreservedAnalyses::all();
651 }
652 
655  if (Options.Kernel)
656  return PreservedAnalyses::all();
657  insertModuleCtor(M);
658  return PreservedAnalyses::none();
659 }
660 
662  raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
664  OS, MapClassName2PassName);
665  OS << "<";
666  if (Options.Recover)
667  OS << "recover;";
668  if (Options.Kernel)
669  OS << "kernel;";
670  if (Options.EagerChecks)
671  OS << "eager-checks;";
672  OS << "track-origins=" << Options.TrackOrigins;
673  OS << ">";
674 }
675 
676 /// Create a non-const global initialized with the given string.
677 ///
678 /// Creates a writable global for Str so that we can pass it to the
679 /// run-time lib. Runtime uses first 4 bytes of the string to store the
680 /// frame ID, so the string needs to be mutable.
682  StringRef Str) {
683  Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
684  return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false,
685  GlobalValue::PrivateLinkage, StrConst, "");
686 }
687 
688 /// Create KMSAN API callbacks.
689 void MemorySanitizer::createKernelApi(Module &M) {
690  IRBuilder<> IRB(*C);
691 
692  // These will be initialized in insertKmsanPrologue().
693  RetvalTLS = nullptr;
694  RetvalOriginTLS = nullptr;
695  ParamTLS = nullptr;
696  ParamOriginTLS = nullptr;
697  VAArgTLS = nullptr;
698  VAArgOriginTLS = nullptr;
699  VAArgOverflowSizeTLS = nullptr;
700 
701  WarningFn = M.getOrInsertFunction("__msan_warning", IRB.getVoidTy(),
702  IRB.getInt32Ty());
703  // Requests the per-task context state (kmsan_context_state*) from the
704  // runtime library.
705  MsanContextStateTy = StructType::get(
706  ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
707  ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
708  ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
709  ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
710  IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
711  OriginTy);
712  MsanGetContextStateFn = M.getOrInsertFunction(
713  "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0));
714 
715  Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
716  PointerType::get(IRB.getInt32Ty(), 0));
717 
718  for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
719  std::string name_load =
720  "__msan_metadata_ptr_for_load_" + std::to_string(size);
721  std::string name_store =
722  "__msan_metadata_ptr_for_store_" + std::to_string(size);
723  MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction(
724  name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
725  MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction(
726  name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
727  }
728 
729  MsanMetadataPtrForLoadN = M.getOrInsertFunction(
730  "__msan_metadata_ptr_for_load_n", RetTy,
731  PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
732  MsanMetadataPtrForStoreN = M.getOrInsertFunction(
733  "__msan_metadata_ptr_for_store_n", RetTy,
734  PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
735 
736  // Functions for poisoning and unpoisoning memory.
737  MsanPoisonAllocaFn =
738  M.getOrInsertFunction("__msan_poison_alloca", IRB.getVoidTy(),
739  IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy());
740  MsanUnpoisonAllocaFn = M.getOrInsertFunction(
741  "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
742 }
743 
745  return M.getOrInsertGlobal(Name, Ty, [&] {
746  return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
747  nullptr, Name, nullptr,
749  });
750 }
751 
752 /// Insert declarations for userspace-specific functions and globals.
753 void MemorySanitizer::createUserspaceApi(Module &M) {
754  IRBuilder<> IRB(*C);
755 
756  // Create the callback.
757  // FIXME: this function should have "Cold" calling conv,
758  // which is not yet implemented.
759  StringRef WarningFnName = Recover ? "__msan_warning_with_origin"
760  : "__msan_warning_with_origin_noreturn";
761  WarningFn =
762  M.getOrInsertFunction(WarningFnName, IRB.getVoidTy(), IRB.getInt32Ty());
763 
764  // Create the global TLS variables.
765  RetvalTLS =
766  getOrInsertGlobal(M, "__msan_retval_tls",
767  ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
768 
769  RetvalOriginTLS = getOrInsertGlobal(M, "__msan_retval_origin_tls", OriginTy);
770 
771  ParamTLS =
772  getOrInsertGlobal(M, "__msan_param_tls",
773  ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
774 
775  ParamOriginTLS =
776  getOrInsertGlobal(M, "__msan_param_origin_tls",
777  ArrayType::get(OriginTy, kParamTLSSize / 4));
778 
779  VAArgTLS =
780  getOrInsertGlobal(M, "__msan_va_arg_tls",
781  ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
782 
783  VAArgOriginTLS =
784  getOrInsertGlobal(M, "__msan_va_arg_origin_tls",
785  ArrayType::get(OriginTy, kParamTLSSize / 4));
786 
787  VAArgOverflowSizeTLS =
788  getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
789 
790  for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
791  AccessSizeIndex++) {
792  unsigned AccessSize = 1 << AccessSizeIndex;
793  std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
794  SmallVector<std::pair<unsigned, Attribute>, 2> MaybeWarningFnAttrs;
795  MaybeWarningFnAttrs.push_back(std::make_pair(
796  AttributeList::FirstArgIndex, Attribute::get(*C, Attribute::ZExt)));
797  MaybeWarningFnAttrs.push_back(std::make_pair(
798  AttributeList::FirstArgIndex + 1, Attribute::get(*C, Attribute::ZExt)));
799  MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
800  FunctionName, AttributeList::get(*C, MaybeWarningFnAttrs),
801  IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
802 
803  FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
804  SmallVector<std::pair<unsigned, Attribute>, 2> MaybeStoreOriginFnAttrs;
805  MaybeStoreOriginFnAttrs.push_back(std::make_pair(
806  AttributeList::FirstArgIndex, Attribute::get(*C, Attribute::ZExt)));
807  MaybeStoreOriginFnAttrs.push_back(std::make_pair(
808  AttributeList::FirstArgIndex + 2, Attribute::get(*C, Attribute::ZExt)));
809  MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
810  FunctionName, AttributeList::get(*C, MaybeStoreOriginFnAttrs),
811  IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt8PtrTy(),
812  IRB.getInt32Ty());
813  }
814 
815  MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
816  "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
817  IRB.getInt8PtrTy(), IntptrTy);
818  MsanPoisonStackFn =
819  M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
820  IRB.getInt8PtrTy(), IntptrTy);
821 }
822 
823 /// Insert extern declaration of runtime-provided functions and globals.
824 void MemorySanitizer::initializeCallbacks(Module &M) {
825  // Only do this once.
826  if (CallbacksInitialized)
827  return;
828 
829  IRBuilder<> IRB(*C);
830  // Initialize callbacks that are common for kernel and userspace
831  // instrumentation.
832  MsanChainOriginFn = M.getOrInsertFunction(
833  "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
834  MsanSetOriginFn =
835  M.getOrInsertFunction("__msan_set_origin", IRB.getVoidTy(),
836  IRB.getInt8PtrTy(), IntptrTy, IRB.getInt32Ty());
837  MemmoveFn = M.getOrInsertFunction(
838  "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
839  IRB.getInt8PtrTy(), IntptrTy);
840  MemcpyFn = M.getOrInsertFunction(
841  "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
842  IntptrTy);
843  MemsetFn = M.getOrInsertFunction(
844  "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
845  IntptrTy);
846 
847  MsanInstrumentAsmStoreFn =
848  M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
849  PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
850 
851  if (CompileKernel) {
852  createKernelApi(M);
853  } else {
854  createUserspaceApi(M);
855  }
856  CallbacksInitialized = true;
857 }
858 
859 FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
860  int size) {
861  FunctionCallee *Fns =
862  isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
863  switch (size) {
864  case 1:
865  return Fns[0];
866  case 2:
867  return Fns[1];
868  case 4:
869  return Fns[2];
870  case 8:
871  return Fns[3];
872  default:
873  return nullptr;
874  }
875 }
876 
877 /// Module-level initialization.
878 ///
879 /// inserts a call to __msan_init to the module's constructor list.
880 void MemorySanitizer::initializeModule(Module &M) {
881  auto &DL = M.getDataLayout();
882 
883  bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
884  bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
885  // Check the overrides first
886  if (ShadowPassed || OriginPassed) {
887  CustomMapParams.AndMask = ClAndMask;
888  CustomMapParams.XorMask = ClXorMask;
889  CustomMapParams.ShadowBase = ClShadowBase;
890  CustomMapParams.OriginBase = ClOriginBase;
891  MapParams = &CustomMapParams;
892  } else {
893  Triple TargetTriple(M.getTargetTriple());
894  switch (TargetTriple.getOS()) {
895  case Triple::FreeBSD:
896  switch (TargetTriple.getArch()) {
897  case Triple::x86_64:
898  MapParams = FreeBSD_X86_MemoryMapParams.bits64;
899  break;
900  case Triple::x86:
901  MapParams = FreeBSD_X86_MemoryMapParams.bits32;
902  break;
903  default:
904  report_fatal_error("unsupported architecture");
905  }
906  break;
907  case Triple::NetBSD:
908  switch (TargetTriple.getArch()) {
909  case Triple::x86_64:
910  MapParams = NetBSD_X86_MemoryMapParams.bits64;
911  break;
912  default:
913  report_fatal_error("unsupported architecture");
914  }
915  break;
916  case Triple::Linux:
917  switch (TargetTriple.getArch()) {
918  case Triple::x86_64:
919  MapParams = Linux_X86_MemoryMapParams.bits64;
920  break;
921  case Triple::x86:
922  MapParams = Linux_X86_MemoryMapParams.bits32;
923  break;
924  case Triple::mips64:
925  case Triple::mips64el:
926  MapParams = Linux_MIPS_MemoryMapParams.bits64;
927  break;
928  case Triple::ppc64:
929  case Triple::ppc64le:
930  MapParams = Linux_PowerPC_MemoryMapParams.bits64;
931  break;
932  case Triple::systemz:
933  MapParams = Linux_S390_MemoryMapParams.bits64;
934  break;
935  case Triple::aarch64:
936  case Triple::aarch64_be:
937  MapParams = Linux_ARM_MemoryMapParams.bits64;
938  break;
939  default:
940  report_fatal_error("unsupported architecture");
941  }
942  break;
943  default:
944  report_fatal_error("unsupported operating system");
945  }
946  }
947 
948  C = &(M.getContext());
949  IRBuilder<> IRB(*C);
950  IntptrTy = IRB.getIntPtrTy(DL);
951  OriginTy = IRB.getInt32Ty();
952 
953  ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
954  OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
955 
956  if (!CompileKernel) {
957  if (TrackOrigins)
958  M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
959  return new GlobalVariable(
960  M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
961  IRB.getInt32(TrackOrigins), "__msan_track_origins");
962  });
963 
964  if (Recover)
965  M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
966  return new GlobalVariable(M, IRB.getInt32Ty(), true,
967  GlobalValue::WeakODRLinkage,
968  IRB.getInt32(Recover), "__msan_keep_going");
969  });
970 }
971 }
972 
973 namespace {
974 
975 /// A helper class that handles instrumentation of VarArg
976 /// functions on a particular platform.
977 ///
978 /// Implementations are expected to insert the instrumentation
979 /// necessary to propagate argument shadow through VarArg function
980 /// calls. Visit* methods are called during an InstVisitor pass over
981 /// the function, and should avoid creating new basic blocks. A new
982 /// instance of this class is created for each instrumented function.
983 struct VarArgHelper {
984  virtual ~VarArgHelper() = default;
985 
986  /// Visit a CallBase.
987  virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
988 
989  /// Visit a va_start call.
990  virtual void visitVAStartInst(VAStartInst &I) = 0;
991 
992  /// Visit a va_copy call.
993  virtual void visitVACopyInst(VACopyInst &I) = 0;
994 
995  /// Finalize function instrumentation.
996  ///
997  /// This method is called after visiting all interesting (see above)
998  /// instructions in a function.
999  virtual void finalizeInstrumentation() = 0;
1000 };
1001 
1002 struct MemorySanitizerVisitor;
1003 
1004 } // end anonymous namespace
1005 
1006 static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1007  MemorySanitizerVisitor &Visitor);
1008 
1009 static unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
1010  if (TypeSize <= 8) return 0;
1011  return Log2_32_Ceil((TypeSize + 7) / 8);
1012 }
1013 
1014 namespace {
1015 
1016 /// This class does all the work for a given function. Store and Load
1017 /// instructions store and load corresponding shadow and origin
1018 /// values. Most instructions propagate shadow from arguments to their
1019 /// return values. Certain instructions (most importantly, BranchInst)
1020 /// test their argument shadow and print reports (with a runtime call) if it's
1021 /// non-zero.
1022 struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1023  Function &F;
1024  MemorySanitizer &MS;
1025  SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1026  ValueMap<Value*, Value*> ShadowMap, OriginMap;
1027  std::unique_ptr<VarArgHelper> VAHelper;
1028  const TargetLibraryInfo *TLI;
1029  Instruction *FnPrologueEnd;
1030 
1031  // The following flags disable parts of MSan instrumentation based on
1032  // exclusion list contents and command-line options.
1033  bool InsertChecks;
1034  bool PropagateShadow;
1035  bool PoisonStack;
1036  bool PoisonUndef;
1037 
1038  struct ShadowOriginAndInsertPoint {
1039  Value *Shadow;
1040  Value *Origin;
1041  Instruction *OrigIns;
1042 
1043  ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1044  : Shadow(S), Origin(O), OrigIns(I) {}
1045  };
1046  SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1047  bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics;
1048  SmallSet<AllocaInst *, 16> AllocaSet;
1050  SmallVector<StoreInst *, 16> StoreList;
1051 
1052  MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1053  const TargetLibraryInfo &TLI)
1054  : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
1055  bool SanitizeFunction =
1056  F.hasFnAttribute(Attribute::SanitizeMemory) && !ClDisableChecks;
1057  InsertChecks = SanitizeFunction;
1058  PropagateShadow = SanitizeFunction;
1059  PoisonStack = SanitizeFunction && ClPoisonStack;
1060  PoisonUndef = SanitizeFunction && ClPoisonUndef;
1061 
1062  // In the presence of unreachable blocks, we may see Phi nodes with
1063  // incoming nodes from such blocks. Since InstVisitor skips unreachable
1064  // blocks, such nodes will not have any shadow value associated with them.
1065  // It's easier to remove unreachable blocks than deal with missing shadow.
1067 
1068  MS.initializeCallbacks(*F.getParent());
1069  FnPrologueEnd = IRBuilder<>(F.getEntryBlock().getFirstNonPHI())
1070  .CreateIntrinsic(Intrinsic::donothing, {}, {});
1071 
1072  if (MS.CompileKernel) {
1073  IRBuilder<> IRB(FnPrologueEnd);
1074  insertKmsanPrologue(IRB);
1075  }
1076 
1077  LLVM_DEBUG(if (!InsertChecks) dbgs()
1078  << "MemorySanitizer is not inserting checks into '"
1079  << F.getName() << "'\n");
1080  }
1081 
1082  bool isInPrologue(Instruction &I) {
1083  return I.getParent() == FnPrologueEnd->getParent() &&
1084  (&I == FnPrologueEnd || I.comesBefore(FnPrologueEnd));
1085  }
1086 
1087  Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1088  if (MS.TrackOrigins <= 1) return V;
1089  return IRB.CreateCall(MS.MsanChainOriginFn, V);
1090  }
1091 
1092  Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1093  const DataLayout &DL = F.getParent()->getDataLayout();
1094  unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1095  if (IntptrSize == kOriginSize) return Origin;
1096  assert(IntptrSize == kOriginSize * 2);
1097  Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
1098  return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
1099  }
1100 
1101  /// Fill memory range with the given origin value.
1102  void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1103  unsigned Size, Align Alignment) {
1104  const DataLayout &DL = F.getParent()->getDataLayout();
1105  const Align IntptrAlignment = DL.getABITypeAlign(MS.IntptrTy);
1106  unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1107  assert(IntptrAlignment >= kMinOriginAlignment);
1108  assert(IntptrSize >= kOriginSize);
1109 
1110  unsigned Ofs = 0;
1111  Align CurrentAlignment = Alignment;
1112  if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1113  Value *IntptrOrigin = originToIntptr(IRB, Origin);
1114  Value *IntptrOriginPtr =
1115  IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
1116  for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1117  Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
1118  : IntptrOriginPtr;
1119  IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
1120  Ofs += IntptrSize / kOriginSize;
1121  CurrentAlignment = IntptrAlignment;
1122  }
1123  }
1124 
1125  for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1126  Value *GEP =
1127  i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
1128  IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
1129  CurrentAlignment = kMinOriginAlignment;
1130  }
1131  }
1132 
1133  void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1134  Value *OriginPtr, Align Alignment, bool AsCall) {
1135  const DataLayout &DL = F.getParent()->getDataLayout();
1136  const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1137  unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
1138  Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1139  if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1140  if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
1141  paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1142  OriginAlignment);
1143  return;
1144  }
1145 
1146  unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
1147  unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1148  if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1149  FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1150  Value *ConvertedShadow2 =
1151  IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1152  CallBase *CB = IRB.CreateCall(
1153  Fn, {ConvertedShadow2,
1154  IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), Origin});
1155  CB->addParamAttr(0, Attribute::ZExt);
1156  CB->addParamAttr(2, Attribute::ZExt);
1157  } else {
1158  Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1160  Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
1161  IRBuilder<> IRBNew(CheckTerm);
1162  paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1163  OriginAlignment);
1164  }
1165  }
1166 
1167  void materializeStores(bool InstrumentWithCalls) {
1168  for (StoreInst *SI : StoreList) {
1169  IRBuilder<> IRB(SI);
1170  Value *Val = SI->getValueOperand();
1171  Value *Addr = SI->getPointerOperand();
1172  Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1173  Value *ShadowPtr, *OriginPtr;
1174  Type *ShadowTy = Shadow->getType();
1175  const Align Alignment = SI->getAlign();
1176  const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1177  std::tie(ShadowPtr, OriginPtr) =
1178  getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1179 
1180  StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
1181  LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
1182  (void)NewSI;
1183 
1184  if (SI->isAtomic())
1185  SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
1186 
1187  if (MS.TrackOrigins && !SI->isAtomic())
1188  storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
1189  OriginAlignment, InstrumentWithCalls);
1190  }
1191  }
1192 
1193  /// Helper function to insert a warning at IRB's current insert point.
1194  void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1195  if (!Origin)
1196  Origin = (Value *)IRB.getInt32(0);
1197  assert(Origin->getType()->isIntegerTy());
1198  IRB.CreateCall(MS.WarningFn, Origin)->setCannotMerge();
1199  // FIXME: Insert UnreachableInst if !MS.Recover?
1200  // This may invalidate some of the following checks and needs to be done
1201  // at the very end.
1202  }
1203 
1204  void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
1205  bool AsCall) {
1206  IRBuilder<> IRB(OrigIns);
1207  LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n");
1208  Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1209  LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n");
1210 
1211  if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1212  if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
1213  insertWarningFn(IRB, Origin);
1214  }
1215  return;
1216  }
1217 
1218  const DataLayout &DL = OrigIns->getModule()->getDataLayout();
1219 
1220  unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
1221  unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1222  if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1223  FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1224  Value *ConvertedShadow2 =
1225  IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1226  CallBase *CB = IRB.CreateCall(
1227  Fn, {ConvertedShadow2,
1228  MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)});
1229  CB->addParamAttr(0, Attribute::ZExt);
1230  CB->addParamAttr(1, Attribute::ZExt);
1231  } else {
1232  Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1234  Cmp, OrigIns,
1235  /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
1236 
1237  IRB.SetInsertPoint(CheckTerm);
1238  insertWarningFn(IRB, Origin);
1239  LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
1240  }
1241  }
1242 
1243  void materializeChecks(bool InstrumentWithCalls) {
1244  for (const auto &ShadowData : InstrumentationList) {
1245  Instruction *OrigIns = ShadowData.OrigIns;
1246  Value *Shadow = ShadowData.Shadow;
1247  Value *Origin = ShadowData.Origin;
1248  materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
1249  }
1250  LLVM_DEBUG(dbgs() << "DONE:\n" << F);
1251  }
1252 
1253  // Returns the last instruction in the new prologue
1254  void insertKmsanPrologue(IRBuilder<> &IRB) {
1255  Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
1256  Constant *Zero = IRB.getInt32(0);
1257  MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1258  {Zero, IRB.getInt32(0)}, "param_shadow");
1259  MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1260  {Zero, IRB.getInt32(1)}, "retval_shadow");
1261  MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1262  {Zero, IRB.getInt32(2)}, "va_arg_shadow");
1263  MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1264  {Zero, IRB.getInt32(3)}, "va_arg_origin");
1265  MS.VAArgOverflowSizeTLS =
1266  IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1267  {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
1268  MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1269  {Zero, IRB.getInt32(5)}, "param_origin");
1270  MS.RetvalOriginTLS =
1271  IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1272  {Zero, IRB.getInt32(6)}, "retval_origin");
1273  }
1274 
1275  /// Add MemorySanitizer instrumentation to a function.
1276  bool runOnFunction() {
1277  // Iterate all BBs in depth-first order and create shadow instructions
1278  // for all instructions (where applicable).
1279  // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1280  for (BasicBlock *BB : depth_first(FnPrologueEnd->getParent()))
1281  visit(*BB);
1282 
1283  // Finalize PHI nodes.
1284  for (PHINode *PN : ShadowPHINodes) {
1285  PHINode *PNS = cast<PHINode>(getShadow(PN));
1286  PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1287  size_t NumValues = PN->getNumIncomingValues();
1288  for (size_t v = 0; v < NumValues; v++) {
1289  PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1290  if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1291  }
1292  }
1293 
1294  VAHelper->finalizeInstrumentation();
1295 
1296  // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1297  // instrumenting only allocas.
1298  if (InstrumentLifetimeStart) {
1299  for (auto Item : LifetimeStartList) {
1300  instrumentAlloca(*Item.second, Item.first);
1301  AllocaSet.erase(Item.second);
1302  }
1303  }
1304  // Poison the allocas for which we didn't instrument the corresponding
1305  // lifetime intrinsics.
1306  for (AllocaInst *AI : AllocaSet)
1307  instrumentAlloca(*AI);
1308 
1309  bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
1310  InstrumentationList.size() + StoreList.size() >
1312 
1313  // Insert shadow value checks.
1314  materializeChecks(InstrumentWithCalls);
1315 
1316  // Delayed instrumentation of StoreInst.
1317  // This may not add new address checks.
1318  materializeStores(InstrumentWithCalls);
1319 
1320  return true;
1321  }
1322 
1323  /// Compute the shadow type that corresponds to a given Value.
1324  Type *getShadowTy(Value *V) {
1325  return getShadowTy(V->getType());
1326  }
1327 
1328  /// Compute the shadow type that corresponds to a given Type.
1329  Type *getShadowTy(Type *OrigTy) {
1330  if (!OrigTy->isSized()) {
1331  return nullptr;
1332  }
1333  // For integer type, shadow is the same as the original type.
1334  // This may return weird-sized types like i1.
1335  if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
1336  return IT;
1337  const DataLayout &DL = F.getParent()->getDataLayout();
1338  if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1339  uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
1340  return FixedVectorType::get(IntegerType::get(*MS.C, EltSize),
1341  cast<FixedVectorType>(VT)->getNumElements());
1342  }
1343  if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1344  return ArrayType::get(getShadowTy(AT->getElementType()),
1345  AT->getNumElements());
1346  }
1347  if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1348  SmallVector<Type*, 4> Elements;
1349  for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1350  Elements.push_back(getShadowTy(ST->getElementType(i)));
1351  StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
1352  LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
1353  return Res;
1354  }
1355  uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
1356  return IntegerType::get(*MS.C, TypeSize);
1357  }
1358 
1359  /// Flatten a vector type.
1360  Type *getShadowTyNoVec(Type *ty) {
1361  if (VectorType *vt = dyn_cast<VectorType>(ty))
1362  return IntegerType::get(*MS.C,
1363  vt->getPrimitiveSizeInBits().getFixedSize());
1364  return ty;
1365  }
1366 
1367  /// Extract combined shadow of struct elements as a bool
1368  Value *collapseStructShadow(StructType *Struct, Value *Shadow,
1369  IRBuilder<> &IRB) {
1370  Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0);
1371  Value *Aggregator = FalseVal;
1372 
1373  for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
1374  // Combine by ORing together each element's bool shadow
1375  Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1376  Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1377  Value *ShadowBool = convertToBool(ShadowInner, IRB);
1378 
1379  if (Aggregator != FalseVal)
1380  Aggregator = IRB.CreateOr(Aggregator, ShadowBool);
1381  else
1382  Aggregator = ShadowBool;
1383  }
1384 
1385  return Aggregator;
1386  }
1387 
1388  // Extract combined shadow of array elements
1389  Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
1390  IRBuilder<> &IRB) {
1391  if (!Array->getNumElements())
1392  return IRB.getIntN(/* width */ 1, /* value */ 0);
1393 
1394  Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
1395  Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1396 
1397  for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
1398  Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1399  Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1400  Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
1401  }
1402  return Aggregator;
1403  }
1404 
1405  /// Convert a shadow value to it's flattened variant. The resulting
1406  /// shadow may not necessarily have the same bit width as the input
1407  /// value, but it will always be comparable to zero.
1408  Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1409  if (StructType *Struct = dyn_cast<StructType>(V->getType()))
1410  return collapseStructShadow(Struct, V, IRB);
1411  if (ArrayType *Array = dyn_cast<ArrayType>(V->getType()))
1412  return collapseArrayShadow(Array, V, IRB);
1413  Type *Ty = V->getType();
1414  Type *NoVecTy = getShadowTyNoVec(Ty);
1415  if (Ty == NoVecTy) return V;
1416  return IRB.CreateBitCast(V, NoVecTy);
1417  }
1418 
1419  // Convert a scalar value to an i1 by comparing with 0
1420  Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1421  Type *VTy = V->getType();
1422  assert(VTy->isIntegerTy());
1423  if (VTy->getIntegerBitWidth() == 1)
1424  // Just converting a bool to a bool, so do nothing.
1425  return V;
1426  return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name);
1427  }
1428 
1429  /// Compute the integer shadow offset that corresponds to a given
1430  /// application address.
1431  ///
1432  /// Offset = (Addr & ~AndMask) ^ XorMask
1433  Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1434  Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
1435 
1436  uint64_t AndMask = MS.MapParams->AndMask;
1437  if (AndMask)
1438  OffsetLong =
1439  IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
1440 
1441  uint64_t XorMask = MS.MapParams->XorMask;
1442  if (XorMask)
1443  OffsetLong =
1444  IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
1445  return OffsetLong;
1446  }
1447 
1448  /// Compute the shadow and origin addresses corresponding to a given
1449  /// application address.
1450  ///
1451  /// Shadow = ShadowBase + Offset
1452  /// Origin = (OriginBase + Offset) & ~3ULL
1453  std::pair<Value *, Value *>
1454  getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1455  MaybeAlign Alignment) {
1456  Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1457  Value *ShadowLong = ShadowOffset;
1458  uint64_t ShadowBase = MS.MapParams->ShadowBase;
1459  if (ShadowBase != 0) {
1460  ShadowLong =
1461  IRB.CreateAdd(ShadowLong,
1462  ConstantInt::get(MS.IntptrTy, ShadowBase));
1463  }
1464  Value *ShadowPtr =
1465  IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1466  Value *OriginPtr = nullptr;
1467  if (MS.TrackOrigins) {
1468  Value *OriginLong = ShadowOffset;
1469  uint64_t OriginBase = MS.MapParams->OriginBase;
1470  if (OriginBase != 0)
1471  OriginLong = IRB.CreateAdd(OriginLong,
1472  ConstantInt::get(MS.IntptrTy, OriginBase));
1473  if (!Alignment || *Alignment < kMinOriginAlignment) {
1475  OriginLong =
1476  IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask));
1477  }
1478  OriginPtr =
1479  IRB.CreateIntToPtr(OriginLong, PointerType::get(MS.OriginTy, 0));
1480  }
1481  return std::make_pair(ShadowPtr, OriginPtr);
1482  }
1483 
1484  std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1485  IRBuilder<> &IRB,
1486  Type *ShadowTy,
1487  bool isStore) {
1488  Value *ShadowOriginPtrs;
1489  const DataLayout &DL = F.getParent()->getDataLayout();
1490  int Size = DL.getTypeStoreSize(ShadowTy);
1491 
1492  FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size);
1493  Value *AddrCast =
1495  if (Getter) {
1496  ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast);
1497  } else {
1498  Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
1499  ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN
1500  : MS.MsanMetadataPtrForLoadN,
1501  {AddrCast, SizeVal});
1502  }
1503  Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
1504  ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
1505  Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
1506 
1507  return std::make_pair(ShadowPtr, OriginPtr);
1508  }
1509 
1510  std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1511  Type *ShadowTy,
1512  MaybeAlign Alignment,
1513  bool isStore) {
1514  if (MS.CompileKernel)
1515  return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1516  return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1517  }
1518 
1519  /// Compute the shadow address for a given function argument.
1520  ///
1521  /// Shadow = ParamTLS+ArgOffset.
1522  Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
1523  int ArgOffset) {
1524  Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1525  if (ArgOffset)
1526  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1527  return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1528  "_msarg");
1529  }
1530 
1531  /// Compute the origin address for a given function argument.
1532  Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
1533  int ArgOffset) {
1534  if (!MS.TrackOrigins)
1535  return nullptr;
1536  Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1537  if (ArgOffset)
1538  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1539  return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1540  "_msarg_o");
1541  }
1542 
1543  /// Compute the shadow address for a retval.
1544  Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1545  return IRB.CreatePointerCast(MS.RetvalTLS,
1546  PointerType::get(getShadowTy(A), 0),
1547  "_msret");
1548  }
1549 
1550  /// Compute the origin address for a retval.
1551  Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1552  // We keep a single origin for the entire retval. Might be too optimistic.
1553  return MS.RetvalOriginTLS;
1554  }
1555 
1556  /// Set SV to be the shadow value for V.
1557  void setShadow(Value *V, Value *SV) {
1558  assert(!ShadowMap.count(V) && "Values may only have one shadow");
1559  ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1560  }
1561 
1562  /// Set Origin to be the origin value for V.
1563  void setOrigin(Value *V, Value *Origin) {
1564  if (!MS.TrackOrigins) return;
1565  assert(!OriginMap.count(V) && "Values may only have one origin");
1566  LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
1567  OriginMap[V] = Origin;
1568  }
1569 
1570  Constant *getCleanShadow(Type *OrigTy) {
1571  Type *ShadowTy = getShadowTy(OrigTy);
1572  if (!ShadowTy)
1573  return nullptr;
1574  return Constant::getNullValue(ShadowTy);
1575  }
1576 
1577  /// Create a clean shadow value for a given value.
1578  ///
1579  /// Clean shadow (all zeroes) means all bits of the value are defined
1580  /// (initialized).
1581  Constant *getCleanShadow(Value *V) {
1582  return getCleanShadow(V->getType());
1583  }
1584 
1585  /// Create a dirty shadow of a given shadow type.
1586  Constant *getPoisonedShadow(Type *ShadowTy) {
1587  assert(ShadowTy);
1588  if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1589  return Constant::getAllOnesValue(ShadowTy);
1590  if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1591  SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1592  getPoisonedShadow(AT->getElementType()));
1593  return ConstantArray::get(AT, Vals);
1594  }
1595  if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1597  for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1598  Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1599  return ConstantStruct::get(ST, Vals);
1600  }
1601  llvm_unreachable("Unexpected shadow type");
1602  }
1603 
1604  /// Create a dirty shadow for a given value.
1605  Constant *getPoisonedShadow(Value *V) {
1606  Type *ShadowTy = getShadowTy(V);
1607  if (!ShadowTy)
1608  return nullptr;
1609  return getPoisonedShadow(ShadowTy);
1610  }
1611 
1612  /// Create a clean (zero) origin.
1613  Value *getCleanOrigin() {
1614  return Constant::getNullValue(MS.OriginTy);
1615  }
1616 
1617  /// Get the shadow value for a given Value.
1618  ///
1619  /// This function either returns the value set earlier with setShadow,
1620  /// or extracts if from ParamTLS (for function arguments).
1621  Value *getShadow(Value *V) {
1622  if (Instruction *I = dyn_cast<Instruction>(V)) {
1623  if (!PropagateShadow || I->getMetadata("nosanitize"))
1624  return getCleanShadow(V);
1625  // For instructions the shadow is already stored in the map.
1626  Value *Shadow = ShadowMap[V];
1627  if (!Shadow) {
1628  LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
1629  (void)I;
1630  assert(Shadow && "No shadow for a value");
1631  }
1632  return Shadow;
1633  }
1634  if (UndefValue *U = dyn_cast<UndefValue>(V)) {
1635  Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1636  : getCleanShadow(V);
1637  LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
1638  (void)U;
1639  return AllOnes;
1640  }
1641  if (Argument *A = dyn_cast<Argument>(V)) {
1642  // For arguments we compute the shadow on demand and store it in the map.
1643  Value *&ShadowPtr = ShadowMap[V];
1644  if (ShadowPtr)
1645  return ShadowPtr;
1646  Function *F = A->getParent();
1647  IRBuilder<> EntryIRB(FnPrologueEnd);
1648  unsigned ArgOffset = 0;
1649  const DataLayout &DL = F->getParent()->getDataLayout();
1650  for (auto &FArg : F->args()) {
1651  if (!FArg.getType()->isSized()) {
1652  LLVM_DEBUG(dbgs() << "Arg is not sized\n");
1653  continue;
1654  }
1655 
1656  unsigned Size = FArg.hasByValAttr()
1657  ? DL.getTypeAllocSize(FArg.getParamByValType())
1658  : DL.getTypeAllocSize(FArg.getType());
1659 
1660  if (A == &FArg) {
1661  bool Overflow = ArgOffset + Size > kParamTLSSize;
1662  if (FArg.hasByValAttr()) {
1663  // ByVal pointer itself has clean shadow. We copy the actual
1664  // argument shadow to the underlying memory.
1665  // Figure out maximal valid memcpy alignment.
1666  const Align ArgAlign = DL.getValueOrABITypeAlignment(
1667  MaybeAlign(FArg.getParamAlignment()), FArg.getParamByValType());
1668  Value *CpShadowPtr, *CpOriginPtr;
1669  std::tie(CpShadowPtr, CpOriginPtr) =
1670  getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1671  /*isStore*/ true);
1672  if (!PropagateShadow || Overflow) {
1673  // ParamTLS overflow.
1674  EntryIRB.CreateMemSet(
1675  CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()),
1676  Size, ArgAlign);
1677  } else {
1678  Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1679  const Align CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1680  Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
1681  CopyAlign, Size);
1682  LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
1683  (void)Cpy;
1684 
1685  if (MS.TrackOrigins) {
1686  Value *OriginPtr =
1687  getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1688  // FIXME: OriginSize should be:
1689  // alignTo(V % kMinOriginAlignment + Size, kMinOriginAlignment)
1690  unsigned OriginSize = alignTo(Size, kMinOriginAlignment);
1691  EntryIRB.CreateMemCpy(
1692  CpOriginPtr,
1693  /* by getShadowOriginPtr */ kMinOriginAlignment, OriginPtr,
1694  /* by origin_tls[ArgOffset] */ kMinOriginAlignment,
1695  OriginSize);
1696  }
1697  }
1698  }
1699 
1700  if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
1701  (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
1702  ShadowPtr = getCleanShadow(V);
1703  setOrigin(A, getCleanOrigin());
1704  } else {
1705  // Shadow over TLS
1706  Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1707  ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg), Base,
1709  if (MS.TrackOrigins) {
1710  Value *OriginPtr =
1711  getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1712  setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
1713  }
1714  }
1715  LLVM_DEBUG(dbgs()
1716  << " ARG: " << FArg << " ==> " << *ShadowPtr << "\n");
1717  break;
1718  }
1719 
1720  ArgOffset += alignTo(Size, kShadowTLSAlignment);
1721  }
1722  assert(ShadowPtr && "Could not find shadow for an argument");
1723  return ShadowPtr;
1724  }
1725  // For everything else the shadow is zero.
1726  return getCleanShadow(V);
1727  }
1728 
1729  /// Get the shadow for i-th argument of the instruction I.
1730  Value *getShadow(Instruction *I, int i) {
1731  return getShadow(I->getOperand(i));
1732  }
1733 
1734  /// Get the origin for a value.
1735  Value *getOrigin(Value *V) {
1736  if (!MS.TrackOrigins) return nullptr;
1737  if (!PropagateShadow) return getCleanOrigin();
1738  if (isa<Constant>(V)) return getCleanOrigin();
1739  assert((isa<Instruction>(V) || isa<Argument>(V)) &&
1740  "Unexpected value type in getOrigin()");
1741  if (Instruction *I = dyn_cast<Instruction>(V)) {
1742  if (I->getMetadata("nosanitize"))
1743  return getCleanOrigin();
1744  }
1745  Value *Origin = OriginMap[V];
1746  assert(Origin && "Missing origin");
1747  return Origin;
1748  }
1749 
1750  /// Get the origin for i-th argument of the instruction I.
1751  Value *getOrigin(Instruction *I, int i) {
1752  return getOrigin(I->getOperand(i));
1753  }
1754 
1755  /// Remember the place where a shadow check should be inserted.
1756  ///
1757  /// This location will be later instrumented with a check that will print a
1758  /// UMR warning in runtime if the shadow value is not 0.
1759  void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1760  assert(Shadow);
1761  if (!InsertChecks) return;
1762 #ifndef NDEBUG
1763  Type *ShadowTy = Shadow->getType();
1764  assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
1765  isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
1766  "Can only insert checks for integer, vector, and aggregate shadow "
1767  "types");
1768 #endif
1769  InstrumentationList.push_back(
1770  ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1771  }
1772 
1773  /// Remember the place where a shadow check should be inserted.
1774  ///
1775  /// This location will be later instrumented with a check that will print a
1776  /// UMR warning in runtime if the value is not fully defined.
1777  void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1778  assert(Val);
1779  Value *Shadow, *Origin;
1780  if (ClCheckConstantShadow) {
1781  Shadow = getShadow(Val);
1782  if (!Shadow) return;
1783  Origin = getOrigin(Val);
1784  } else {
1785  Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1786  if (!Shadow) return;
1787  Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1788  }
1789  insertShadowCheck(Shadow, Origin, OrigIns);
1790  }
1791 
1793  switch (a) {
1799  return AtomicOrdering::Release;
1805  }
1806  llvm_unreachable("Unknown ordering");
1807  }
1808 
1809  Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
1810  constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
1811  uint32_t OrderingTable[NumOrderings] = {};
1812 
1813  OrderingTable[(int)AtomicOrderingCABI::relaxed] =
1814  OrderingTable[(int)AtomicOrderingCABI::release] =
1816  OrderingTable[(int)AtomicOrderingCABI::consume] =
1817  OrderingTable[(int)AtomicOrderingCABI::acquire] =
1818  OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
1820  OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
1822 
1823  return ConstantDataVector::get(IRB.getContext(),
1824  makeArrayRef(OrderingTable, NumOrderings));
1825  }
1826 
1828  switch (a) {
1834  return AtomicOrdering::Acquire;
1840  }
1841  llvm_unreachable("Unknown ordering");
1842  }
1843 
1844  Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
1845  constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
1846  uint32_t OrderingTable[NumOrderings] = {};
1847 
1848  OrderingTable[(int)AtomicOrderingCABI::relaxed] =
1849  OrderingTable[(int)AtomicOrderingCABI::acquire] =
1850  OrderingTable[(int)AtomicOrderingCABI::consume] =
1852  OrderingTable[(int)AtomicOrderingCABI::release] =
1853  OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
1855  OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
1857 
1858  return ConstantDataVector::get(IRB.getContext(),
1859  makeArrayRef(OrderingTable, NumOrderings));
1860  }
1861 
1862  // ------------------- Visitors.
1864  void visit(Instruction &I) {
1865  if (I.getMetadata("nosanitize"))
1866  return;
1867  // Don't want to visit if we're in the prologue
1868  if (isInPrologue(I))
1869  return;
1871  }
1872 
1873  /// Instrument LoadInst
1874  ///
1875  /// Loads the corresponding shadow and (optionally) origin.
1876  /// Optionally, checks that the load address is fully defined.
1877  void visitLoadInst(LoadInst &I) {
1878  assert(I.getType()->isSized() && "Load type must have size");
1879  assert(!I.getMetadata("nosanitize"));
1880  IRBuilder<> IRB(I.getNextNode());
1881  Type *ShadowTy = getShadowTy(&I);
1882  Value *Addr = I.getPointerOperand();
1883  Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
1884  const Align Alignment = assumeAligned(I.getAlignment());
1885  if (PropagateShadow) {
1886  std::tie(ShadowPtr, OriginPtr) =
1887  getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
1888  setShadow(&I,
1889  IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
1890  } else {
1891  setShadow(&I, getCleanShadow(&I));
1892  }
1893 
1895  insertShadowCheck(I.getPointerOperand(), &I);
1896 
1897  if (I.isAtomic())
1898  I.setOrdering(addAcquireOrdering(I.getOrdering()));
1899 
1900  if (MS.TrackOrigins) {
1901  if (PropagateShadow) {
1902  const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1903  setOrigin(
1904  &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment));
1905  } else {
1906  setOrigin(&I, getCleanOrigin());
1907  }
1908  }
1909  }
1910 
1911  /// Instrument StoreInst
1912  ///
1913  /// Stores the corresponding shadow and (optionally) origin.
1914  /// Optionally, checks that the store address is fully defined.
1915  void visitStoreInst(StoreInst &I) {
1916  StoreList.push_back(&I);
1918  insertShadowCheck(I.getPointerOperand(), &I);
1919  }
1920 
1921  void handleCASOrRMW(Instruction &I) {
1922  assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
1923 
1924  IRBuilder<> IRB(&I);
1925  Value *Addr = I.getOperand(0);
1926  Value *Val = I.getOperand(1);
1927  Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, Val->getType(), Align(1),
1928  /*isStore*/ true)
1929  .first;
1930 
1932  insertShadowCheck(Addr, &I);
1933 
1934  // Only test the conditional argument of cmpxchg instruction.
1935  // The other argument can potentially be uninitialized, but we can not
1936  // detect this situation reliably without possible false positives.
1937  if (isa<AtomicCmpXchgInst>(I))
1938  insertShadowCheck(Val, &I);
1939 
1940  IRB.CreateStore(getCleanShadow(Val), ShadowPtr);
1941 
1942  setShadow(&I, getCleanShadow(&I));
1943  setOrigin(&I, getCleanOrigin());
1944  }
1945 
1946  void visitAtomicRMWInst(AtomicRMWInst &I) {
1947  handleCASOrRMW(I);
1948  I.setOrdering(addReleaseOrdering(I.getOrdering()));
1949  }
1950 
1951  void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1952  handleCASOrRMW(I);
1953  I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
1954  }
1955 
1956  // Vector manipulation.
1957  void visitExtractElementInst(ExtractElementInst &I) {
1958  insertShadowCheck(I.getOperand(1), &I);
1959  IRBuilder<> IRB(&I);
1960  setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
1961  "_msprop"));
1962  setOrigin(&I, getOrigin(&I, 0));
1963  }
1964 
1965  void visitInsertElementInst(InsertElementInst &I) {
1966  insertShadowCheck(I.getOperand(2), &I);
1967  IRBuilder<> IRB(&I);
1968  setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
1969  I.getOperand(2), "_msprop"));
1970  setOriginForNaryOp(I);
1971  }
1972 
1973  void visitShuffleVectorInst(ShuffleVectorInst &I) {
1974  IRBuilder<> IRB(&I);
1975  setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
1976  I.getShuffleMask(), "_msprop"));
1977  setOriginForNaryOp(I);
1978  }
1979 
1980  // Casts.
1981  void visitSExtInst(SExtInst &I) {
1982  IRBuilder<> IRB(&I);
1983  setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
1984  setOrigin(&I, getOrigin(&I, 0));
1985  }
1986 
1987  void visitZExtInst(ZExtInst &I) {
1988  IRBuilder<> IRB(&I);
1989  setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
1990  setOrigin(&I, getOrigin(&I, 0));
1991  }
1992 
1993  void visitTruncInst(TruncInst &I) {
1994  IRBuilder<> IRB(&I);
1995  setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
1996  setOrigin(&I, getOrigin(&I, 0));
1997  }
1998 
1999  void visitBitCastInst(BitCastInst &I) {
2000  // Special case: if this is the bitcast (there is exactly 1 allowed) between
2001  // a musttail call and a ret, don't instrument. New instructions are not
2002  // allowed after a musttail call.
2003  if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
2004  if (CI->isMustTailCall())
2005  return;
2006  IRBuilder<> IRB(&I);
2007  setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
2008  setOrigin(&I, getOrigin(&I, 0));
2009  }
2010 
2011  void visitPtrToIntInst(PtrToIntInst &I) {
2012  IRBuilder<> IRB(&I);
2013  setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2014  "_msprop_ptrtoint"));
2015  setOrigin(&I, getOrigin(&I, 0));
2016  }
2017 
2018  void visitIntToPtrInst(IntToPtrInst &I) {
2019  IRBuilder<> IRB(&I);
2020  setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2021  "_msprop_inttoptr"));
2022  setOrigin(&I, getOrigin(&I, 0));
2023  }
2024 
2025  void visitFPToSIInst(CastInst& I) { handleShadowOr(I); }
2026  void visitFPToUIInst(CastInst& I) { handleShadowOr(I); }
2027  void visitSIToFPInst(CastInst& I) { handleShadowOr(I); }
2028  void visitUIToFPInst(CastInst& I) { handleShadowOr(I); }
2029  void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
2030  void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
2031 
2032  /// Propagate shadow for bitwise AND.
2033  ///
2034  /// This code is exact, i.e. if, for example, a bit in the left argument
2035  /// is defined and 0, then neither the value not definedness of the
2036  /// corresponding bit in B don't affect the resulting shadow.
2037  void visitAnd(BinaryOperator &I) {
2038  IRBuilder<> IRB(&I);
2039  // "And" of 0 and a poisoned value results in unpoisoned value.
2040  // 1&1 => 1; 0&1 => 0; p&1 => p;
2041  // 1&0 => 0; 0&0 => 0; p&0 => 0;
2042  // 1&p => p; 0&p => 0; p&p => p;
2043  // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
2044  Value *S1 = getShadow(&I, 0);
2045  Value *S2 = getShadow(&I, 1);
2046  Value *V1 = I.getOperand(0);
2047  Value *V2 = I.getOperand(1);
2048  if (V1->getType() != S1->getType()) {
2049  V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2050  V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2051  }
2052  Value *S1S2 = IRB.CreateAnd(S1, S2);
2053  Value *V1S2 = IRB.CreateAnd(V1, S2);
2054  Value *S1V2 = IRB.CreateAnd(S1, V2);
2055  setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2056  setOriginForNaryOp(I);
2057  }
2058 
2059  void visitOr(BinaryOperator &I) {
2060  IRBuilder<> IRB(&I);
2061  // "Or" of 1 and a poisoned value results in unpoisoned value.
2062  // 1|1 => 1; 0|1 => 1; p|1 => 1;
2063  // 1|0 => 1; 0|0 => 0; p|0 => p;
2064  // 1|p => 1; 0|p => p; p|p => p;
2065  // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
2066  Value *S1 = getShadow(&I, 0);
2067  Value *S2 = getShadow(&I, 1);
2068  Value *V1 = IRB.CreateNot(I.getOperand(0));
2069  Value *V2 = IRB.CreateNot(I.getOperand(1));
2070  if (V1->getType() != S1->getType()) {
2071  V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2072  V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2073  }
2074  Value *S1S2 = IRB.CreateAnd(S1, S2);
2075  Value *V1S2 = IRB.CreateAnd(V1, S2);
2076  Value *S1V2 = IRB.CreateAnd(S1, V2);
2077  setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2078  setOriginForNaryOp(I);
2079  }
2080 
2081  /// Default propagation of shadow and/or origin.
2082  ///
2083  /// This class implements the general case of shadow propagation, used in all
2084  /// cases where we don't know and/or don't care about what the operation
2085  /// actually does. It converts all input shadow values to a common type
2086  /// (extending or truncating as necessary), and bitwise OR's them.
2087  ///
2088  /// This is much cheaper than inserting checks (i.e. requiring inputs to be
2089  /// fully initialized), and less prone to false positives.
2090  ///
2091  /// This class also implements the general case of origin propagation. For a
2092  /// Nary operation, result origin is set to the origin of an argument that is
2093  /// not entirely initialized. If there is more than one such arguments, the
2094  /// rightmost of them is picked. It does not matter which one is picked if all
2095  /// arguments are initialized.
2096  template <bool CombineShadow>
2097  class Combiner {
2098  Value *Shadow = nullptr;
2099  Value *Origin = nullptr;
2100  IRBuilder<> &IRB;
2101  MemorySanitizerVisitor *MSV;
2102 
2103  public:
2104  Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2105  : IRB(IRB), MSV(MSV) {}
2106 
2107  /// Add a pair of shadow and origin values to the mix.
2108  Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2109  if (CombineShadow) {
2110  assert(OpShadow);
2111  if (!Shadow)
2112  Shadow = OpShadow;
2113  else {
2114  OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2115  Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
2116  }
2117  }
2118 
2119  if (MSV->MS.TrackOrigins) {
2120  assert(OpOrigin);
2121  if (!Origin) {
2122  Origin = OpOrigin;
2123  } else {
2124  Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2125  // No point in adding something that might result in 0 origin value.
2126  if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2127  Value *FlatShadow = MSV->convertShadowToScalar(OpShadow, IRB);
2128  Value *Cond =
2129  IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
2130  Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2131  }
2132  }
2133  }
2134  return *this;
2135  }
2136 
2137  /// Add an application value to the mix.
2138  Combiner &Add(Value *V) {
2139  Value *OpShadow = MSV->getShadow(V);
2140  Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2141  return Add(OpShadow, OpOrigin);
2142  }
2143 
2144  /// Set the current combined values as the given instruction's shadow
2145  /// and origin.
2146  void Done(Instruction *I) {
2147  if (CombineShadow) {
2148  assert(Shadow);
2149  Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
2150  MSV->setShadow(I, Shadow);
2151  }
2152  if (MSV->MS.TrackOrigins) {
2153  assert(Origin);
2154  MSV->setOrigin(I, Origin);
2155  }
2156  }
2157  };
2158 
2159  using ShadowAndOriginCombiner = Combiner<true>;
2160  using OriginCombiner = Combiner<false>;
2161 
2162  /// Propagate origin for arbitrary operation.
2163  void setOriginForNaryOp(Instruction &I) {
2164  if (!MS.TrackOrigins) return;
2165  IRBuilder<> IRB(&I);
2166  OriginCombiner OC(this, IRB);
2167  for (Use &Op : I.operands())
2168  OC.Add(Op.get());
2169  OC.Done(&I);
2170  }
2171 
2172  size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2173  assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
2174  "Vector of pointers is not a valid shadow type");
2175  return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2176  Ty->getScalarSizeInBits()
2177  : Ty->getPrimitiveSizeInBits();
2178  }
2179 
2180  /// Cast between two shadow types, extending or truncating as
2181  /// necessary.
2182  Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2183  bool Signed = false) {
2184  Type *srcTy = V->getType();
2185  size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2186  size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2187  if (srcSizeInBits > 1 && dstSizeInBits == 1)
2188  return IRB.CreateICmpNE(V, getCleanShadow(V));
2189 
2190  if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2191  return IRB.CreateIntCast(V, dstTy, Signed);
2192  if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2193  cast<FixedVectorType>(dstTy)->getNumElements() ==
2194  cast<FixedVectorType>(srcTy)->getNumElements())
2195  return IRB.CreateIntCast(V, dstTy, Signed);
2196  Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
2197  Value *V2 =
2198  IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
2199  return IRB.CreateBitCast(V2, dstTy);
2200  // TODO: handle struct types.
2201  }
2202 
2203  /// Cast an application value to the type of its own shadow.
2204  Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2205  Type *ShadowTy = getShadowTy(V);
2206  if (V->getType() == ShadowTy)
2207  return V;
2208  if (V->getType()->isPtrOrPtrVectorTy())
2209  return IRB.CreatePtrToInt(V, ShadowTy);
2210  else
2211  return IRB.CreateBitCast(V, ShadowTy);
2212  }
2213 
2214  /// Propagate shadow for arbitrary operation.
2215  void handleShadowOr(Instruction &I) {
2216  IRBuilder<> IRB(&I);
2217  ShadowAndOriginCombiner SC(this, IRB);
2218  for (Use &Op : I.operands())
2219  SC.Add(Op.get());
2220  SC.Done(&I);
2221  }
2222 
2223  void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
2224 
2225  // Handle multiplication by constant.
2226  //
2227  // Handle a special case of multiplication by constant that may have one or
2228  // more zeros in the lower bits. This makes corresponding number of lower bits
2229  // of the result zero as well. We model it by shifting the other operand
2230  // shadow left by the required number of bits. Effectively, we transform
2231  // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
2232  // We use multiplication by 2**N instead of shift to cover the case of
2233  // multiplication by 0, which may occur in some elements of a vector operand.
2234  void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
2235  Value *OtherArg) {
2236  Constant *ShadowMul;
2237  Type *Ty = ConstArg->getType();
2238  if (auto *VTy = dyn_cast<VectorType>(Ty)) {
2239  unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2240  Type *EltTy = VTy->getElementType();
2241  SmallVector<Constant *, 16> Elements;
2242  for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
2243  if (ConstantInt *Elt =
2244  dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
2245  const APInt &V = Elt->getValue();
2246  APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2247  Elements.push_back(ConstantInt::get(EltTy, V2));
2248  } else {
2249  Elements.push_back(ConstantInt::get(EltTy, 1));
2250  }
2251  }
2252  ShadowMul = ConstantVector::get(Elements);
2253  } else {
2254  if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2255  const APInt &V = Elt->getValue();
2256  APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2257  ShadowMul = ConstantInt::get(Ty, V2);
2258  } else {
2259  ShadowMul = ConstantInt::get(Ty, 1);
2260  }
2261  }
2262 
2263  IRBuilder<> IRB(&I);
2264  setShadow(&I,
2265  IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
2266  setOrigin(&I, getOrigin(OtherArg));
2267  }
2268 
2269  void visitMul(BinaryOperator &I) {
2270  Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
2271  Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
2272  if (constOp0 && !constOp1)
2273  handleMulByConstant(I, constOp0, I.getOperand(1));
2274  else if (constOp1 && !constOp0)
2275  handleMulByConstant(I, constOp1, I.getOperand(0));
2276  else
2277  handleShadowOr(I);
2278  }
2279 
2280  void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
2281  void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
2282  void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
2283  void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
2284  void visitSub(BinaryOperator &I) { handleShadowOr(I); }
2285  void visitXor(BinaryOperator &I) { handleShadowOr(I); }
2286 
2287  void handleIntegerDiv(Instruction &I) {
2288  IRBuilder<> IRB(&I);
2289  // Strict on the second argument.
2290  insertShadowCheck(I.getOperand(1), &I);
2291  setShadow(&I, getShadow(&I, 0));
2292  setOrigin(&I, getOrigin(&I, 0));
2293  }
2294 
2295  void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2296  void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2297  void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
2298  void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
2299 
2300  // Floating point division is side-effect free. We can not require that the
2301  // divisor is fully initialized and must propagate shadow. See PR37523.
2302  void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
2303  void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
2304 
2305  /// Instrument == and != comparisons.
2306  ///
2307  /// Sometimes the comparison result is known even if some of the bits of the
2308  /// arguments are not.
2309  void handleEqualityComparison(ICmpInst &I) {
2310  IRBuilder<> IRB(&I);
2311  Value *A = I.getOperand(0);
2312  Value *B = I.getOperand(1);
2313  Value *Sa = getShadow(A);
2314  Value *Sb = getShadow(B);
2315 
2316  // Get rid of pointers and vectors of pointers.
2317  // For ints (and vectors of ints), types of A and Sa match,
2318  // and this is a no-op.
2319  A = IRB.CreatePointerCast(A, Sa->getType());
2320  B = IRB.CreatePointerCast(B, Sb->getType());
2321 
2322  // A == B <==> (C = A^B) == 0
2323  // A != B <==> (C = A^B) != 0
2324  // Sc = Sa | Sb
2325  Value *C = IRB.CreateXor(A, B);
2326  Value *Sc = IRB.CreateOr(Sa, Sb);
2327  // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2328  // Result is defined if one of the following is true
2329  // * there is a defined 1 bit in C
2330  // * C is fully defined
2331  // Si = !(C & ~Sc) && Sc
2332  Value *Zero = Constant::getNullValue(Sc->getType());
2333  Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
2334  Value *Si =
2335  IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero),
2336  IRB.CreateICmpEQ(
2337  IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
2338  Si->setName("_msprop_icmp");
2339  setShadow(&I, Si);
2340  setOriginForNaryOp(I);
2341  }
2342 
2343  /// Build the lowest possible value of V, taking into account V's
2344  /// uninitialized bits.
2345  Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2346  bool isSigned) {
2347  if (isSigned) {
2348  // Split shadow into sign bit and other bits.
2349  Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2350  Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2351  // Maximise the undefined shadow bit, minimize other undefined bits.
2352  return
2353  IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit);
2354  } else {
2355  // Minimize undefined bits.
2356  return IRB.CreateAnd(A, IRB.CreateNot(Sa));
2357  }
2358  }
2359 
2360  /// Build the highest possible value of V, taking into account V's
2361  /// uninitialized bits.
2362  Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2363  bool isSigned) {
2364  if (isSigned) {
2365  // Split shadow into sign bit and other bits.
2366  Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2367  Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2368  // Minimise the undefined shadow bit, maximise other undefined bits.
2369  return
2370  IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits);
2371  } else {
2372  // Maximize undefined bits.
2373  return IRB.CreateOr(A, Sa);
2374  }
2375  }
2376 
2377  /// Instrument relational comparisons.
2378  ///
2379  /// This function does exact shadow propagation for all relational
2380  /// comparisons of integers, pointers and vectors of those.
2381  /// FIXME: output seems suboptimal when one of the operands is a constant
2382  void handleRelationalComparisonExact(ICmpInst &I) {
2383  IRBuilder<> IRB(&I);
2384  Value *A = I.getOperand(0);
2385  Value *B = I.getOperand(1);
2386  Value *Sa = getShadow(A);
2387  Value *Sb = getShadow(B);
2388 
2389  // Get rid of pointers and vectors of pointers.
2390  // For ints (and vectors of ints), types of A and Sa match,
2391  // and this is a no-op.
2392  A = IRB.CreatePointerCast(A, Sa->getType());
2393  B = IRB.CreatePointerCast(B, Sb->getType());
2394 
2395  // Let [a0, a1] be the interval of possible values of A, taking into account
2396  // its undefined bits. Let [b0, b1] be the interval of possible values of B.
2397  // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
2398  bool IsSigned = I.isSigned();
2399  Value *S1 = IRB.CreateICmp(I.getPredicate(),
2400  getLowestPossibleValue(IRB, A, Sa, IsSigned),
2401  getHighestPossibleValue(IRB, B, Sb, IsSigned));
2402  Value *S2 = IRB.CreateICmp(I.getPredicate(),
2403  getHighestPossibleValue(IRB, A, Sa, IsSigned),
2404  getLowestPossibleValue(IRB, B, Sb, IsSigned));
2405  Value *Si = IRB.CreateXor(S1, S2);
2406  setShadow(&I, Si);
2407  setOriginForNaryOp(I);
2408  }
2409 
2410  /// Instrument signed relational comparisons.
2411  ///
2412  /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
2413  /// bit of the shadow. Everything else is delegated to handleShadowOr().
2414  void handleSignedRelationalComparison(ICmpInst &I) {
2415  Constant *constOp;
2416  Value *op = nullptr;
2417  CmpInst::Predicate pre;
2418  if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
2419  op = I.getOperand(0);
2420  pre = I.getPredicate();
2421  } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
2422  op = I.getOperand(1);
2423  pre = I.getSwappedPredicate();
2424  } else {
2425  handleShadowOr(I);
2426  return;
2427  }
2428 
2429  if ((constOp->isNullValue() &&
2430  (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
2431  (constOp->isAllOnesValue() &&
2432  (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
2433  IRBuilder<> IRB(&I);
2434  Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2435  "_msprop_icmp_s");
2436  setShadow(&I, Shadow);
2437  setOrigin(&I, getOrigin(op));
2438  } else {
2439  handleShadowOr(I);
2440  }
2441  }
2442 
2443  void visitICmpInst(ICmpInst &I) {
2444  if (!ClHandleICmp) {
2445  handleShadowOr(I);
2446  return;
2447  }
2448  if (I.isEquality()) {
2449  handleEqualityComparison(I);
2450  return;
2451  }
2452 
2453  assert(I.isRelational());
2454  if (ClHandleICmpExact) {
2455  handleRelationalComparisonExact(I);
2456  return;
2457  }
2458  if (I.isSigned()) {
2459  handleSignedRelationalComparison(I);
2460  return;
2461  }
2462 
2463  assert(I.isUnsigned());
2464  if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
2465  handleRelationalComparisonExact(I);
2466  return;
2467  }
2468 
2469  handleShadowOr(I);
2470  }
2471 
2472  void visitFCmpInst(FCmpInst &I) {
2473  handleShadowOr(I);
2474  }
2475 
2476  void handleShift(BinaryOperator &I) {
2477  IRBuilder<> IRB(&I);
2478  // If any of the S2 bits are poisoned, the whole thing is poisoned.
2479  // Otherwise perform the same shift on S1.
2480  Value *S1 = getShadow(&I, 0);
2481  Value *S2 = getShadow(&I, 1);
2482  Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)),
2483  S2->getType());
2484  Value *V2 = I.getOperand(1);
2485  Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2486  setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2487  setOriginForNaryOp(I);
2488  }
2489 
2490  void visitShl(BinaryOperator &I) { handleShift(I); }
2491  void visitAShr(BinaryOperator &I) { handleShift(I); }
2492  void visitLShr(BinaryOperator &I) { handleShift(I); }
2493 
2494  void handleFunnelShift(IntrinsicInst &I) {
2495  IRBuilder<> IRB(&I);
2496  // If any of the S2 bits are poisoned, the whole thing is poisoned.
2497  // Otherwise perform the same shift on S0 and S1.
2498  Value *S0 = getShadow(&I, 0);
2499  Value *S1 = getShadow(&I, 1);
2500  Value *S2 = getShadow(&I, 2);
2501  Value *S2Conv =
2502  IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
2503  Value *V2 = I.getOperand(2);
2505  I.getModule(), I.getIntrinsicID(), S2Conv->getType());
2506  Value *Shift = IRB.CreateCall(Intrin, {S0, S1, V2});
2507  setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2508  setOriginForNaryOp(I);
2509  }
2510 
2511  /// Instrument llvm.memmove
2512  ///
2513  /// At this point we don't know if llvm.memmove will be inlined or not.
2514  /// If we don't instrument it and it gets inlined,
2515  /// our interceptor will not kick in and we will lose the memmove.
2516  /// If we instrument the call here, but it does not get inlined,
2517  /// we will memove the shadow twice: which is bad in case
2518  /// of overlapping regions. So, we simply lower the intrinsic to a call.
2519  ///
2520  /// Similar situation exists for memcpy and memset.
2521  void visitMemMoveInst(MemMoveInst &I) {
2522  getShadow(I.getArgOperand(1)); // Ensure shadow initialized
2523  IRBuilder<> IRB(&I);
2524  IRB.CreateCall(
2525  MS.MemmoveFn,
2526  {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2527  IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2528  IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2529  I.eraseFromParent();
2530  }
2531 
2532  // Similar to memmove: avoid copying shadow twice.
2533  // This is somewhat unfortunate as it may slowdown small constant memcpys.
2534  // FIXME: consider doing manual inline for small constant sizes and proper
2535  // alignment.
2536  void visitMemCpyInst(MemCpyInst &I) {
2537  getShadow(I.getArgOperand(1)); // Ensure shadow initialized
2538  IRBuilder<> IRB(&I);
2539  IRB.CreateCall(
2540  MS.MemcpyFn,
2541  {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2542  IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2543  IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2544  I.eraseFromParent();
2545  }
2546 
2547  // Same as memcpy.
2548  void visitMemSetInst(MemSetInst &I) {
2549  IRBuilder<> IRB(&I);
2550  IRB.CreateCall(
2551  MS.MemsetFn,
2552  {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2553  IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2554  IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2555  I.eraseFromParent();
2556  }
2557 
2558  void visitVAStartInst(VAStartInst &I) {
2559  VAHelper->visitVAStartInst(I);
2560  }
2561 
2562  void visitVACopyInst(VACopyInst &I) {
2563  VAHelper->visitVACopyInst(I);
2564  }
2565 
2566  /// Handle vector store-like intrinsics.
2567  ///
2568  /// Instrument intrinsics that look like a simple SIMD store: writes memory,
2569  /// has 1 pointer argument and 1 vector argument, returns void.
2570  bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
2571  IRBuilder<> IRB(&I);
2572  Value* Addr = I.getArgOperand(0);
2573  Value *Shadow = getShadow(&I, 1);
2574  Value *ShadowPtr, *OriginPtr;
2575 
2576  // We don't know the pointer alignment (could be unaligned SSE store!).
2577  // Have to assume to worst case.
2578  std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2579  Addr, IRB, Shadow->getType(), Align(1), /*isStore*/ true);
2580  IRB.CreateAlignedStore(Shadow, ShadowPtr, Align(1));
2581 
2583  insertShadowCheck(Addr, &I);
2584 
2585  // FIXME: factor out common code from materializeStores
2586  if (MS.TrackOrigins) IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2587  return true;
2588  }
2589 
2590  /// Handle vector load-like intrinsics.
2591  ///
2592  /// Instrument intrinsics that look like a simple SIMD load: reads memory,
2593  /// has 1 pointer argument, returns a vector.
2594  bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
2595  IRBuilder<> IRB(&I);
2596  Value *Addr = I.getArgOperand(0);
2597 
2598  Type *ShadowTy = getShadowTy(&I);
2599  Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2600  if (PropagateShadow) {
2601  // We don't know the pointer alignment (could be unaligned SSE load!).
2602  // Have to assume to worst case.
2603  const Align Alignment = Align(1);
2604  std::tie(ShadowPtr, OriginPtr) =
2605  getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2606  setShadow(&I,
2607  IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
2608  } else {
2609  setShadow(&I, getCleanShadow(&I));
2610  }
2611 
2613  insertShadowCheck(Addr, &I);
2614 
2615  if (MS.TrackOrigins) {
2616  if (PropagateShadow)
2617  setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
2618  else
2619  setOrigin(&I, getCleanOrigin());
2620  }
2621  return true;
2622  }
2623 
2624  /// Handle (SIMD arithmetic)-like intrinsics.
2625  ///
2626  /// Instrument intrinsics with any number of arguments of the same type,
2627  /// equal to the return type. The type should be simple (no aggregates or
2628  /// pointers; vectors are fine).
2629  /// Caller guarantees that this intrinsic does not access memory.
2630  bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
2631  Type *RetTy = I.getType();
2632  if (!(RetTy->isIntOrIntVectorTy() ||
2633  RetTy->isFPOrFPVectorTy() ||
2634  RetTy->isX86_MMXTy()))
2635  return false;
2636 
2637  unsigned NumArgOperands = I.arg_size();
2638  for (unsigned i = 0; i < NumArgOperands; ++i) {
2639  Type *Ty = I.getArgOperand(i)->getType();
2640  if (Ty != RetTy)
2641  return false;
2642  }
2643 
2644  IRBuilder<> IRB(&I);
2645  ShadowAndOriginCombiner SC(this, IRB);
2646  for (unsigned i = 0; i < NumArgOperands; ++i)
2647  SC.Add(I.getArgOperand(i));
2648  SC.Done(&I);
2649 
2650  return true;
2651  }
2652 
2653  /// Heuristically instrument unknown intrinsics.
2654  ///
2655  /// The main purpose of this code is to do something reasonable with all
2656  /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2657  /// We recognize several classes of intrinsics by their argument types and
2658  /// ModRefBehaviour and apply special instrumentation when we are reasonably
2659  /// sure that we know what the intrinsic does.
2660  ///
2661  /// We special-case intrinsics where this approach fails. See llvm.bswap
2662  /// handling as an example of that.
2663  bool handleUnknownIntrinsic(IntrinsicInst &I) {
2664  unsigned NumArgOperands = I.arg_size();
2665  if (NumArgOperands == 0)
2666  return false;
2667 
2668  if (NumArgOperands == 2 &&
2669  I.getArgOperand(0)->getType()->isPointerTy() &&
2670  I.getArgOperand(1)->getType()->isVectorTy() &&
2671  I.getType()->isVoidTy() &&
2672  !I.onlyReadsMemory()) {
2673  // This looks like a vector store.
2674  return handleVectorStoreIntrinsic(I);
2675  }
2676 
2677  if (NumArgOperands == 1 &&
2678  I.getArgOperand(0)->getType()->isPointerTy() &&
2679  I.getType()->isVectorTy() &&
2680  I.onlyReadsMemory()) {
2681  // This looks like a vector load.
2682  return handleVectorLoadIntrinsic(I);
2683  }
2684 
2685  if (I.doesNotAccessMemory())
2686  if (maybeHandleSimpleNomemIntrinsic(I))
2687  return true;
2688 
2689  // FIXME: detect and handle SSE maskstore/maskload
2690  return false;
2691  }
2692 
2693  void handleInvariantGroup(IntrinsicInst &I) {
2694  setShadow(&I, getShadow(&I, 0));
2695  setOrigin(&I, getOrigin(&I, 0));
2696  }
2697 
2698  void handleLifetimeStart(IntrinsicInst &I) {
2699  if (!PoisonStack)
2700  return;
2701  AllocaInst *AI = llvm::findAllocaForValue(I.getArgOperand(1));
2702  if (!AI)
2703  InstrumentLifetimeStart = false;
2704  LifetimeStartList.push_back(std::make_pair(&I, AI));
2705  }
2706 
2707  void handleBswap(IntrinsicInst &I) {
2708  IRBuilder<> IRB(&I);
2709  Value *Op = I.getArgOperand(0);
2710  Type *OpType = Op->getType();
2711  Function *BswapFunc = Intrinsic::getDeclaration(
2712  F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2713  setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2714  setOrigin(&I, getOrigin(Op));
2715  }
2716 
2717  // Instrument vector convert intrinsic.
2718  //
2719  // This function instruments intrinsics like cvtsi2ss:
2720  // %Out = int_xxx_cvtyyy(%ConvertOp)
2721  // or
2722  // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2723  // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2724  // number \p Out elements, and (if has 2 arguments) copies the rest of the
2725  // elements from \p CopyOp.
2726  // In most cases conversion involves floating-point value which may trigger a
2727  // hardware exception when not fully initialized. For this reason we require
2728  // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2729  // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2730  // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2731  // return a fully initialized value.
2732  void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
2733  bool HasRoundingMode = false) {
2734  IRBuilder<> IRB(&I);
2735  Value *CopyOp, *ConvertOp;
2736 
2737  assert((!HasRoundingMode ||
2738  isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) &&
2739  "Invalid rounding mode");
2740 
2741  switch (I.arg_size() - HasRoundingMode) {
2742  case 2:
2743  CopyOp = I.getArgOperand(0);
2744  ConvertOp = I.getArgOperand(1);
2745  break;
2746  case 1:
2747  ConvertOp = I.getArgOperand(0);
2748  CopyOp = nullptr;
2749  break;
2750  default:
2751  llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
2752  }
2753 
2754  // The first *NumUsedElements* elements of ConvertOp are converted to the
2755  // same number of output elements. The rest of the output is copied from
2756  // CopyOp, or (if not available) filled with zeroes.
2757  // Combine shadow for elements of ConvertOp that are used in this operation,
2758  // and insert a check.
2759  // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2760  // int->any conversion.
2761  Value *ConvertShadow = getShadow(ConvertOp);
2762  Value *AggShadow = nullptr;
2763  if (ConvertOp->getType()->isVectorTy()) {
2764  AggShadow = IRB.CreateExtractElement(
2765  ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2766  for (int i = 1; i < NumUsedElements; ++i) {
2767  Value *MoreShadow = IRB.CreateExtractElement(
2768  ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2769  AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2770  }
2771  } else {
2772  AggShadow = ConvertShadow;
2773  }
2774  assert(AggShadow->getType()->isIntegerTy());
2775  insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2776 
2777  // Build result shadow by zero-filling parts of CopyOp shadow that come from
2778  // ConvertOp.
2779  if (CopyOp) {
2780  assert(CopyOp->getType() == I.getType());
2781  assert(CopyOp->getType()->isVectorTy());
2782  Value *ResultShadow = getShadow(CopyOp);
2783  Type *EltTy = cast<VectorType>(ResultShadow->getType())->getElementType();
2784  for (int i = 0; i < NumUsedElements; ++i) {
2785  ResultShadow = IRB.CreateInsertElement(
2786  ResultShadow, ConstantInt::getNullValue(EltTy),
2787  ConstantInt::get(IRB.getInt32Ty(), i));
2788  }
2789  setShadow(&I, ResultShadow);
2790  setOrigin(&I, getOrigin(CopyOp));
2791  } else {
2792  setShadow(&I, getCleanShadow(&I));
2793  setOrigin(&I, getCleanOrigin());
2794  }
2795  }
2796 
2797  // Given a scalar or vector, extract lower 64 bits (or less), and return all
2798  // zeroes if it is zero, and all ones otherwise.
2799  Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2800  if (S->getType()->isVectorTy())
2801  S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2802  assert(S->getType()->getPrimitiveSizeInBits() <= 64);
2803  Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2804  return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2805  }
2806 
2807  // Given a vector, extract its first element, and return all
2808  // zeroes if it is zero, and all ones otherwise.
2809  Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2810  Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2811  Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2812  return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2813  }
2814 
2815  Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2816  Type *T = S->getType();
2817  assert(T->isVectorTy());
2818  Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2819  return IRB.CreateSExt(S2, T);
2820  }
2821 
2822  // Instrument vector shift intrinsic.
2823  //
2824  // This function instruments intrinsics like int_x86_avx2_psll_w.
2825  // Intrinsic shifts %In by %ShiftSize bits.
2826  // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2827  // size, and the rest is ignored. Behavior is defined even if shift size is
2828  // greater than register (or field) width.
2829  void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2830  assert(I.arg_size() == 2);
2831  IRBuilder<> IRB(&I);
2832  // If any of the S2 bits are poisoned, the whole thing is poisoned.
2833  // Otherwise perform the same shift on S1.
2834  Value *S1 = getShadow(&I, 0);
2835  Value *S2 = getShadow(&I, 1);
2836  Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2837  : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2838  Value *V1 = I.getOperand(0);
2839  Value *V2 = I.getOperand(1);
2840  Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(),
2841  {IRB.CreateBitCast(S1, V1->getType()), V2});
2842  Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2843  setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2844  setOriginForNaryOp(I);
2845  }
2846 
2847  // Get an X86_MMX-sized vector type.
2848  Type *getMMXVectorTy(unsigned EltSizeInBits) {
2849  const unsigned X86_MMXSizeInBits = 64;
2850  assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
2851  "Illegal MMX vector element size");
2852  return FixedVectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2853  X86_MMXSizeInBits / EltSizeInBits);
2854  }
2855 
2856  // Returns a signed counterpart for an (un)signed-saturate-and-pack
2857  // intrinsic.
2858  Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2859  switch (id) {
2860  case Intrinsic::x86_sse2_packsswb_128:
2861  case Intrinsic::x86_sse2_packuswb_128:
2862  return Intrinsic::x86_sse2_packsswb_128;
2863 
2864  case Intrinsic::x86_sse2_packssdw_128:
2865  case Intrinsic::x86_sse41_packusdw:
2866  return Intrinsic::x86_sse2_packssdw_128;
2867 
2868  case Intrinsic::x86_avx2_packsswb:
2869  case Intrinsic::x86_avx2_packuswb:
2870  return Intrinsic::x86_avx2_packsswb;
2871 
2872  case Intrinsic::x86_avx2_packssdw:
2873  case Intrinsic::x86_avx2_packusdw:
2874  return Intrinsic::x86_avx2_packssdw;
2875 
2876  case Intrinsic::x86_mmx_packsswb:
2877  case Intrinsic::x86_mmx_packuswb:
2878  return Intrinsic::x86_mmx_packsswb;
2879 
2880  case Intrinsic::x86_mmx_packssdw:
2881  return Intrinsic::x86_mmx_packssdw;
2882  default:
2883  llvm_unreachable("unexpected intrinsic id");
2884  }
2885  }
2886 
2887  // Instrument vector pack intrinsic.
2888  //
2889  // This function instruments intrinsics like x86_mmx_packsswb, that
2890  // packs elements of 2 input vectors into half as many bits with saturation.
2891  // Shadow is propagated with the signed variant of the same intrinsic applied
2892  // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
2893  // EltSizeInBits is used only for x86mmx arguments.
2894  void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
2895  assert(I.arg_size() == 2);
2896  bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2897  IRBuilder<> IRB(&I);
2898  Value *S1 = getShadow(&I, 0);
2899  Value *S2 = getShadow(&I, 1);
2900  assert(isX86_MMX || S1->getType()->isVectorTy());
2901 
2902  // SExt and ICmpNE below must apply to individual elements of input vectors.
2903  // In case of x86mmx arguments, cast them to appropriate vector types and
2904  // back.
2905  Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
2906  if (isX86_MMX) {
2907  S1 = IRB.CreateBitCast(S1, T);
2908  S2 = IRB.CreateBitCast(S2, T);
2909  }
2910  Value *S1_ext = IRB.CreateSExt(
2911  IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
2912  Value *S2_ext = IRB.CreateSExt(
2913  IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
2914  if (isX86_MMX) {
2915  Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
2916  S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
2917  S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
2918  }
2919 
2920  Function *ShadowFn = Intrinsic::getDeclaration(
2921  F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
2922 
2923  Value *S =
2924  IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
2925  if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I));
2926  setShadow(&I, S);
2927  setOriginForNaryOp(I);
2928  }
2929 
2930  // Instrument sum-of-absolute-differences intrinsic.
2931  void handleVectorSadIntrinsic(IntrinsicInst &I) {
2932  const unsigned SignificantBitsPerResultElement = 16;
2933  bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2934  Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
2935  unsigned ZeroBitsPerResultElement =
2936  ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
2937 
2938  IRBuilder<> IRB(&I);
2939  Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2940  S = IRB.CreateBitCast(S, ResTy);
2941  S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2942  ResTy);
2943  S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
2944  S = IRB.CreateBitCast(S, getShadowTy(&I));
2945  setShadow(&I, S);
2946  setOriginForNaryOp(I);
2947  }
2948 
2949  // Instrument multiply-add intrinsic.
2950  void handleVectorPmaddIntrinsic(IntrinsicInst &I,
2951  unsigned EltSizeInBits = 0) {
2952  bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2953  Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
2954  IRBuilder<> IRB(&I);
2955  Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2956  S = IRB.CreateBitCast(S, ResTy);
2957  S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2958  ResTy);
2959  S = IRB.CreateBitCast(S, getShadowTy(&I));
2960  setShadow(&I, S);
2961  setOriginForNaryOp(I);
2962  }
2963 
2964  // Instrument compare-packed intrinsic.
2965  // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
2966  // all-ones shadow.
2967  void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
2968  IRBuilder<> IRB(&I);
2969  Type *ResTy = getShadowTy(&I);
2970  Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2971  Value *S = IRB.CreateSExt(
2972  IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
2973  setShadow(&I, S);
2974  setOriginForNaryOp(I);
2975  }
2976 
2977  // Instrument compare-scalar intrinsic.
2978  // This handles both cmp* intrinsics which return the result in the first
2979  // element of a vector, and comi* which return the result as i32.
2980  void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
2981  IRBuilder<> IRB(&I);
2982  Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2983  Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
2984  setShadow(&I, S);
2985  setOriginForNaryOp(I);
2986  }
2987 
2988  // Instrument generic vector reduction intrinsics
2989  // by ORing together all their fields.
2990  void handleVectorReduceIntrinsic(IntrinsicInst &I) {
2991  IRBuilder<> IRB(&I);
2992  Value *S = IRB.CreateOrReduce(getShadow(&I, 0));
2993  setShadow(&I, S);
2994  setOrigin(&I, getOrigin(&I, 0));
2995  }
2996 
2997  // Instrument vector.reduce.or intrinsic.
2998  // Valid (non-poisoned) set bits in the operand pull low the
2999  // corresponding shadow bits.
3000  void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
3001  IRBuilder<> IRB(&I);
3002  Value *OperandShadow = getShadow(&I, 0);
3003  Value *OperandUnsetBits = IRB.CreateNot(I.getOperand(0));
3004  Value *OperandUnsetOrPoison = IRB.CreateOr(OperandUnsetBits, OperandShadow);
3005  // Bit N is clean if any field's bit N is 1 and unpoison
3006  Value *OutShadowMask = IRB.CreateAndReduce(OperandUnsetOrPoison);
3007  // Otherwise, it is clean if every field's bit N is unpoison
3008  Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3009  Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3010 
3011  setShadow(&I, S);
3012  setOrigin(&I, getOrigin(&I, 0));
3013  }
3014 
3015  // Instrument vector.reduce.and intrinsic.
3016  // Valid (non-poisoned) unset bits in the operand pull down the
3017  // corresponding shadow bits.
3018  void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
3019  IRBuilder<> IRB(&I);
3020  Value *OperandShadow = getShadow(&I, 0);
3021  Value *OperandSetOrPoison = IRB.CreateOr(I.getOperand(0), OperandShadow);
3022  // Bit N is clean if any field's bit N is 0 and unpoison
3023  Value *OutShadowMask = IRB.CreateAndReduce(OperandSetOrPoison);
3024  // Otherwise, it is clean if every field's bit N is unpoison
3025  Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3026  Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3027 
3028  setShadow(&I, S);
3029  setOrigin(&I, getOrigin(&I, 0));
3030  }
3031 
3032  void handleStmxcsr(IntrinsicInst &I) {
3033  IRBuilder<> IRB(&I);
3034  Value* Addr = I.getArgOperand(0);
3035  Type *Ty = IRB.getInt32Ty();
3036  Value *ShadowPtr =
3037  getShadowOriginPtr(Addr, IRB, Ty, Align(1), /*isStore*/ true).first;
3038 
3039  IRB.CreateStore(getCleanShadow(Ty),
3040  IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
3041 
3043  insertShadowCheck(Addr, &I);
3044  }
3045 
3046  void handleLdmxcsr(IntrinsicInst &I) {
3047  if (!InsertChecks) return;
3048 
3049  IRBuilder<> IRB(&I);
3050  Value *Addr = I.getArgOperand(0);
3051  Type *Ty = IRB.getInt32Ty();
3052  const Align Alignment = Align(1);
3053  Value *ShadowPtr, *OriginPtr;
3054  std::tie(ShadowPtr, OriginPtr) =
3055  getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
3056 
3058  insertShadowCheck(Addr, &I);
3059 
3060  Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
3061  Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
3062  : getCleanOrigin();
3063  insertShadowCheck(Shadow, Origin, &I);
3064  }
3065 
3066  void handleMaskedStore(IntrinsicInst &I) {
3067  IRBuilder<> IRB(&I);
3068  Value *V = I.getArgOperand(0);
3069  Value *Addr = I.getArgOperand(1);
3070  const Align Alignment(
3071  cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
3072  Value *Mask = I.getArgOperand(3);
3073  Value *Shadow = getShadow(V);
3074 
3075  Value *ShadowPtr;
3076  Value *OriginPtr;
3077  std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3078  Addr, IRB, Shadow->getType(), Alignment, /*isStore*/ true);
3079 
3080  if (ClCheckAccessAddress) {
3081  insertShadowCheck(Addr, &I);
3082  // Uninitialized mask is kind of like uninitialized address, but not as
3083  // scary.
3084  insertShadowCheck(Mask, &I);
3085  }
3086 
3087  IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment, Mask);
3088 
3089  if (MS.TrackOrigins) {
3090  auto &DL = F.getParent()->getDataLayout();
3091  paintOrigin(IRB, getOrigin(V), OriginPtr,
3092  DL.getTypeStoreSize(Shadow->getType()),
3093  std::max(Alignment, kMinOriginAlignment));
3094  }
3095  }
3096 
3097  bool handleMaskedLoad(IntrinsicInst &I) {
3098  IRBuilder<> IRB(&I);
3099  Value *Addr = I.getArgOperand(0);
3100  const Align Alignment(
3101  cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
3102  Value *Mask = I.getArgOperand(2);
3103  Value *PassThru = I.getArgOperand(3);
3104 
3105  Type *ShadowTy = getShadowTy(&I);
3106  Value *ShadowPtr, *OriginPtr;
3107  if (PropagateShadow) {
3108  std::tie(ShadowPtr, OriginPtr) =
3109  getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3110  setShadow(&I, IRB.CreateMaskedLoad(ShadowTy, ShadowPtr, Alignment, Mask,
3111  getShadow(PassThru), "_msmaskedld"));
3112  } else {
3113  setShadow(&I, getCleanShadow(&I));
3114  }
3115 
3116  if (ClCheckAccessAddress) {
3117  insertShadowCheck(Addr, &I);
3118  insertShadowCheck(Mask, &I);
3119  }
3120 
3121  if (MS.TrackOrigins) {
3122  if (PropagateShadow) {
3123  // Choose between PassThru's and the loaded value's origins.
3124  Value *MaskedPassThruShadow = IRB.CreateAnd(
3125  getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
3126 
3127  Value *Acc = IRB.CreateExtractElement(
3128  MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
3129  for (int i = 1, N = cast<FixedVectorType>(PassThru->getType())
3130  ->getNumElements();
3131  i < N; ++i) {
3132  Value *More = IRB.CreateExtractElement(
3133  MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
3134  Acc = IRB.CreateOr(Acc, More);
3135  }
3136 
3137  Value *Origin = IRB.CreateSelect(
3138  IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
3139  getOrigin(PassThru), IRB.CreateLoad(MS.OriginTy, OriginPtr));
3140 
3141  setOrigin(&I, Origin);
3142  } else {
3143  setOrigin(&I, getCleanOrigin());
3144  }
3145  }
3146  return true;
3147  }
3148 
3149  // Instrument BMI / BMI2 intrinsics.
3150  // All of these intrinsics are Z = I(X, Y)
3151  // where the types of all operands and the result match, and are either i32 or i64.
3152  // The following instrumentation happens to work for all of them:
3153  // Sz = I(Sx, Y) | (sext (Sy != 0))
3154  void handleBmiIntrinsic(IntrinsicInst &I) {
3155  IRBuilder<> IRB(&I);
3156  Type *ShadowTy = getShadowTy(&I);
3157 
3158  // If any bit of the mask operand is poisoned, then the whole thing is.
3159  Value *SMask = getShadow(&I, 1);
3160  SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)),
3161  ShadowTy);
3162  // Apply the same intrinsic to the shadow of the first operand.
3163  Value *S = IRB.CreateCall(I.getCalledFunction(),
3164  {getShadow(&I, 0), I.getOperand(1)});
3165  S = IRB.CreateOr(SMask, S);
3166  setShadow(&I, S);
3167  setOriginForNaryOp(I);
3168  }
3169 
3170  SmallVector<int, 8> getPclmulMask(unsigned Width, bool OddElements) {
3172  for (unsigned X = OddElements ? 1 : 0; X < Width; X += 2) {
3173  Mask.append(2, X);
3174  }
3175  return Mask;
3176  }
3177 
3178  // Instrument pclmul intrinsics.
3179  // These intrinsics operate either on odd or on even elements of the input
3180  // vectors, depending on the constant in the 3rd argument, ignoring the rest.
3181  // Replace the unused elements with copies of the used ones, ex:
3182  // (0, 1, 2, 3) -> (0, 0, 2, 2) (even case)
3183  // or
3184  // (0, 1, 2, 3) -> (1, 1, 3, 3) (odd case)
3185  // and then apply the usual shadow combining logic.
3186  void handlePclmulIntrinsic(IntrinsicInst &I) {
3187  IRBuilder<> IRB(&I);
3188  unsigned Width =
3189  cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
3190  assert(isa<ConstantInt>(I.getArgOperand(2)) &&
3191  "pclmul 3rd operand must be a constant");
3192  unsigned Imm = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3193  Value *Shuf0 = IRB.CreateShuffleVector(getShadow(&I, 0),
3194  getPclmulMask(Width, Imm & 0x01));
3195  Value *Shuf1 = IRB.CreateShuffleVector(getShadow(&I, 1),
3196  getPclmulMask(Width, Imm & 0x10));
3197  ShadowAndOriginCombiner SOC(this, IRB);
3198  SOC.Add(Shuf0, getOrigin(&I, 0));
3199  SOC.Add(Shuf1, getOrigin(&I, 1));
3200  SOC.Done(&I);
3201  }
3202 
3203  // Instrument _mm_*_sd intrinsics
3204  void handleUnarySdIntrinsic(IntrinsicInst &I) {
3205  IRBuilder<> IRB(&I);
3206  Value *First = getShadow(&I, 0);
3207  Value *Second = getShadow(&I, 1);
3208  // High word of first operand, low word of second
3209  Value *Shadow =
3210  IRB.CreateShuffleVector(First, Second, llvm::makeArrayRef<int>({2, 1}));
3211 
3212  setShadow(&I, Shadow);
3213  setOriginForNaryOp(I);
3214  }
3215 
3216  void handleBinarySdIntrinsic(IntrinsicInst &I) {
3217  IRBuilder<> IRB(&I);
3218  Value *First = getShadow(&I, 0);
3219  Value *Second = getShadow(&I, 1);
3220  Value *OrShadow = IRB.CreateOr(First, Second);
3221  // High word of first operand, low word of both OR'd together
3222  Value *Shadow = IRB.CreateShuffleVector(First, OrShadow,
3223  llvm::makeArrayRef<int>({2, 1}));
3224 
3225  setShadow(&I, Shadow);
3226  setOriginForNaryOp(I);
3227  }
3228 
3229  // Instrument abs intrinsic.
3230  // handleUnknownIntrinsic can't handle it because of the last
3231  // is_int_min_poison argument which does not match the result type.
3232  void handleAbsIntrinsic(IntrinsicInst &I) {
3233  assert(I.getType()->isIntOrIntVectorTy());
3234  assert(I.getArgOperand(0)->getType() == I.getType());
3235 
3236  // FIXME: Handle is_int_min_poison.
3237  IRBuilder<> IRB(&I);
3238  setShadow(&I, getShadow(&I, 0));
3239  setOrigin(&I, getOrigin(&I, 0));
3240  }
3241 
3242  void visitIntrinsicInst(IntrinsicInst &I) {
3243  switch (I.getIntrinsicID()) {
3244  case Intrinsic::abs:
3245  handleAbsIntrinsic(I);
3246  break;
3247  case Intrinsic::lifetime_start:
3248  handleLifetimeStart(I);
3249  break;
3250  case Intrinsic::launder_invariant_group:
3251  case Intrinsic::strip_invariant_group:
3252  handleInvariantGroup(I);
3253  break;
3254  case Intrinsic::bswap:
3255  handleBswap(I);
3256  break;
3257  case Intrinsic::masked_store:
3258  handleMaskedStore(I);
3259  break;
3260  case Intrinsic::masked_load:
3261  handleMaskedLoad(I);
3262  break;
3263  case Intrinsic::vector_reduce_and:
3264  handleVectorReduceAndIntrinsic(I);
3265  break;
3266  case Intrinsic::vector_reduce_or:
3267  handleVectorReduceOrIntrinsic(I);
3268  break;
3269  case Intrinsic::vector_reduce_add:
3270  case Intrinsic::vector_reduce_xor:
3271  case Intrinsic::vector_reduce_mul:
3272  handleVectorReduceIntrinsic(I);
3273  break;
3274  case Intrinsic::x86_sse_stmxcsr:
3275  handleStmxcsr(I);
3276  break;
3277  case Intrinsic::x86_sse_ldmxcsr:
3278  handleLdmxcsr(I);
3279  break;
3280  case Intrinsic::x86_avx512_vcvtsd2usi64:
3281  case Intrinsic::x86_avx512_vcvtsd2usi32:
3282  case Intrinsic::x86_avx512_vcvtss2usi64:
3283  case Intrinsic::x86_avx512_vcvtss2usi32:
3284  case Intrinsic::x86_avx512_cvttss2usi64:
3285  case Intrinsic::x86_avx512_cvttss2usi:
3286  case Intrinsic::x86_avx512_cvttsd2usi64:
3287  case Intrinsic::x86_avx512_cvttsd2usi:
3288  case Intrinsic::x86_avx512_cvtusi2ss:
3289  case Intrinsic::x86_avx512_cvtusi642sd:
3290  case Intrinsic::x86_avx512_cvtusi642ss:
3291  handleVectorConvertIntrinsic(I, 1, true);
3292  break;
3293  case Intrinsic::x86_sse2_cvtsd2si64:
3294  case Intrinsic::x86_sse2_cvtsd2si:
3295  case Intrinsic::x86_sse2_cvtsd2ss:
3296  case Intrinsic::x86_sse2_cvttsd2si64:
3297  case Intrinsic::x86_sse2_cvttsd2si:
3298  case Intrinsic::x86_sse_cvtss2si64:
3299  case Intrinsic::x86_sse_cvtss2si:
3300  case Intrinsic::x86_sse_cvttss2si64:
3301  case Intrinsic::x86_sse_cvttss2si:
3302  handleVectorConvertIntrinsic(I, 1);
3303  break;
3304  case Intrinsic::x86_sse_cvtps2pi:
3305  case Intrinsic::x86_sse_cvttps2pi:
3306  handleVectorConvertIntrinsic(I, 2);
3307  break;
3308 
3309  case Intrinsic::x86_avx512_psll_w_512:
3310  case Intrinsic::x86_avx512_psll_d_512:
3311  case Intrinsic::x86_avx512_psll_q_512:
3312  case Intrinsic::x86_avx512_pslli_w_512:
3313  case Intrinsic::x86_avx512_pslli_d_512:
3314  case Intrinsic::x86_avx512_pslli_q_512:
3315  case Intrinsic::x86_avx512_psrl_w_512:
3316  case Intrinsic::x86_avx512_psrl_d_512:
3317  case Intrinsic::x86_avx512_psrl_q_512:
3318  case Intrinsic::x86_avx512_psra_w_512:
3319  case Intrinsic::x86_avx512_psra_d_512:
3320  case Intrinsic::x86_avx512_psra_q_512:
3321  case Intrinsic::x86_avx512_psrli_w_512:
3322  case Intrinsic::x86_avx512_psrli_d_512:
3323  case Intrinsic::x86_avx512_psrli_q_512:
3324  case Intrinsic::x86_avx512_psrai_w_512:
3325  case Intrinsic::x86_avx512_psrai_d_512:
3326  case Intrinsic::x86_avx512_psrai_q_512:
3327  case Intrinsic::x86_avx512_psra_q_256:
3328  case Intrinsic::x86_avx512_psra_q_128:
3329  case Intrinsic::x86_avx512_psrai_q_256:
3330  case Intrinsic::x86_avx512_psrai_q_128:
3331  case Intrinsic::x86_avx2_psll_w:
3332  case Intrinsic::x86_avx2_psll_d:
3333  case Intrinsic::x86_avx2_psll_q:
3334  case Intrinsic::x86_avx2_pslli_w:
3335  case Intrinsic::x86_avx2_pslli_d:
3336  case Intrinsic::x86_avx2_pslli_q:
3337  case Intrinsic::x86_avx2_psrl_w:
3338  case Intrinsic::x86_avx2_psrl_d:
3339  case Intrinsic::x86_avx2_psrl_q:
3340  case Intrinsic::x86_avx2_psra_w:
3341  case Intrinsic::x86_avx2_psra_d:
3342  case Intrinsic::x86_avx2_psrli_w:
3343  case Intrinsic::x86_avx2_psrli_d:
3344  case Intrinsic::x86_avx2_psrli_q:
3345  case Intrinsic::x86_avx2_psrai_w:
3346  case Intrinsic::x86_avx2_psrai_d:
3347  case Intrinsic::x86_sse2_psll_w:
3348  case Intrinsic::x86_sse2_psll_d:
3349  case Intrinsic::x86_sse2_psll_q:
3350  case Intrinsic::x86_sse2_pslli_w:
3351  case Intrinsic::x86_sse2_pslli_d:
3352  case Intrinsic::x86_sse2_pslli_q:
3353  case Intrinsic::x86_sse2_psrl_w:
3354  case Intrinsic::x86_sse2_psrl_d:
3355  case Intrinsic::x86_sse2_psrl_q:
3356  case Intrinsic::x86_sse2_psra_w:
3357  case Intrinsic::x86_sse2_psra_d:
3358  case Intrinsic::x86_sse2_psrli_w:
3359  case Intrinsic::x86_sse2_psrli_d:
3360  case Intrinsic::x86_sse2_psrli_q:
3361  case Intrinsic::x86_sse2_psrai_w:
3362  case Intrinsic::x86_sse2_psrai_d:
3363  case Intrinsic::x86_mmx_psll_w:
3364  case Intrinsic::x86_mmx_psll_d:
3365  case Intrinsic::x86_mmx_psll_q:
3366  case Intrinsic::x86_mmx_pslli_w:
3367  case Intrinsic::x86_mmx_pslli_d:
3368  case Intrinsic::x86_mmx_pslli_q:
3369  case Intrinsic::x86_mmx_psrl_w:
3370  case Intrinsic::x86_mmx_psrl_d:
3371  case Intrinsic::x86_mmx_psrl_q:
3372  case Intrinsic::x86_mmx_psra_w:
3373  case Intrinsic::x86_mmx_psra_d:
3374  case Intrinsic::x86_mmx_psrli_w:
3375  case Intrinsic::x86_mmx_psrli_d:
3376  case Intrinsic::x86_mmx_psrli_q:
3377  case Intrinsic::x86_mmx_psrai_w:
3378  case Intrinsic::x86_mmx_psrai_d:
3379  handleVectorShiftIntrinsic(I, /* Variable */ false);
3380  break;
3381  case Intrinsic::x86_avx2_psllv_d:
3382  case Intrinsic::x86_avx2_psllv_d_256:
3383  case Intrinsic::x86_avx512_psllv_d_512:
3384  case Intrinsic::x86_avx2_psllv_q:
3385  case Intrinsic::x86_avx2_psllv_q_256:
3386  case Intrinsic::x86_avx512_psllv_q_512:
3387  case Intrinsic::x86_avx2_psrlv_d:
3388  case Intrinsic::x86_avx2_psrlv_d_256:
3389  case Intrinsic::x86_avx512_psrlv_d_512:
3390  case Intrinsic::x86_avx2_psrlv_q:
3391  case Intrinsic::x86_avx2_psrlv_q_256:
3392  case Intrinsic::x86_avx512_psrlv_q_512:
3393  case Intrinsic::x86_avx2_psrav_d:
3394  case Intrinsic::x86_avx2_psrav_d_256:
3395  case Intrinsic::x86_avx512_psrav_d_512:
3396  case Intrinsic::x86_avx512_psrav_q_128:
3397  case Intrinsic::x86_avx512_psrav_q_256:
3398  case Intrinsic::x86_avx512_psrav_q_512:
3399  handleVectorShiftIntrinsic(I, /* Variable */ true);
3400  break;
3401 
3402  case Intrinsic::x86_sse2_packsswb_128:
3403  case Intrinsic::x86_sse2_packssdw_128:
3404  case Intrinsic::x86_sse2_packuswb_128:
3405  case Intrinsic::x86_sse41_packusdw:
3406  case Intrinsic::x86_avx2_packsswb:
3407  case Intrinsic::x86_avx2_packssdw:
3408  case Intrinsic::x86_avx2_packuswb:
3409  case Intrinsic::x86_avx2_packusdw:
3410  handleVectorPackIntrinsic(I);
3411  break;
3412 
3413  case Intrinsic::x86_mmx_packsswb:
3414  case Intrinsic::x86_mmx_packuswb:
3415  handleVectorPackIntrinsic(I, 16);
3416  break;
3417 
3418  case Intrinsic::x86_mmx_packssdw:
3419  handleVectorPackIntrinsic(I, 32);
3420  break;
3421 
3422  case Intrinsic::x86_mmx_psad_bw:
3423  case Intrinsic::x86_sse2_psad_bw:
3424  case Intrinsic::x86_avx2_psad_bw:
3425  handleVectorSadIntrinsic(I);
3426  break;
3427 
3428  case Intrinsic::x86_sse2_pmadd_wd:
3429  case Intrinsic::x86_avx2_pmadd_wd:
3430  case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3431  case Intrinsic::x86_avx2_pmadd_ub_sw:
3432  handleVectorPmaddIntrinsic(I);
3433  break;
3434 
3435  case Intrinsic::x86_ssse3_pmadd_ub_sw:
3436  handleVectorPmaddIntrinsic(I, 8);
3437  break;
3438 
3439  case Intrinsic::x86_mmx_pmadd_wd:
3440  handleVectorPmaddIntrinsic(I, 16);
3441  break;
3442 
3443  case Intrinsic::x86_sse_cmp_ss:
3444  case Intrinsic::x86_sse2_cmp_sd:
3445  case Intrinsic::x86_sse_comieq_ss:
3446  case Intrinsic::x86_sse_comilt_ss:
3447  case Intrinsic::x86_sse_comile_ss:
3448  case Intrinsic::x86_sse_comigt_ss:
3449  case Intrinsic::x86_sse_comige_ss:
3450  case Intrinsic::x86_sse_comineq_ss:
3451  case Intrinsic::x86_sse_ucomieq_ss:
3452  case Intrinsic::x86_sse_ucomilt_ss:
3453  case Intrinsic::x86_sse_ucomile_ss:
3454  case Intrinsic::x86_sse_ucomigt_ss:
3455  case Intrinsic::x86_sse_ucomige_ss:
3456  case Intrinsic::x86_sse_ucomineq_ss:
3457  case Intrinsic::x86_sse2_comieq_sd:
3458  case Intrinsic::x86_sse2_comilt_sd:
3459  case Intrinsic::x86_sse2_comile_sd:
3460  case Intrinsic::x86_sse2_comigt_sd:
3461  case Intrinsic::x86_sse2_comige_sd:
3462  case Intrinsic::x86_sse2_comineq_sd:
3463  case Intrinsic::x86_sse2_ucomieq_sd:
3464  case Intrinsic::x86_sse2_ucomilt_sd:
3465  case Intrinsic::x86_sse2_ucomile_sd:
3466  case Intrinsic::x86_sse2_ucomigt_sd:
3467  case Intrinsic::x86_sse2_ucomige_sd:
3468  case Intrinsic::x86_sse2_ucomineq_sd:
3469  handleVectorCompareScalarIntrinsic(I);
3470  break;
3471 
3472  case Intrinsic::x86_sse_cmp_ps:
3473  case Intrinsic::x86_sse2_cmp_pd:
3474  // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
3475  // generates reasonably looking IR that fails in the backend with "Do not
3476  // know how to split the result of this operator!".
3477  handleVectorComparePackedIntrinsic(I);
3478  break;
3479 
3480  case Intrinsic::x86_bmi_bextr_32:
3481  case Intrinsic::x86_bmi_bextr_64:
3482  case Intrinsic::x86_bmi_bzhi_32:
3483  case Intrinsic::x86_bmi_bzhi_64:
3484  case Intrinsic::x86_bmi_pdep_32:
3485  case Intrinsic::x86_bmi_pdep_64:
3486  case Intrinsic::x86_bmi_pext_32:
3487  case Intrinsic::x86_bmi_pext_64:
3488  handleBmiIntrinsic(I);
3489  break;
3490 
3491  case Intrinsic::x86_pclmulqdq:
3492  case Intrinsic::x86_pclmulqdq_256:
3493  case Intrinsic::x86_pclmulqdq_512:
3494  handlePclmulIntrinsic(I);
3495  break;
3496 
3497  case Intrinsic::x86_sse41_round_sd:
3498  handleUnarySdIntrinsic(I);
3499  break;
3500  case Intrinsic::x86_sse2_max_sd:
3501  case Intrinsic::x86_sse2_min_sd:
3502  handleBinarySdIntrinsic(I);
3503  break;
3504 
3505  case Intrinsic::fshl:
3506  case Intrinsic::fshr:
3507  handleFunnelShift(I);
3508  break;
3509 
3510  case Intrinsic::is_constant:
3511  // The result of llvm.is.constant() is always defined.
3512  setShadow(&I, getCleanShadow(&I));
3513  setOrigin(&I, getCleanOrigin());
3514  break;
3515 
3516  default:
3517  if (!handleUnknownIntrinsic(I))
3518  visitInstruction(I);
3519  break;
3520  }
3521  }
3522 
3523  void visitLibAtomicLoad(CallBase &CB) {
3524  // Since we use getNextNode here, we can't have CB terminate the BB.
3525  assert(isa<CallInst>(CB));
3526 
3527  IRBuilder<> IRB(&CB);
3528  Value *Size = CB.getArgOperand(0);
3529  Value *SrcPtr = CB.getArgOperand(1);
3530  Value *DstPtr = CB.getArgOperand(2);
3531  Value *Ordering = CB.getArgOperand(3);
3532  // Convert the call to have at least Acquire ordering to make sure
3533  // the shadow operations aren't reordered before it.
3534  Value *NewOrdering =
3535  IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
3536  CB.setArgOperand(3, NewOrdering);
3537 
3538  IRBuilder<> NextIRB(CB.getNextNode());
3539  NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3540 
3541  Value *SrcShadowPtr, *SrcOriginPtr;
3542  std::tie(SrcShadowPtr, SrcOriginPtr) =
3543  getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
3544  /*isStore*/ false);
3545  Value *DstShadowPtr =
3546  getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
3547  /*isStore*/ true)
3548  .first;
3549 
3550  NextIRB.CreateMemCpy(DstShadowPtr, Align(1), SrcShadowPtr, Align(1), Size);
3551  if (MS.TrackOrigins) {
3552  Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
3554  Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
3555  NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
3556  }
3557  }
3558 
3559  void visitLibAtomicStore(CallBase &CB) {
3560  IRBuilder<> IRB(&CB);
3561  Value *Size = CB.getArgOperand(0);
3562  Value *DstPtr = CB.getArgOperand(2);
3563  Value *Ordering = CB.getArgOperand(3);
3564  // Convert the call to have at least Release ordering to make sure
3565  // the shadow operations aren't reordered after it.
3566  Value *NewOrdering =
3567  IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
3568  CB.setArgOperand(3, NewOrdering);
3569 
3570  Value *DstShadowPtr =
3571  getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1),
3572  /*isStore*/ true)
3573  .first;
3574 
3575  // Atomic store always paints clean shadow/origin. See file header.
3576  IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size,
3577  Align(1));
3578  }
3579 
3580  void visitCallBase(CallBase &CB) {
3581  assert(!CB.getMetadata("nosanitize"));
3582  if (CB.isInlineAsm()) {
3583  // For inline asm (either a call to asm function, or callbr instruction),
3584  // do the usual thing: check argument shadow and mark all outputs as
3585  // clean. Note that any side effects of the inline asm that are not
3586  // immediately visible in its constraints are not handled.
3587  if (ClHandleAsmConservative && MS.CompileKernel)
3588  visitAsmInstruction(CB);
3589  else
3590  visitInstruction(CB);
3591  return;
3592  }
3593  LibFunc LF;
3594  if (TLI->getLibFunc(CB, LF)) {
3595  // libatomic.a functions need to have special handling because there isn't
3596  // a good way to intercept them or compile the library with
3597  // instrumentation.
3598  switch (LF) {
3599  case LibFunc_atomic_load:
3600  if (!isa<CallInst>(CB)) {
3601  llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
3602  "Ignoring!\n";
3603  break;
3604  }
3605  visitLibAtomicLoad(CB);
3606  return;
3607  case LibFunc_atomic_store:
3608  visitLibAtomicStore(CB);
3609  return;
3610  default:
3611  break;
3612  }
3613  }
3614 
3615  if (auto *Call = dyn_cast<CallInst>(&CB)) {
3616  assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere");
3617 
3618  // We are going to insert code that relies on the fact that the callee
3619  // will become a non-readonly function after it is instrumented by us. To
3620  // prevent this code from being optimized out, mark that function
3621  // non-readonly in advance.
3622  AttributeMask B;
3623  B.addAttribute(Attribute::ReadOnly)
3624  .addAttribute(Attribute::ReadNone)
3625  .addAttribute(Attribute::WriteOnly)
3626  .addAttribute(Attribute::ArgMemOnly)
3627  .addAttribute(Attribute::Speculatable);
3628 
3629  Call->removeFnAttrs(B);
3630  if (Function *Func = Call->getCalledFunction()) {
3631  Func->removeFnAttrs(B);
3632  }
3633 
3635  }
3636  IRBuilder<> IRB(&CB);
3637  bool MayCheckCall = MS.EagerChecks;
3638  if (Function *Func = CB.getCalledFunction()) {
3639  // __sanitizer_unaligned_{load,store} functions may be called by users
3640  // and always expects shadows in the TLS. So don't check them.
3641  MayCheckCall &= !Func->getName().startswith("__sanitizer_unaligned_");
3642  }
3643 
3644  unsigned ArgOffset = 0;
3645  LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n");
3646  for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
3647  ++ArgIt) {
3648  Value *A = *ArgIt;
3649  unsigned i = ArgIt - CB.arg_begin();
3650  if (!A->getType()->isSized()) {
3651  LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << CB << "\n");
3652  continue;
3653  }
3654  unsigned Size = 0;
3655  const DataLayout &DL = F.getParent()->getDataLayout();
3656 
3657  bool ByVal = CB.paramHasAttr(i, Attribute::ByVal);
3658  bool NoUndef = CB.paramHasAttr(i, Attribute::NoUndef);
3659  bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
3660 
3661  if (EagerCheck) {
3662  insertShadowCheck(A, &CB);
3663  Size = DL.getTypeAllocSize(A->getType());
3664  } else {
3665  Value *Store = nullptr;
3666  // Compute the Shadow for arg even if it is ByVal, because
3667  // in that case getShadow() will copy the actual arg shadow to
3668  // __msan_param_tls.
3669  Value *ArgShadow = getShadow(A);
3670  Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
3671  LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
3672  << " Shadow: " << *ArgShadow << "\n");
3673  if (ByVal) {
3674  // ByVal requires some special handling as it's too big for a single
3675  // load
3676  assert(A->getType()->isPointerTy() &&
3677  "ByVal argument is not a pointer!");
3678  Size = DL.getTypeAllocSize(CB.getParamByValType(i));
3679  if (ArgOffset + Size > kParamTLSSize)
3680  break;
3681  const MaybeAlign ParamAlignment(CB.getParamAlign(i));
3683  if (ParamAlignment)
3684  Alignment = std::min(*ParamAlignment, kShadowTLSAlignment);
3685  Value *AShadowPtr, *AOriginPtr;
3686  std::tie(AShadowPtr, AOriginPtr) =
3687  getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment,
3688  /*isStore*/ false);
3689  if (!PropagateShadow) {
3690  Store = IRB.CreateMemSet(ArgShadowBase,
3692  Size, Alignment);
3693  } else {
3694  Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
3695  Alignment, Size);
3696  if (MS.TrackOrigins) {
3697  Value *ArgOriginBase = getOriginPtrForArgument(A, IRB, ArgOffset);
3698  // FIXME: OriginSize should be:
3699  // alignTo(A % kMinOriginAlignment + Size, kMinOriginAlignment)
3700  unsigned OriginSize = alignTo(Size, kMinOriginAlignment);
3701  IRB.CreateMemCpy(
3702  ArgOriginBase,
3703  /* by origin_tls[ArgOffset] */ kMinOriginAlignment,
3704  AOriginPtr,
3705  /* by getShadowOriginPtr */ kMinOriginAlignment, OriginSize);
3706  }
3707  }
3708  } else {
3709  // Any other parameters mean we need bit-grained tracking of uninit
3710  // data
3711  Size = DL.getTypeAllocSize(A->getType());
3712  if (ArgOffset + Size > kParamTLSSize)
3713  break;
3714  Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
3716  Constant *Cst = dyn_cast<Constant>(ArgShadow);
3717  if (MS.TrackOrigins && !(Cst && Cst->isNullValue())) {
3718  IRB.CreateStore(getOrigin(A),
3719  getOriginPtrForArgument(A, IRB, ArgOffset));
3720  }
3721  }
3722  (void)Store;
3723  assert(Store != nullptr);
3724  LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
3725  }
3726  assert(Size != 0);
3727  ArgOffset += alignTo(Size, kShadowTLSAlignment);
3728  }
3729  LLVM_DEBUG(dbgs() << " done with call args\n");
3730 
3731  FunctionType *FT = CB.getFunctionType();
3732  if (FT->isVarArg()) {
3733  VAHelper->visitCallBase(CB, IRB);
3734  }
3735 
3736  // Now, get the shadow for the RetVal.
3737  if (!CB.getType()->isSized())
3738  return;
3739  // Don't emit the epilogue for musttail call returns.
3740  if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
3741  return;
3742 
3743  if (MayCheckCall && CB.hasRetAttr(Attribute::NoUndef)) {
3744  setShadow(&CB, getCleanShadow(&CB));
3745  setOrigin(&CB, getCleanOrigin());
3746  return;
3747  }
3748 
3749  IRBuilder<> IRBBefore(&CB);
3750  // Until we have full dynamic coverage, make sure the retval shadow is 0.
3751  Value *Base = getShadowPtrForRetval(&CB, IRBBefore);
3752  IRBBefore.CreateAlignedStore(getCleanShadow(&CB), Base,
3754  BasicBlock::iterator NextInsn;
3755  if (isa<CallInst>(CB)) {
3756  NextInsn = ++CB.getIterator();
3757  assert(NextInsn != CB.getParent()->end());
3758  } else {
3759  BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
3760  if (!NormalDest->getSinglePredecessor()) {
3761  // FIXME: this case is tricky, so we are just conservative here.
3762  // Perhaps we need to split the edge between this BB and NormalDest,
3763  // but a naive attempt to use SplitEdge leads to a crash.
3764  setShadow(&CB, getCleanShadow(&CB));
3765  setOrigin(&CB, getCleanOrigin());
3766  return;
3767  }
3768  // FIXME: NextInsn is likely in a basic block that has not been visited yet.
3769  // Anything inserted there will be instrumented by MSan later!
3770  NextInsn = NormalDest->getFirstInsertionPt();
3771  assert(NextInsn != NormalDest->end() &&
3772  "Could not find insertion point for retval shadow load");
3773  }
3774  IRBuilder<> IRBAfter(&*NextInsn);
3775  Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
3776  getShadowTy(&CB), getShadowPtrForRetval(&CB, IRBAfter),
3777  kShadowTLSAlignment, "_msret");
3778  setShadow(&CB, RetvalShadow);
3779  if (MS.TrackOrigins)
3780  setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
3781  getOriginPtrForRetval(IRBAfter)));
3782  }
3783 
3784  bool isAMustTailRetVal(Value *RetVal) {
3785  if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
3786  RetVal = I->getOperand(0);
3787  }
3788  if (auto *I = dyn_cast<CallInst>(RetVal)) {
3789  return I->isMustTailCall();
3790  }
3791  return false;
3792  }
3793 
3794  void visitReturnInst(ReturnInst &I) {
3795  IRBuilder<> IRB(&I);
3796  Value *RetVal = I.getReturnValue();
3797  if (!RetVal) return;
3798  // Don't emit the epilogue for musttail call returns.
3799  if (isAMustTailRetVal(RetVal)) return;
3800  Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
3801  bool HasNoUndef =
3802  F.hasRetAttribute(Attribute::NoUndef);
3803  bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
3804  // FIXME: Consider using SpecialCaseList to specify a list of functions that
3805  // must always return fully initialized values. For now, we hardcode "main".
3806  bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (F.getName() == "main");
3807 
3808  Value *Shadow = getShadow(RetVal);
3809  bool StoreOrigin = true;
3810  if (EagerCheck) {
3811  insertShadowCheck(RetVal, &I);
3812  Shadow = getCleanShadow(RetVal);
3813  StoreOrigin = false;
3814  }
3815 
3816  // The caller may still expect information passed over TLS if we pass our
3817  // check
3818  if (StoreShadow) {
3819  IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3820  if (MS.TrackOrigins && StoreOrigin)
3821  IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
3822  }
3823  }
3824 
3825  void visitPHINode(PHINode &I) {
3826  IRBuilder<> IRB(&I);
3827  if (!PropagateShadow) {
3828  setShadow(&I, getCleanShadow(&I));
3829  setOrigin(&I, getCleanOrigin());
3830  return;
3831  }
3832 
3833  ShadowPHINodes.push_back(&I);
3834  setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
3835  "_msphi_s"));
3836  if (MS.TrackOrigins)
3837  setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(),
3838  "_msphi_o"));
3839  }
3840 
3841  Value *getLocalVarDescription(AllocaInst &I) {
3842  SmallString<2048> StackDescriptionStorage;
3843  raw_svector_ostream StackDescription(StackDescriptionStorage);
3844  // We create a string with a description of the stack allocation and
3845  // pass it into __msan_set_alloca_origin.
3846  // It will be printed by the run-time if stack-originated UMR is found.
3847  // The first 4 bytes of the string are set to '----' and will be replaced
3848  // by __msan_va_arg_overflow_size_tls at the first call.
3849  StackDescription << "----" << I.getName() << "@" << F.getName();
3850  return createPrivateNonConstGlobalForString(*F.getParent(),
3851  StackDescription.str());
3852  }
3853 
3854  void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3855  if (PoisonStack && ClPoisonStackWithCall) {
3856  IRB.CreateCall(MS.MsanPoisonStackFn,
3857  {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3858  } else {
3859  Value *ShadowBase, *OriginBase;
3860  std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
3861  &I, IRB, IRB.getInt8Ty(), Align(1), /*isStore*/ true);
3862 
3863  Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
3864  IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlign());
3865  }
3866 
3867  if (PoisonStack && MS.TrackOrigins) {
3868  Value *Descr = getLocalVarDescription(I);
3869  IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
3870  {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3871  IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
3872  IRB.CreatePointerCast(&F, MS.IntptrTy)});
3873  }
3874  }
3875 
3876  void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3877  Value *Descr = getLocalVarDescription(I);
3878  if (PoisonStack) {
3879  IRB.CreateCall(MS.MsanPoisonAllocaFn,
3880  {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3881  IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
3882  } else {
3883  IRB.CreateCall(MS.MsanUnpoisonAllocaFn,
3884  {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3885  }
3886  }
3887 
3888  void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
3889  if (!InsPoint)
3890  InsPoint = &I;
3891  IRBuilder<> IRB(InsPoint->getNextNode());
3892  const DataLayout &DL = F.getParent()->getDataLayout();
3893  uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
3894  Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
3895  if (I.isArrayAllocation())
3896  Len = IRB.CreateMul(Len, I.getArraySize());
3897 
3898  if (MS.CompileKernel)
3899  poisonAllocaKmsan(I, IRB, Len);
3900  else
3901  poisonAllocaUserspace(I, IRB, Len);
3902  }
3903 
3904  void visitAllocaInst(AllocaInst &I) {
3905  setShadow(&I, getCleanShadow(&I));
3906  setOrigin(&I, getCleanOrigin());
3907  // We'll get to this alloca later unless it's poisoned at the corresponding
3908  // llvm.lifetime.start.
3909  AllocaSet.insert(&I);
3910  }
3911 
3912  void visitSelectInst(SelectInst& I) {
3913  IRBuilder<> IRB(&I);
3914  // a = select b, c, d
3915  Value *B = I.getCondition();
3916  Value *C = I.getTrueValue();
3917  Value *D = I.getFalseValue();
3918  Value *Sb = getShadow(B);
3919  Value *Sc = getShadow(C);
3920  Value *Sd = getShadow(D);
3921 
3922  // Result shadow if condition shadow is 0.
3923  Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
3924  Value *Sa1;
3925  if (I.getType()->isAggregateType()) {
3926  // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
3927  // an extra "select". This results in much more compact IR.
3928  // Sa = select Sb, poisoned, (select b, Sc, Sd)
3929  Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
3930  } else {
3931  // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
3932  // If Sb (condition is poisoned), look for bits in c and d that are equal
3933  // and both unpoisoned.
3934  // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
3935 
3936  // Cast arguments to shadow-compatible type.
3937  C = CreateAppToShadowCast(IRB, C);
3938  D = CreateAppToShadowCast(IRB, D);
3939 
3940  // Result shadow if condition shadow is 1.
3941  Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd});
3942  }
3943  Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
3944  setShadow(&I, Sa);
3945  if (MS.TrackOrigins) {
3946  // Origins are always i32, so any vector conditions must be flattened.
3947  // FIXME: consider tracking vector origins for app vectors?
3948  if (B->getType()->isVectorTy()) {
3949  Type *FlatTy = getShadowTyNoVec(B->getType());
3950  B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
3951  ConstantInt::getNullValue(FlatTy));
3952  Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
3953  ConstantInt::getNullValue(FlatTy));
3954  }
3955  // a = select b, c, d
3956  // Oa = Sb ? Ob : (b ? Oc : Od)
3957  setOrigin(
3958  &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
3959  IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
3960  getOrigin(I.getFalseValue()))));
3961  }
3962  }
3963 
3964  void visitLandingPadInst(LandingPadInst &I) {
3965  // Do nothing.
3966  // See https://github.com/google/sanitizers/issues/504
3967  setShadow(&I, getCleanShadow(&I));
3968  setOrigin(&I, getCleanOrigin());
3969  }
3970 
3971  void visitCatchSwitchInst(CatchSwitchInst &I) {
3972  setShadow(&I, getCleanShadow(&I));
3973  setOrigin(&I, getCleanOrigin());
3974  }
3975 
3976  void visitFuncletPadInst(FuncletPadInst &I) {
3977  setShadow(&I, getCleanShadow(&I));
3978  setOrigin(&I, getCleanOrigin());
3979  }
3980 
3981  void visitGetElementPtrInst(GetElementPtrInst &I) {
3982  handleShadowOr(I);
3983  }
3984 
3985  void visitExtractValueInst(ExtractValueInst &I) {
3986  IRBuilder<> IRB(&I);
3987  Value *Agg = I.getAggregateOperand();
3988  LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
3989  Value *AggShadow = getShadow(Agg);
3990  LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
3991  Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
3992  LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
3993  setShadow(&I, ResShadow);
3994  setOriginForNaryOp(I);
3995  }
3996 
3997  void visitInsertValueInst(InsertValueInst &I) {
3998  IRBuilder<> IRB(&I);
3999  LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
4000  Value *AggShadow = getShadow(I.getAggregateOperand());
4001  Value *InsShadow = getShadow(I.getInsertedValueOperand());
4002  LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
4003  LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
4004  Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
4005  LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
4006  setShadow(&I, Res);
4007  setOriginForNaryOp(I);
4008  }
4009 
4010  void dumpInst(Instruction &I) {
4011  if (CallInst *CI = dyn_cast<CallInst>(&I)) {
4012  errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
4013  } else {
4014  errs() << "ZZZ " << I.getOpcodeName() << "\n";
4015  }
4016  errs() << "QQQ " << I << "\n";
4017  }
4018 
4019  void visitResumeInst(ResumeInst &I) {
4020  LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
4021  // Nothing to do here.
4022  }
4023 
4024  void visitCleanupReturnInst(CleanupReturnInst &CRI) {
4025  LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
4026  // Nothing to do here.
4027  }
4028 
4029  void visitCatchReturnInst(CatchReturnInst &CRI) {
4030  LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
4031  // Nothing to do here.
4032  }
4033 
4034  void instrumentAsmArgument(Value *Operand, Type *ElemTy, Instruction &I,
4035  IRBuilder<> &IRB, const DataLayout &DL,
4036  bool isOutput) {
4037  // For each assembly argument, we check its value for being initialized.
4038  // If the argument is a pointer, we assume it points to a single element
4039  // of the corresponding type (or to a 8-byte word, if the type is unsized).
4040  // Each such pointer is instrumented with a call to the runtime library.
4041  Type *OpType = Operand->getType();
4042  // Check the operand value itself.
4043  insertShadowCheck(Operand, &I);
4044  if (!OpType->isPointerTy() || !isOutput) {
4045  assert(!isOutput);
4046  return;
4047  }
4048  if (!ElemTy->isSized())
4049  return;
4050  int Size = DL.getTypeStoreSize(ElemTy);
4051  Value *Ptr = IRB.CreatePointerCast(Operand, IRB.getInt8PtrTy());
4052  Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
4053  IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal});
4054  }
4055 
4056  /// Get the number of output arguments returned by pointers.
4057  int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
4058  int NumRetOutputs = 0;
4059  int NumOutputs = 0;
4060  Type *RetTy = cast<Value>(CB)->getType();
4061  if (!RetTy->isVoidTy()) {
4062  // Register outputs are returned via the CallInst return value.
4063  auto *ST = dyn_cast<StructType>(RetTy);
4064  if (ST)
4065  NumRetOutputs = ST->getNumElements();
4066  else
4067  NumRetOutputs = 1;
4068  }
4070  for (const InlineAsm::ConstraintInfo &Info : Constraints) {
4071  switch (Info.Type) {
4072  case InlineAsm::isOutput:
4073  NumOutputs++;
4074  break;
4075  default:
4076  break;
4077  }
4078  }
4079  return NumOutputs - NumRetOutputs;
4080  }
4081 
4082  void visitAsmInstruction(Instruction &I) {
4083  // Conservative inline assembly handling: check for poisoned shadow of
4084  // asm() arguments, then unpoison the result and all the memory locations
4085  // pointed to by those arguments.
4086  // An inline asm() statement in C++ contains lists of input and output
4087  // arguments used by the assembly code. These are mapped to operands of the
4088  // CallInst as follows:
4089  // - nR register outputs ("=r) are returned by value in a single structure
4090  // (SSA value of the CallInst);
4091  // - nO other outputs ("=m" and others) are returned by pointer as first
4092  // nO operands of the CallInst;
4093  // - nI inputs ("r", "m" and others) are passed to CallInst as the
4094  // remaining nI operands.
4095  // The total number of asm() arguments in the source is nR+nO+nI, and the
4096  // corresponding CallInst has nO+nI+1 operands (the last operand is the
4097  // function to be called).
4098  const DataLayout &DL = F.getParent()->getDataLayout();
4099  CallBase *CB = cast<CallBase>(&I);
4100  IRBuilder<> IRB(&I);
4101  InlineAsm *IA = cast<InlineAsm>(CB->getCalledOperand());
4102  int OutputArgs = getNumOutputArgs(IA, CB);
4103  // The last operand of a CallInst is the function itself.
4104  int NumOperands = CB->getNumOperands() - 1;
4105 
4106  // Check input arguments. Doing so before unpoisoning output arguments, so
4107  // that we won't overwrite uninit values before checking them.
4108  for (int i = OutputArgs; i < NumOperands; i++) {
4109  Value *Operand = CB->getOperand(i);
4110  instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL,
4111  /*isOutput*/ false);
4112  }
4113  // Unpoison output arguments. This must happen before the actual InlineAsm
4114  // call, so that the shadow for memory published in the asm() statement
4115  // remains valid.
4116  for (int i = 0; i < OutputArgs; i++) {
4117  Value *Operand = CB->getOperand(i);
4118  instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL,
4119  /*isOutput*/ true);
4120  }
4121 
4122  setShadow(&I, getCleanShadow(&I));
4123  setOrigin(&I, getCleanOrigin());
4124  }
4125 
4126  void visitFreezeInst(FreezeInst &I) {
4127  // Freeze always returns a fully defined value.
4128  setShadow(&I, getCleanShadow(&I));
4129  setOrigin(&I, getCleanOrigin());
4130  }
4131 
4132  void visitInstruction(Instruction &I) {
4133  // Everything else: stop propagating and check for poisoned shadow.
4135  dumpInst(I);
4136  LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
4137  for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
4138  Value *Operand = I.getOperand(i);
4139  if (Operand->getType()->isSized())
4140  insertShadowCheck(Operand, &I);
4141  }
4142  setShadow(&I, getCleanShadow(&I));
4143  setOrigin(&I, getCleanOrigin());
4144  }
4145 };
4146 
4147 /// AMD64-specific implementation of VarArgHelper.
4148 struct VarArgAMD64Helper : public VarArgHelper {
4149  // An unfortunate workaround for asymmetric lowering of va_arg stuff.
4150  // See a comment in visitCallBase for more details.
4151  static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
4152  static const unsigned AMD64FpEndOffsetSSE = 176;
4153  // If SSE is disabled, fp_offset in va_list is zero.
4154  static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4155 
4156  unsigned AMD64FpEndOffset;
4157  Function &F;
4158  MemorySanitizer &MS;
4159  MemorySanitizerVisitor &MSV;
4160  Value *VAArgTLSCopy = nullptr;
4161  Value *VAArgTLSOriginCopy = nullptr;
4162  Value *VAArgOverflowSize = nullptr;
4163 
4164  SmallVector<CallInst*, 16> VAStartInstrumentationList;
4165 
4166  enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4167 
4168  VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
4169  MemorySanitizerVisitor &MSV)
4170  : F(F), MS(MS), MSV(MSV) {
4171  AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4172  for (const auto &Attr : F.getAttributes().getFnAttrs()) {
4173  if (Attr.isStringAttribute() &&
4174  (Attr.getKindAsString() == "target-features")) {
4175  if (Attr.getValueAsString().contains("-sse"))
4176  AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4177  break;
4178  }
4179  }
4180  }
4181 
4182  ArgKind classifyArgument(Value* arg) {
4183  // A very rough approximation of X86_64 argument classification rules.
4184  Type *T = arg->getType();
4185  if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
4186  return AK_FloatingPoint;
4187  if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4188  return AK_GeneralPurpose;
4189  if (T->isPointerTy())
4190  return AK_GeneralPurpose;
4191  return AK_Memory;
4192  }
4193 
4194  // For VarArg functions, store the argument shadow in an ABI-specific format
4195  // that corresponds to va_list layout.
4196  // We do this because Clang lowers va_arg in the frontend, and this pass
4197  // only sees the low level code that deals with va_list internals.
4198  // A much easier alternative (provided that Clang emits va_arg instructions)
4199  // would have been to associate each live instance of va_list with a copy of
4200  // MSanParamTLS, and extract shadow on va_arg() call in the argument list
4201  // order.
4202  void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4203  unsigned GpOffset = 0;
4204  unsigned FpOffset = AMD64GpEndOffset;
4205  unsigned OverflowOffset = AMD64FpEndOffset;
4206  const DataLayout &DL = F.getParent()->getDataLayout();
4207  for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
4208  ++ArgIt) {
4209  Value *A = *ArgIt;
4210  unsigned ArgNo = CB.getArgOperandNo(ArgIt);
4211  bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4212  bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
4213  if (IsByVal) {
4214  // ByVal arguments always go to the overflow area.
4215  // Fixed arguments passed through the overflow area will be stepped
4216  // over by va_start, so don't count them towards the offset.
4217  if (IsFixed)
4218  continue;
4219  assert(A->getType()->isPointerTy());
4220  Type *RealTy = CB.getParamByValType(ArgNo);
4221  uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4222  Value *ShadowBase = getShadowPtrForVAArgument(
4223  RealTy, IRB, OverflowOffset, alignTo(ArgSize, 8));
4224  Value *OriginBase = nullptr;
4225  if (MS.TrackOrigins)
4226  OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset);
4227  OverflowOffset += alignTo(ArgSize, 8);
4228  if (!ShadowBase)
4229  continue;
4230  Value *ShadowPtr, *OriginPtr;
4231  std::tie(ShadowPtr, OriginPtr) =
4232  MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
4233  /*isStore*/ false);
4234 
4235  IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
4236  kShadowTLSAlignment, ArgSize);
4237  if (MS.TrackOrigins)
4238  IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr,
4239  kShadowTLSAlignment, ArgSize);
4240  } else {
4241  ArgKind AK = classifyArgument(A);
4242  if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4243  AK = AK_Memory;
4244  if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4245  AK = AK_Memory;
4246  Value *ShadowBase, *OriginBase = nullptr;
4247  switch (AK) {
4248  case AK_GeneralPurpose:
4249  ShadowBase =
4250  getShadowPtrForVAArgument(A->getType(), IRB, GpOffset, 8);
4251  if (MS.TrackOrigins)
4252  OriginBase =
4253  getOriginPtrForVAArgument(A->getType(), IRB, GpOffset);
4254  GpOffset += 8;
4255  break;
4256  case AK_FloatingPoint:
4257  ShadowBase =
4258  getShadowPtrForVAArgument(A->getType(), IRB, FpOffset, 16);
4259  if (MS.TrackOrigins)
4260  OriginBase =
4261  getOriginPtrForVAArgument(A->getType(), IRB, FpOffset);
4262  FpOffset += 16;
4263  break;
4264  case AK_Memory:
4265  if (IsFixed)
4266  continue;
4267  uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4268  ShadowBase =
4269  getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset, 8);
4270  if (MS.TrackOrigins)
4271  OriginBase =
4272  getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset);
4273  OverflowOffset += alignTo(ArgSize, 8);
4274  }
4275  // Take fixed arguments into account for GpOffset and FpOffset,
4276  // but don't actually store shadows for them.
4277  // TODO(glider): don't call get*PtrForVAArgument() for them.
4278  if (IsFixed)
4279  continue;
4280  if (!ShadowBase)
4281  continue;
4282  Value *Shadow = MSV.getShadow(A);
4283  IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment);
4284  if (MS.TrackOrigins) {
4285  Value *Origin = MSV.getOrigin(A);
4286  unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
4287  MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
4289  }
4290  }
4291  }
4292  Constant *OverflowSize =
4293  ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
4294  IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4295  }
4296 
4297  /// Compute the shadow address for a given va_arg.
4298  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4299  unsigned ArgOffset, unsigned ArgSize) {
4300  // Make sure we don't overflow __msan_va_arg_tls.
4301  if (ArgOffset + ArgSize > kParamTLSSize)
4302  return nullptr;
4303  Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4304  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4305  return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4306  "_msarg_va_s");
4307  }
4308 
4309  /// Compute the origin address for a given va_arg.
4310  Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) {
4311  Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
4312  // getOriginPtrForVAArgument() is always called after
4313  // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
4314  // overflow.
4315  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4316  return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
4317  "_msarg_va_o");
4318  }
4319 
4320  void unpoisonVAListTagForInst(IntrinsicInst &I) {
4321  IRBuilder<> IRB(&I);
4322  Value *VAListTag = I.getArgOperand(0);
4323  Value *ShadowPtr, *OriginPtr;
4324  const Align Alignment = Align(8);
4325  std::tie(ShadowPtr, OriginPtr) =
4326  MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
4327  /*isStore*/ true);
4328 
4329  // Unpoison the whole __va_list_tag.
4330  // FIXME: magic ABI constants.
4331  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4332  /* size */ 24, Alignment, false);
4333  // We shouldn't need to zero out the origins, as they're only checked for
4334  // nonzero shadow.
4335  }
4336 
4337  void visitVAStartInst(VAStartInst &I) override {
4338  if (F.getCallingConv() == CallingConv::Win64)
4339  return;
4340  VAStartInstrumentationList.push_back(&I);
4341  unpoisonVAListTagForInst(I);
4342  }
4343 
4344  void visitVACopyInst(VACopyInst &I) override {
4345  if (F.getCallingConv() == CallingConv::Win64) return;
4346  unpoisonVAListTagForInst(I);
4347  }
4348 
4349  void finalizeInstrumentation() override {
4350  assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4351  "finalizeInstrumentation called twice");
4352  if (!VAStartInstrumentationList.empty()) {
4353  // If there is a va_start in this function, make a backup copy of
4354  // va_arg_tls somewhere in the function entry block.
4355  IRBuilder<> IRB(MSV.FnPrologueEnd);
4356  VAArgOverflowSize =
4357  IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4358  Value *CopySize =
4359  IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
4360  VAArgOverflowSize);
4361  VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4362  IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4363  if (MS.TrackOrigins) {
4364  VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4365  IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS,
4366  Align(8), CopySize);
4367  }
4368  }
4369 
4370  // Instrument va_start.
4371  // Copy va_list shadow from the backup copy of the TLS contents.
4372  for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4373  CallInst *OrigInst = VAStartInstrumentationList[i];
4374  IRBuilder<> IRB(OrigInst->getNextNode());
4375  Value *VAListTag = OrigInst->getArgOperand(0);
4376 
4377  Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4378  Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
4379  IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4380  ConstantInt::get(MS.IntptrTy, 16)),
4381  PointerType::get(RegSaveAreaPtrTy, 0));
4382  Value *RegSaveAreaPtr =
4383  IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4384  Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4385  const Align Alignment = Align(16);
4386  std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4387  MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4388  Alignment, /*isStore*/ true);
4389  IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4390  AMD64FpEndOffset);
4391  if (MS.TrackOrigins)
4392  IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
4393  Alignment, AMD64FpEndOffset);
4394  Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4395  Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
4396  IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4397  ConstantInt::get(MS.IntptrTy, 8)),
4398  PointerType::get(OverflowArgAreaPtrTy, 0));
4399  Value *OverflowArgAreaPtr =
4400  IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
4401  Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
4402  std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
4403  MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
4404  Alignment, /*isStore*/ true);
4405  Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
4406  AMD64FpEndOffset);
4407  IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
4408  VAArgOverflowSize);
4409  if (MS.TrackOrigins) {
4410  SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
4411  AMD64FpEndOffset);
4412  IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
4413  VAArgOverflowSize);
4414  }
4415  }
4416  }
4417 };
4418 
4419 /// MIPS64-specific implementation of VarArgHelper.
4420 struct VarArgMIPS64Helper : public VarArgHelper {
4421  Function &F;
4422  MemorySanitizer &MS;
4423  MemorySanitizerVisitor &MSV;
4424  Value *VAArgTLSCopy = nullptr;
4425  Value *VAArgSize = nullptr;
4426 
4427  SmallVector<CallInst*, 16> VAStartInstrumentationList;
4428 
4429  VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
4430  MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4431 
4432  void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4433  unsigned VAArgOffset = 0;
4434  const DataLayout &DL = F.getParent()->getDataLayout();
4435  for (auto ArgIt = CB.arg_begin() + CB.getFunctionType()->getNumParams(),
4436  End = CB.arg_end();
4437  ArgIt != End; ++ArgIt) {
4438  Triple TargetTriple(F.getParent()->getTargetTriple());
4439  Value *A = *ArgIt;
4440  Value *Base;
4441  uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4442  if (TargetTriple.getArch() == Triple::mips64) {
4443  // Adjusting the shadow for argument with size < 8 to match the placement
4444  // of bits in big endian system
4445  if (ArgSize < 8)
4446  VAArgOffset += (8 - ArgSize);
4447  }
4448  Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize);
4449  VAArgOffset += ArgSize;
4450  VAArgOffset = alignTo(VAArgOffset, 8);
4451  if (!Base)
4452  continue;
4453  IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4454  }
4455 
4456  Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
4457  // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4458  // a new class member i.e. it is the total size of all VarArgs.
4459  IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4460  }
4461 
4462  /// Compute the shadow address for a given va_arg.
4463  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4464  unsigned ArgOffset, unsigned ArgSize) {
4465  // Make sure we don't overflow __msan_va_arg_tls.
4466  if (ArgOffset + ArgSize > kParamTLSSize)
4467  return nullptr;
4468  Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4469  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4470  return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4471  "_msarg");
4472  }
4473 
4474  void visitVAStartInst(VAStartInst &I) override {
4475  IRBuilder<> IRB(&I);
4476  VAStartInstrumentationList.push_back(&I);
4477  Value *VAListTag = I.getArgOperand(0);
4478  Value *ShadowPtr, *OriginPtr;
4479  const Align Alignment = Align(8);
4480  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4481  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4482  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4483  /* size */ 8, Alignment, false);
4484  }
4485 
4486  void visitVACopyInst(VACopyInst &I) override {
4487  IRBuilder<> IRB(&I);
4488  VAStartInstrumentationList.push_back(&I);
4489  Value *VAListTag = I.getArgOperand(0);
4490  Value *ShadowPtr, *OriginPtr;
4491  const Align Alignment = Align(8);
4492  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4493  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4494  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4495  /* size */ 8, Alignment, false);
4496  }
4497 
4498  void finalizeInstrumentation() override {
4499  assert(!VAArgSize && !VAArgTLSCopy &&
4500  "finalizeInstrumentation called twice");
4501  IRBuilder<> IRB(MSV.FnPrologueEnd);
4502  VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4503  Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4504  VAArgSize);
4505 
4506  if (!VAStartInstrumentationList.empty()) {
4507  // If there is a va_start in this function, make a backup copy of
4508  // va_arg_tls somewhere in the function entry block.
4509  VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4510  IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4511  }
4512 
4513  // Instrument va_start.
4514  // Copy va_list shadow from the backup copy of the TLS contents.
4515  for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4516  CallInst *OrigInst = VAStartInstrumentationList[i];
4517  IRBuilder<> IRB(OrigInst->getNextNode());
4518  Value *VAListTag = OrigInst->getArgOperand(0);
4519  Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4520  Value *RegSaveAreaPtrPtr =
4521  IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4522  PointerType::get(RegSaveAreaPtrTy, 0));
4523  Value *RegSaveAreaPtr =
4524  IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4525  Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4526  const Align Alignment = Align(8);
4527  std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4528  MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4529  Alignment, /*isStore*/ true);
4530  IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4531  CopySize);
4532  }
4533  }
4534 };
4535 
4536 /// AArch64-specific implementation of VarArgHelper.
4537 struct VarArgAArch64Helper : public VarArgHelper {
4538  static const unsigned kAArch64GrArgSize = 64;
4539  static const unsigned kAArch64VrArgSize = 128;
4540 
4541  static const unsigned AArch64GrBegOffset = 0;
4542  static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
4543  // Make VR space aligned to 16 bytes.
4544  static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
4545  static const unsigned AArch64VrEndOffset = AArch64VrBegOffset
4546  + kAArch64VrArgSize;
4547  static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
4548 
4549  Function &F;
4550  MemorySanitizer &MS;
4551  MemorySanitizerVisitor &MSV;
4552  Value *VAArgTLSCopy = nullptr;
4553  Value *VAArgOverflowSize = nullptr;
4554 
4555  SmallVector<CallInst*, 16> VAStartInstrumentationList;
4556 
4557  enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4558 
4559  VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
4560  MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4561 
4562  ArgKind classifyArgument(Value* arg) {
4563  Type *T = arg->getType();
4564  if (T->isFPOrFPVectorTy())
4565  return AK_FloatingPoint;
4566  if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4567  || (T->isPointerTy()))
4568  return AK_GeneralPurpose;
4569  return AK_Memory;
4570  }
4571 
4572  // The instrumentation stores the argument shadow in a non ABI-specific
4573  // format because it does not know which argument is named (since Clang,
4574  // like x86_64 case, lowers the va_args in the frontend and this pass only
4575  // sees the low level code that deals with va_list internals).
4576  // The first seven GR registers are saved in the first 56 bytes of the
4577  // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
4578  // the remaining arguments.
4579  // Using constant offset within the va_arg TLS array allows fast copy
4580  // in the finalize instrumentation.
4581  void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4582  unsigned GrOffset = AArch64GrBegOffset;
4583  unsigned VrOffset = AArch64VrBegOffset;
4584  unsigned OverflowOffset = AArch64VAEndOffset;
4585 
4586  const DataLayout &DL = F.getParent()->getDataLayout();
4587  for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
4588  ++ArgIt) {
4589  Value *A = *ArgIt;
4590  unsigned ArgNo = CB.getArgOperandNo(ArgIt);
4591  bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4592  ArgKind AK = classifyArgument(A);
4593  if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
4594  AK = AK_Memory;
4595  if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset)
4596  AK = AK_Memory;
4597  Value *Base;
4598  switch (AK) {
4599  case AK_GeneralPurpose:
4600  Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset, 8);
4601  GrOffset += 8;
4602  break;
4603  case AK_FloatingPoint:
4604  Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset, 8);
4605  VrOffset += 16;
4606  break;
4607  case AK_Memory:
4608  // Don't count fixed arguments in the overflow area - va_start will
4609  // skip right over them.
4610  if (IsFixed)
4611  continue;
4612  uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4613  Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset,
4614  alignTo(ArgSize, 8));
4615  OverflowOffset += alignTo(ArgSize, 8);
4616  break;
4617  }
4618  // Count Gp/Vr fixed arguments to their respective offsets, but don't
4619  // bother to actually store a shadow.
4620  if (IsFixed)
4621  continue;
4622  if (!Base)
4623  continue;
4624  IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4625  }
4626  Constant *OverflowSize =
4627  ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
4628  IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4629  }
4630 
4631  /// Compute the shadow address for a given va_arg.
4632  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4633  unsigned ArgOffset, unsigned ArgSize) {
4634  // Make sure we don't overflow __msan_va_arg_tls.
4635  if (ArgOffset + ArgSize > kParamTLSSize)
4636  return nullptr;
4637  Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4638  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4639  return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4640  "_msarg");
4641  }
4642 
4643  void visitVAStartInst(VAStartInst &I) override {
4644  IRBuilder<> IRB(&I);
4645  VAStartInstrumentationList.push_back(&I);
4646  Value *VAListTag = I.getArgOperand(0);
4647  Value *ShadowPtr, *OriginPtr;
4648  const Align Alignment = Align(8);
4649  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4650  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4651  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4652  /* size */ 32, Alignment, false);
4653  }
4654 
4655  void visitVACopyInst(VACopyInst &I) override {
4656  IRBuilder<> IRB(&I);
4657  VAStartInstrumentationList.push_back(&I);
4658  Value *VAListTag = I.getArgOperand(0);
4659  Value *ShadowPtr, *OriginPtr;
4660  const Align Alignment = Align(8);
4661  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4662  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4663  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4664  /* size */ 32, Alignment, false);
4665  }
4666 
4667  // Retrieve a va_list field of 'void*' size.
4668  Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4669  Value *SaveAreaPtrPtr =
4670  IRB.CreateIntToPtr(
4671  IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4672  ConstantInt::get(MS.IntptrTy, offset)),
4673  Type::getInt64PtrTy(*MS.C));
4674  return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
4675  }
4676 
4677  // Retrieve a va_list field of 'int' size.
4678  Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4679  Value *SaveAreaPtr =
4680  IRB.CreateIntToPtr(
4681  IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4682  ConstantInt::get(MS.IntptrTy, offset)),
4683  Type::getInt32PtrTy(*MS.C));
4684  Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
4685  return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
4686  }
4687 
4688  void finalizeInstrumentation() override {
4689  assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4690  "finalizeInstrumentation called twice");
4691  if (!VAStartInstrumentationList.empty()) {
4692  // If there is a va_start in this function, make a backup copy of
4693  // va_arg_tls somewhere in the function entry block.
4694  IRBuilder<> IRB(MSV.FnPrologueEnd);
4695  VAArgOverflowSize =
4696  IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4697  Value *CopySize =
4698  IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
4699  VAArgOverflowSize);
4700  VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4701  IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4702  }
4703 
4704  Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
4705  Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
4706 
4707  // Instrument va_start, copy va_list shadow from the backup copy of
4708  // the TLS contents.
4709  for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4710  CallInst *OrigInst = VAStartInstrumentationList[i];
4711  IRBuilder<> IRB(OrigInst->getNextNode());
4712 
4713  Value *VAListTag = OrigInst->getArgOperand(0);
4714 
4715  // The variadic ABI for AArch64 creates two areas to save the incoming
4716  // argument registers (one for 64-bit general register xn-x7 and another
4717  // for 128-bit FP/SIMD vn-v7).
4718  // We need then to propagate the shadow arguments on both regions
4719  // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
4720  // The remaining arguments are saved on shadow for 'va::stack'.
4721  // One caveat is it requires only to propagate the non-named arguments,
4722  // however on the call site instrumentation 'all' the arguments are
4723  // saved. So to copy the shadow values from the va_arg TLS array
4724  // we need to adjust the offset for both GR and VR fields based on
4725  // the __{gr,vr}_offs value (since they are stores based on incoming
4726  // named arguments).
4727 
4728  // Read the stack pointer from the va_list.
4729  Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
4730 
4731  // Read both the __gr_top and __gr_off and add them up.
4732  Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
4733  Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
4734 
4735  Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
4736 
4737  // Read both the __vr_top and __vr_off and add them up.
4738  Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
4739  Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
4740 
4741  Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
4742 
4743  // It does not know how many named arguments is being used and, on the
4744  // callsite all the arguments were saved. Since __gr_off is defined as
4745  // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
4746  // argument by ignoring the bytes of shadow from named arguments.
4747  Value *GrRegSaveAreaShadowPtrOff =
4748  IRB.CreateAdd(GrArgSize, GrOffSaveArea);
4749 
4750  Value *GrRegSaveAreaShadowPtr =
4751  MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4752  Align(8), /*isStore*/ true)
4753  .first;
4754 
4755  Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4756  GrRegSaveAreaShadowPtrOff);
4757  Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
4758 
4759  IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8),
4760  GrCopySize);
4761 
4762  // Again, but for FP/SIMD values.
4763  Value *VrRegSaveAreaShadowPtrOff =
4764  IRB.CreateAdd(VrArgSize, VrOffSaveArea);
4765 
4766  Value *VrRegSaveAreaShadowPtr =
4767  MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4768  Align(8), /*isStore*/ true)
4769  .first;
4770 
4771  Value *VrSrcPtr = IRB.CreateInBoundsGEP(
4772  IRB.getInt8Ty(),
4773  IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4774  IRB.getInt32(AArch64VrBegOffset)),
4775  VrRegSaveAreaShadowPtrOff);
4776  Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
4777 
4778  IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8),
4779  VrCopySize);
4780 
4781  // And finally for remaining arguments.
4782  Value *StackSaveAreaShadowPtr =
4783  MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
4784  Align(16), /*isStore*/ true)
4785  .first;
4786 
4787  Value *StackSrcPtr =
4788  IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4789  IRB.getInt32(AArch64VAEndOffset));
4790 
4791  IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr,
4792  Align(16), VAArgOverflowSize);
4793  }
4794  }
4795 };
4796 
4797 /// PowerPC64-specific implementation of VarArgHelper.
4798 struct VarArgPowerPC64Helper : public VarArgHelper {
4799  Function &F;
4800  MemorySanitizer &MS;
4801  MemorySanitizerVisitor &MSV;
4802  Value *VAArgTLSCopy = nullptr;
4803  Value *VAArgSize = nullptr;
4804 
4805  SmallVector<CallInst*, 16> VAStartInstrumentationList;
4806 
4807  VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
4808  MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4809 
4810  void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4811  // For PowerPC, we need to deal with alignment of stack arguments -
4812  // they are mostly aligned to 8 bytes, but vectors and i128 arrays
4813  // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
4814  // For that reason, we compute current offset from stack pointer (which is
4815  // always properly aligned), and offset for the first vararg, then subtract
4816  // them.
4817  unsigned VAArgBase;
4818  Triple TargetTriple(F.getParent()->getTargetTriple());
4819  // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
4820  // and 32 bytes for ABIv2. This is usually determined by target
4821  // endianness, but in theory could be overridden by function attribute.
4822  if (TargetTriple.getArch() == Triple::ppc64)
4823  VAArgBase = 48;
4824  else
4825  VAArgBase = 32;
4826  unsigned VAArgOffset = VAArgBase;
4827  const DataLayout &DL = F.getParent()->getDataLayout();
4828  for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
4829  ++ArgIt) {
4830  Value *A = *ArgIt;
4831  unsigned ArgNo = CB.getArgOperandNo(ArgIt);
4832  bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4833  bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
4834  if (IsByVal) {
4835  assert(A->getType()->isPointerTy());
4836  Type *RealTy = CB.getParamByValType(ArgNo);
4837  uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4838  MaybeAlign ArgAlign = CB.getParamAlign(ArgNo);
4839  if (!ArgAlign || *ArgAlign < Align(8))
4840  ArgAlign = Align(8);
4841  VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4842  if (!IsFixed) {
4843  Value *Base = getShadowPtrForVAArgument(
4844  RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
4845  if (Base) {
4846  Value *AShadowPtr, *AOriginPtr;
4847  std::tie(AShadowPtr, AOriginPtr) =
4848  MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
4849  kShadowTLSAlignment, /*isStore*/ false);
4850 
4851  IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
4852  kShadowTLSAlignment, ArgSize);
4853  }
4854  }
4855  VAArgOffset += alignTo(ArgSize, 8);
4856  } else {
4857  Value *Base;
4858  uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4859  uint64_t ArgAlign = 8;
4860  if (A->getType()->isArrayTy()) {
4861  // Arrays are aligned to element size, except for long double
4862  // arrays, which are aligned to 8 bytes.
4863  Type *ElementTy = A->getType()->getArrayElementType();
4864  if (!ElementTy->isPPC_FP128Ty())
4865  ArgAlign = DL.getTypeAllocSize(ElementTy);
4866  } else if (A->getType()->isVectorTy()) {
4867  // Vectors are naturally aligned.
4868  ArgAlign = DL.getTypeAllocSize(A->getType());
4869  }
4870  if (ArgAlign < 8)
4871  ArgAlign = 8;
4872  VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4873  if (DL.isBigEndian()) {
4874  // Adjusting the shadow for argument with size < 8 to match the placement
4875  // of bits in big endian system
4876  if (ArgSize < 8)
4877  VAArgOffset += (8 - ArgSize);
4878  }
4879  if (!IsFixed) {
4880  Base = getShadowPtrForVAArgument(A->getType(), IRB,
4881  VAArgOffset - VAArgBase, ArgSize);
4882  if (Base)
4883  IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4884  }
4885  VAArgOffset += ArgSize;
4886  VAArgOffset = alignTo(VAArgOffset, 8);
4887  }
4888  if (IsFixed)
4889  VAArgBase = VAArgOffset;
4890  }
4891 
4892  Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(),
4893  VAArgOffset - VAArgBase);
4894  // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4895  // a new class member i.e. it is the total size of all VarArgs.
4896  IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4897  }
4898 
4899  /// Compute the shadow address for a given va_arg.
4900  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4901  unsigned ArgOffset, unsigned ArgSize) {
4902  // Make sure we don't overflow __msan_va_arg_tls.
4903  if (ArgOffset + ArgSize > kParamTLSSize)
4904  return nullptr;
4905  Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4906  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4907  return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4908  "_msarg");
4909  }
4910 
4911  void visitVAStartInst(VAStartInst &I) override {
4912  IRBuilder<> IRB(&I);
4913  VAStartInstrumentationList.push_back(&I);
4914  Value *VAListTag = I.getArgOperand(0);
4915  Value *ShadowPtr, *OriginPtr;
4916  const Align Alignment = Align(8);
4917  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4918  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4919  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4920  /* size */ 8, Alignment, false);
4921  }
4922 
4923  void visitVACopyInst(VACopyInst &I) override {
4924  IRBuilder<> IRB(&I);
4925  Value *VAListTag = I.getArgOperand(0);
4926  Value *ShadowPtr, *OriginPtr;
4927  const Align Alignment = Align(8);
4928  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4929  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4930  // Unpoison the whole __va_list_tag.
4931  // FIXME: magic ABI constants.
4932  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4933  /* size */ 8, Alignment, false);
4934  }
4935 
4936  void finalizeInstrumentation() override {
4937  assert(!VAArgSize && !VAArgTLSCopy &&
4938  "finalizeInstrumentation called twice");
4939  IRBuilder<> IRB(MSV.FnPrologueEnd);
4940  VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4941  Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4942  VAArgSize);
4943 
4944  if (!VAStartInstrumentationList.empty()) {
4945  // If there is a va_start in this function, make a backup copy of
4946  // va_arg_tls somewhere in the function entry block.
4947  VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4948  IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4949  }
4950 
4951  // Instrument va_start.
4952  // Copy va_list shadow from the backup copy of the TLS contents.
4953  for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4954  CallInst *OrigInst = VAStartInstrumentationList[i];
4955  IRBuilder<> IRB(OrigInst->getNextNode());
4956  Value *VAListTag = OrigInst->getArgOperand(0);
4957  Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4958  Value *RegSaveAreaPtrPtr =
4959  IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4960  PointerType::get(RegSaveAreaPtrTy, 0));
4961  Value *RegSaveAreaPtr =
4962  IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4963  Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4964  const Align Alignment = Align(8);
4965  std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4966  MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4967  Alignment, /*isStore*/ true);
4968  IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4969  CopySize);
4970  }
4971  }
4972 };
4973 
4974 /// SystemZ-specific implementation of VarArgHelper.
4975 struct VarArgSystemZHelper : public VarArgHelper {
4976  static const unsigned SystemZGpOffset = 16;
4977  static const unsigned SystemZGpEndOffset = 56;
4978  static const unsigned SystemZFpOffset = 128;
4979  static const unsigned SystemZFpEndOffset = 160;
4980  static const unsigned SystemZMaxVrArgs = 8;
4981  static const unsigned SystemZRegSaveAreaSize = 160;
4982  static const unsigned SystemZOverflowOffset = 160;
4983  static const unsigned SystemZVAListTagSize = 32;
4984  static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
4985  static const unsigned SystemZRegSaveAreaPtrOffset = 24;
4986 
4987  Function &F;
4988  MemorySanitizer &MS;
4989  MemorySanitizerVisitor &MSV;
4990  Value *VAArgTLSCopy = nullptr;
4991  Value *VAArgTLSOriginCopy = nullptr;
4992  Value *VAArgOverflowSize = nullptr;
4993 
4994  SmallVector<CallInst *, 16> VAStartInstrumentationList;
4995 
4996  enum class ArgKind {
4997  GeneralPurpose,
4998  FloatingPoint,
4999  Vector,
5000  Memory,
5001  Indirect,
5002  };
5003 
5004  enum class ShadowExtension { None, Zero, Sign };
5005 
5006  VarArgSystemZHelper(Function &F, MemorySanitizer &MS,
5007  MemorySanitizerVisitor &MSV)
5008  : F(F), MS(MS), MSV(MSV) {}
5009 
5010  ArgKind classifyArgument(Type *T, bool IsSoftFloatABI) {
5011  // T is a SystemZABIInfo::classifyArgumentType() output, and there are
5012  // only a few possibilities of what it can be. In particular, enums, single
5013  // element structs and large types have already been taken care of.
5014 
5015  // Some i128 and fp128 arguments are converted to pointers only in the
5016  // back end.
5017  if (T->isIntegerTy(128) || T->isFP128Ty())
5018  return ArgKind::Indirect;
5019  if (T->isFloatingPointTy())
5020  return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5021  if (T->isIntegerTy() || T->isPointerTy())
5022  return ArgKind::GeneralPurpose;
5023  if (T->isVectorTy())
5024  return ArgKind::Vector;
5025  return ArgKind::Memory;
5026  }
5027 
5028  ShadowExtension getShadowExtension(const CallBase &CB, unsigned ArgNo) {
5029  // ABI says: "One of the simple integer types no more than 64 bits wide.
5030  // ... If such an argument is shorter than 64 bits, replace it by a full
5031  // 64-bit integer representing the same number, using sign or zero
5032  // extension". Shadow for an integer argument has the same type as the
5033  // argument itself, so it can be sign or zero extended as well.
5034  bool ZExt = CB.paramHasAttr(ArgNo, Attribute::ZExt);
5035  bool SExt = CB.paramHasAttr(ArgNo, Attribute::SExt);
5036  if (ZExt) {
5037  assert(!SExt);
5038  return ShadowExtension::Zero;
5039  }
5040  if (SExt) {
5041  assert(!ZExt);
5042  return ShadowExtension::Sign;
5043  }
5044  return ShadowExtension::None;
5045  }
5046 
5047  void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5048  bool IsSoftFloatABI = CB.getCalledFunction()
5049  ->getFnAttribute("use-soft-float")
5050  .getValueAsBool();
5051  unsigned GpOffset = SystemZGpOffset;
5052  unsigned FpOffset = SystemZFpOffset;
5053  unsigned VrIndex = 0;
5054  unsigned OverflowOffset = SystemZOverflowOffset;
5055  const DataLayout &DL = F.getParent()->getDataLayout();
5056  for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
5057  ++ArgIt) {
5058  Value *A = *ArgIt;
5059  unsigned ArgNo = CB.getArgOperandNo(ArgIt);
5060  bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
5061  // SystemZABIInfo does not produce ByVal parameters.
5062  assert(!CB.paramHasAttr(ArgNo, Attribute::ByVal));
5063  Type *T = A->getType();
5064  ArgKind AK = classifyArgument(T, IsSoftFloatABI);
5065  if (AK == ArgKind::Indirect) {
5066  T = PointerType::get(T, 0);
5067  AK = ArgKind::GeneralPurpose;
5068  }
5069  if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5070  AK = ArgKind::Memory;
5071  if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5072  AK = ArgKind::Memory;
5073  if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5074  AK = ArgKind::Memory;
5075  Value *ShadowBase = nullptr;
5076  Value *OriginBase = nullptr;
5077  ShadowExtension SE = ShadowExtension::None;
5078  switch (AK) {
5079  case ArgKind::GeneralPurpose: {
5080  // Always keep track of GpOffset, but store shadow only for varargs.
5081  uint64_t ArgSize = 8;
5082  if (GpOffset + ArgSize <= kParamTLSSize) {
5083  if (!IsFixed) {
5084  SE = getShadowExtension(CB, ArgNo);
5085  uint64_t GapSize = 0;
5086  if (SE == ShadowExtension::None) {
5087  uint64_t ArgAllocSize = DL.getTypeAllocSize(T);
5088  assert(ArgAllocSize <= ArgSize);
5089  GapSize = ArgSize - ArgAllocSize;
5090  }
5091  ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5092  if (MS.TrackOrigins)
5093  OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5094  }
5095  GpOffset += ArgSize;
5096  } else {
5097  GpOffset = kParamTLSSize;
5098  }
5099  break;
5100  }
5101  case ArgKind::FloatingPoint: {
5102  // Always keep track of FpOffset, but store shadow only for varargs.
5103  uint64_t ArgSize = 8;
5104  if (FpOffset + ArgSize <= kParamTLSSize) {
5105  if (!IsFixed) {
5106  // PoP says: "A short floating-point datum requires only the
5107  // left-most 32 bit positions of a floating-point register".
5108  // Therefore, in contrast to AK_GeneralPurpose and AK_Memory,
5109  // don't extend shadow and don't mind the gap.
5110  ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5111  if (MS.TrackOrigins)
5112  OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5113  }
5114  FpOffset += ArgSize;
5115  } else {
5116  FpOffset = kParamTLSSize;
5117  }
5118  break;
5119  }
5120  case ArgKind::Vector: {
5121  // Keep track of VrIndex. No need to store shadow, since vector varargs
5122  // go through AK_Memory.
5123  assert(IsFixed);
5124  VrIndex++;
5125  break;
5126  }
5127  case ArgKind::Memory: {
5128  // Keep track of OverflowOffset and store shadow only for varargs.
5129  // Ignore fixed args, since we need to copy only the vararg portion of
5130  // the overflow area shadow.
5131  if (!IsFixed) {
5132  uint64_t ArgAllocSize = DL.getTypeAllocSize(T);
5133  uint64_t ArgSize = alignTo(ArgAllocSize, 8);
5134  if (OverflowOffset + ArgSize <= kParamTLSSize) {
5135  SE = getShadowExtension(CB, ArgNo);
5136  uint64_t GapSize =
5137  SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5138  ShadowBase =
5139  getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5140  if (MS.TrackOrigins)
5141  OriginBase =
5142  getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5143  OverflowOffset += ArgSize;
5144  } else {
5145  OverflowOffset = kParamTLSSize;
5146  }
5147  }
5148  break;
5149  }
5150  case ArgKind::Indirect:
5151  llvm_unreachable("Indirect must be converted to GeneralPurpose");